├── .github
    ├── CODEOWNERS
    └── workflows
    │   ├── ci.yml
    │   └── jekyll-gh-pages.yml
├── .gitignore
├── 404.html
├── CNAME
├── CONTRIBUTING.md
├── Cargo.lock
├── Cargo.toml
├── Gemfile
├── LICENSE
├── README.md
├── _config.yml
├── _layouts
    └── default.html
├── about.markdown
├── favicon.png
├── file-change-consumer
    ├── Cargo.toml
    └── src
    │   └── main.rs
├── file-change-router
    ├── Cargo.toml
    └── src
    │   └── main.rs
├── file-tree-merge
    ├── Cargo.toml
    ├── src
    │   ├── apply_change.rs
    │   ├── change_tree.rs
    │   ├── change_tree_merge.rs
    │   ├── lib.rs
    │   ├── main.rs
    │   ├── path_walker.rs
    │   └── tree_creator.rs
    └── tests
    │   └── scripts
    │       ├── cleanup.sh
    │       ├── common.sh
    │       ├── setup.sh
    │       ├── tests.sh
    │       └── tests
    │           ├── add.sh
    │           ├── add_modify_delete.sh
    │           ├── add_modify_delete_multiple_paths.sh
    │           ├── add_more.sh
    │           ├── delete.sh
    │           ├── delete_many.sh
    │           ├── dry-run.sh
    │           ├── large
    │               └── add_modify_delete.sh
    │           ├── modify.sh
    │           ├── modify_many.sh
    │           └── no-crc.sh
├── file-watcher
    ├── Cargo.toml
    └── src
    │   └── main.rs
├── p2p-transfer
    ├── .cargo
    │   └── config.toml
    ├── .gitignore
    ├── .typos.toml
    ├── Cargo.lock
    ├── Cargo.toml
    ├── LICENSE-APACHE
    ├── LICENSE-MIT
    ├── README.md
    ├── Trunk.toml
    ├── assets
    │   ├── favicon.ico
    │   ├── icon-1024.png
    │   ├── icon-256.png
    │   ├── icon_ios_touch_192.png
    │   ├── manifest.json
    │   ├── maskable_icon_x512.png
    │   └── sw.js
    ├── check.sh
    ├── fill_template.ps1
    ├── fill_template.sh
    ├── flake.nix
    ├── index.html
    ├── rust-toolchain
    └── src
    │   ├── app.rs
    │   ├── lib.rs
    │   ├── main.rs
    │   └── node.rs
└── website
    ├── pages
        ├── compare.md
        ├── features.md
        ├── features
        │   ├── analytics.md
        │   ├── antivirus.md
        │   ├── automation.md
        │   ├── backup.md
        │   ├── browser-app.md
        │   ├── cleanup.md
        │   ├── clients.md
        │   ├── emails.md
        │   ├── encrypt.md
        │   ├── global-view.md
        │   ├── integrations.md
        │   ├── large-files.md
        │   ├── links.md
        │   ├── local-app.md
        │   ├── local-view.md
        │   ├── photos.md
        │   ├── receive.md
        │   ├── search.md
        │   ├── share-external.md
        │   ├── share-local.md
        │   ├── share-providers.md
        │   ├── status.md
        │   ├── supported-providers.md
        │   ├── sync-local.md
        │   ├── sync-providers.md
        │   └── versioning.md
        ├── how-it-works.md
        ├── name.md
        ├── poc-demo.md
        ├── stack.md
        └── use-cases.md
    └── resources
        ├── app.png
        ├── automation.png
        ├── backup-emails-locally.png
        ├── backup-emails-with-service.png
        ├── backup-local-files.png
        ├── backup-provider.png
        ├── clients.png
        ├── diagram-encrypt-share.png
        ├── diagram-receive.png
        ├── diagram-share-external.png
        ├── diagram-share-local.png
        ├── diagram-share-provider.png
        ├── diagram-sync-local.png
        ├── diagram-sync-providers.png
        ├── frame-analytics.png
        ├── frame-local-view.png
        ├── frame-receive.png
        ├── mvp.png
        ├── poc.png
        ├── receive-file-with-provider.png
        ├── receive-local-file.png
        ├── services.png
        ├── share-encrypte-with-external-user.png
        ├── share-encryptyed-with-service-user.png
        ├── share-local.png
        ├── share-providers.png
        ├── sync-emails.png
        ├── sync-encrypted.png
        ├── sync-local.png
        ├── sync-providers.png
        └── syncoxiders-icon-20p.png


/.github/CODEOWNERS:
--------------------------------------------------------------------------------
 1 | # This is a comment.
 2 | # Each line is a file pattern followed by one or more owners.
 3 | 
 4 | # These owners will be the default owners for everything in
 5 | # the repo. Unless a later match takes precedence,
 6 | # @global-owner1 and @global-owner2 will be requested for
 7 | # review when someone opens a pull request.
 8 | *       @radumarias
 9 | 
10 | # Order is important; the last matching pattern takes the most
11 | # precedence. When someone opens a pull request that only
12 | # modifies JS files, only @js-owner and not the global
13 | # owner(s) will be requested for a review.
14 | # *.js    @js-owner #This is an inline comment.
15 | 
16 | # You can also use email addresses if you prefer. They'll be
17 | # used to look up users just like we do for commit author
18 | # emails.
19 | # *.go docs@example.com
20 | 
21 | # Teams can be specified as code owners as well. Teams should
22 | # be identified in the format @org/team-name. Teams must have
23 | # explicit write access to the repository. In this example,
24 | # the octocats team in the octo-org organization owns all .txt files.
25 | # *.txt @octo-org/octocats
26 | 
27 | # In this example, @doctocat owns any files in the build/logs
28 | # directory at the root of the repository and any of its
29 | # subdirectories.
30 | # /build/logs/ @doctocat
31 | 
32 | # The `docs/*` pattern will match files like
33 | # `docs/getting-started.md` but not further nested files like
34 | # `docs/build-app/troubleshooting.md`.
35 | # docs/*  docs@example.com
36 | 
37 | # In this example, @octocat owns any file in an apps directory
38 | # anywhere in your repository.
39 | # apps/ @octocat
40 | 
41 | # In this example, @doctocat owns any file in the `/docs`
42 | # directory in the root of your repository and any of its
43 | # subdirectories.
44 | # /docs/ @doctocat
45 | 
46 | # In this example, any change inside the `/scripts` directory
47 | # will require approval from @doctocat or @octocat.
48 | # /scripts/ @doctocat @octocat
49 | 
50 | # In this example, @octocat owns any file in a `/logs` directory such as
51 | # `/build/logs`, `/scripts/logs`, and `/deeply/nested/logs`. Any changes
52 | # in a `/logs` directory will require approval from @octocat.
53 | # **/logs @octocat
54 | 
55 | # In this example, @octocat owns any file in the `/apps`
56 | # directory in the root of your repository except for the `/apps/github`
57 | # subdirectory, as its owners are left empty.
58 | # /apps/ @octocat
59 | # /apps/github
60 | 
61 | # In this example, @octocat owns any file in the `/apps`
62 | # directory in the root of your repository except for the `/apps/github`
63 | # subdirectory, as this subdirectory has its own owner @doctocat
64 | # /apps/ @octocat
65 | # /apps/github @doctocat
66 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
 1 | name: ci
 2 | on:
 3 |   push:
 4 |     branches: [ main, release ]
 5 |   pull_request:
 6 |     branches: [ main, release ]
 7 | concurrency:
 8 |   group: ${{ github.workflow }}-${{ github.ref }}
 9 |   cancel-in-progress: true
10 |   
11 | env:
12 |   CARGO_TERM_COLOR: always
13 |   CARGO_INCREMENTAL: 0 # TODO: remove this when we cache the builds
14 | 
15 | jobs:
16 |   ci:
17 |     runs-on: ${{ matrix.runner }}
18 |     strategy:
19 |       matrix:
20 |         runner: [ubuntu-latest, macos-latest, windows-latest]
21 |     steps:
22 |     - uses: actions/checkout@v4
23 | 
24 |     - name: Setup Rust
25 |       run: rustup update
26 | 
27 |     - name: build
28 |       run: cargo build --all --release
29 | 
30 |     - name: test
31 |       run: |
32 |         cargo test --release --all-targets
33 | 
34 |         #cd file-tree-merge/tests/scripts
35 |         #sh tests.sh
36 |       shell: bash
37 | 
38 |     - name: upload binary
39 |       if: ${{ !startsWith(matrix.runner, 'windows') }}
40 |       uses: actions/upload-artifact@v4
41 |       with:
42 |         name: syncoxiders_${{ matrix.runner }}
43 |         path: target/release/syncoxiders
44 | 
45 |     - name: upload binary
46 |       if: startsWith(matrix.runner, 'windows')
47 |       uses: actions/upload-artifact@v4
48 |       with:
49 |         name: syncoxiders_${{ matrix.runner }}
50 |         path: target/release/syncoxiders.exe


--------------------------------------------------------------------------------
/.github/workflows/jekyll-gh-pages.yml:
--------------------------------------------------------------------------------
 1 | # Sample workflow for building and deploying a Jekyll site to GitHub Pages
 2 | name: Deploy Jekyll with GitHub Pages dependencies preinstalled
 3 | 
 4 | on:
 5 |   # Runs on pushes targeting the default branch
 6 |   push:
 7 |     branches: ["main"]
 8 | 
 9 |   # Allows you to run this workflow manually from the Actions tab
10 |   workflow_dispatch:
11 | 
12 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
13 | permissions:
14 |   contents: read
15 |   pages: write
16 |   id-token: write
17 | 
18 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
19 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
20 | concurrency:
21 |   group: "pages"
22 |   cancel-in-progress: true
23 | 
24 | jobs:
25 |   # Build job
26 |   build:
27 |     runs-on: ubuntu-latest
28 |     steps:
29 |       - name: Checkout
30 |         uses: actions/checkout@v4
31 |       - name: Setup Pages
32 |         uses: actions/configure-pages@v5
33 |       - name: Build with Jekyll
34 |         uses: actions/jekyll-build-pages@v1
35 |         with:
36 |           source: ./
37 |           destination: ./_site
38 |       - name: Upload artifact
39 |         uses: actions/upload-pages-artifact@v3
40 | 
41 |   # Deployment job
42 |   deploy:
43 |     environment:
44 |       name: github-pages
45 |       url: ${{ steps.deployment.outputs.page_url }}
46 |     runs-on: ubuntu-latest
47 |     needs: build
48 |     steps:
49 |       - name: Deploy to GitHub Pages
50 |         id: deployment
51 |         uses: actions/deploy-pages@v4
52 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | target/
2 | Gemfile.lock
3 | .idea/
4 | 


--------------------------------------------------------------------------------
/404.html:
--------------------------------------------------------------------------------
 1 | ---
 2 | permalink: /404.html
 3 | layout: default
 4 | ---
 5 | 
 6 | <style type="text/css" media="screen">
 7 |   .container {
 8 |     margin: 10px auto;
 9 |     max-width: 600px;
10 |     text-align: center;
11 |   }
12 |   h1 {
13 |     margin: 30px 0;
14 |     font-size: 4em;
15 |     line-height: 1;
16 |     letter-spacing: -1px;
17 |   }
18 | </style>
19 | 
20 | <div class="container">
21 |   <h1>404</h1>
22 | 
23 |   <p><strong>Page not found :(</strong></p>
24 |   <p>The requested page could not be found.</p>
25 | </div>
26 | 


--------------------------------------------------------------------------------
/CNAME:
--------------------------------------------------------------------------------
1 | syncoxide.rs


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ## How to contribute
2 | 
3 | You can use this [CONTRIBUTING](https://github.com/radumarias/rencfs/blob/main/CONTRIBUTING.md) guide as a reference, skipping the steps that don't apply.
4 | 


--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [workspace]
 2 | members = ["file-change-consumer", "file-change-router", "file-tree-merge", "file-watcher", "p2p-transfer"]
 3 | 
 4 | resolver = "2"
 5 | 
 6 | [workspace.package]
 7 | description = "Cloud files Sync, Sharing, Backup and Encryption solution written in Rust."
 8 | version = "0.1.0"
 9 | edition = "2021"
10 | license = "MIT OR Apache-2.0"
11 | authors = ["Radu Marias <radumarias@gmail.com>"]
12 | homepage = "https://github.com/radumarias/syncoxiders"
13 | repository = "https://github.com/radumarias/syncoxiders"
14 | readme = "README.md"
15 | keywords = ["sync", "share", "cloud", "filesystem", "encryption"]
16 | categories = ["cryptography", "filesystem"]
17 | documentation = "https://docs.rs/syncoxiders"
18 | exclude = [".github/"]
19 | 
20 | [profile.release]
21 | panic = "abort"
22 | 
23 | [workspace.dependencies]
24 | colored = "2.1.0"
25 | rayon = "^1.10.0"
26 | anyhow = "1.0.80"


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
 1 | source "https://rubygems.org"
 2 | # Hello! This is where you manage which Jekyll version is used to run.
 3 | # When you want to use a different version, change it below, save the
 4 | # file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
 5 | #
 6 | #     bundle exec jekyll serve
 7 | #
 8 | # This will help ensure the proper Jekyll version is running.
 9 | # Happy Jekylling!
10 | # gem "jekyll", "~> 4.3.3"
11 | 
12 | gem "github-pages", "~> 3.9.5", group: :jekyll_plugins
13 | 
14 | # This is the default theme for new Jekyll sites. You may change this to anything you like.
15 | gem "minima", "~> 2.5"
16 | # If you want to use GitHub Pages, remove the "gem "jekyll"" above and
17 | # uncomment the line below. To upgrade, run `bundle update github-pages`.
18 | # gem "github-pages", group: :jekyll_plugins
19 | # If you have any plugins, put them here!
20 | group :jekyll_plugins do
21 |   gem "jekyll-feed", "~> 0.12"
22 | end
23 | 
24 | # Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem
25 | # and associated library.
26 | platforms :mingw, :x64_mingw, :mswin, :jruby do
27 |   gem "tzinfo", ">= 1", "< 3"
28 |   gem "tzinfo-data"
29 | end
30 | 
31 | # Performance-booster for watching directories on Windows
32 | gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin]
33 | 
34 | # Lock `http_parser.rb` gem to `v0.6.x` on JRuby builds since newer versions of the gem
35 | # do not have a Java counterpart.
36 | gem "http_parser.rb", "~> 0.6.0", :platforms => [:jruby]
37 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # ![](website/resources/syncoxiders-icon-20p.png) SyncOxiders
  2 | 
  3 | Cloud file and email Sync, file Sharing, inter-cloud Encryption and Backup solution written in Rust and modern technologies.
  4 | 
  5 | The purpose of this project is to offer an easy and reliable way to sync files and emails between multiple providers and share files between multiple storage providers (like Google Drive, Dropbox, S3, SFTP servers, ...) and local files. Also simple way for backup of your files and emails and encryption. 
  6 | It offers real time sync (from simple Copy One-way to Two-way Sync) all handled in the cloud, without the explicit need of local clients.
  7 | 
  8 | > [!WARNING]  
  9 | > For now it's in PoC phase, it has some or the core components, like encryption, basic Google Drive client and a basic [CLI app](website/pages/poc-demo.md#using-cli).
 10 | 
 11 | It's using [rencfs](https://github.com/radumarias/rencfs) for encryption and [gdrive-rs](https://github.com/radumarias/gdrive-rs) for accesing Google Drive.
 12 | 
 13 | > [!IMPORTANT]  
 14 | > It you could take this [**SURVEY**](https://forms.gle/qgnWBJhzCpzPLSmv5) to express your opinion about the current solution and offer your opinion on what features you would want from a service like this it would help a lot.
 15 | 
 16 | You can see rhe [results](https://docs.google.com/forms/d/1d4V8BZB7TGp08NhY6_L0kUgcGe0glRFOjp4rjrt7_bs/viewanalytics?chromeless=1) of the survey.
 17 | 
 18 | > [!NOTE]  
 19 | In many cases we'll use present tense for several functionality, even though they are not yet implemented, it's used to give an idea of what the system could be.
 20 | 
 21 | [What's with the name](website/pages/name.md)
 22 | 
 23 | # PoC
 24 | 
 25 | You can see more [details](website/pages/poc-demo.md) on what's working now, play with the [CLI app](website/pages/poc-demo.md#using-cli) and see a short [demo](https://www.youtube.com/watch?v=JHQC1XpCzQw).
 26 | 
 27 | Working on having these in up to 2 months:
 28 | - in `Docker` ability to sync 2 folders in the filesystem
 29 | - run `rclone` in `Docker` and mount `Google Drive` and `Dropbox` or `MS OneDrive` in 2 folders
 30 | - from CLI trigger a sync which will make a Two-Way sync between the folders, first sync will do a union between the 2, no delete or rename will be performed
 31 | - do some changes in both local folders and trigger a sync, from now on it will propagate deletes and renames also
 32 | - do some changes on the remotes, trigger a sync and make sure changes are propagated in both local folders and on remotes
 33 | - save files encrypted using `rencfs`
 34 |   - this will save encrypted data on the mount points of `rclone` and expose them with `FUSE`
 35 | 
 36 | <img src="website/resources/poc.png" style = "width: 100%; max-width: 1000px; height: auto;">
 37 | 
 38 | # MVP
 39 | 
 40 | It would be possible to have something in about 6 months with this functionality:
 41 | - integration with `Google Drive` and `Dropbox` or `MS OneDrive`
 42 | - Sync between the two
 43 | - Share files from providers with another user
 44 | - browser app with basic functionality like:
 45 |   - adding providers
 46 |   - setup sync rules
 47 |   - share between providers
 48 | - some basic functionality of sharing local files, no sync between them
 49 | - encryption
 50 | 
 51 | For this phase we will still be using `rclone` to access providers, this is to simplify the access. But for future plan is to:
 52 | - implement our own clients that will directly communicate with the providers API
 53 | - receive changes in close to real-time
 54 | - store the changes in `Kafka` and window them (group them) with `Flink`
 55 | - feed them as changes tree to the `files tree merge` algorithm which will do the merge, resolve conflicts and applying changes to the other providers or local files
 56 | 
 57 | <img src="website/resources/mvp.png" style = "width: 100%; max-width: 1000px; height: auto;">
 58 | 
 59 | # The big picture
 60 | 
 61 | This is what it's planned to have in the end.
 62 | 
 63 | <img src="website/resources/services.png" style = "width: 100%; max-width: 1000px; height: auto;">
 64 | 
 65 | [Use cases](website/pages/use-cases.md)
 66 | 
 67 | [Features](website/pages/features.md)
 68 | 
 69 | [What separates it from other products](website/pages/compare.md)
 70 | 
 71 | [How it works](website/pages/how-it-works.md)
 72 | 
 73 | [Tech stack](website/pages/stack.md)
 74 | 
 75 | # Contribute
 76 | 
 77 | Feel free to fork it, change and use it in any way that you want. If you build something interesting and feel like sharing pull requests are always appreciated.
 78 | 
 79 | ## How to contribute
 80 | 
 81 | ### Browser
 82 | 
 83 | If you want to give it a quick try and not setup anything locally you can  
 84 | [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/radumarias/syncoxiders)
 85 | 
 86 | [![Open Rustlings On Codespaces](https://github.com/codespaces/badge.svg)](https://github.com/codespaces/new/?repo=radumarias%2Fsyncoxiders&ref=main)
 87 | 
 88 | You can compile it, run it, and give it a quick try in browser. After you start it from above
 89 | 
 90 | ```bash
 91 | sudo apt-get update && sudo apt-get install fuse3
 92 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 93 | mkdir sync && mkdir sync/repo sync/a sync/b
 94 | ```
 95 | 
 96 | Add some files in `sync/a` and then run th sync
 97 | 
 98 | ```bash
 99 | cargo run --release --bin syncoxiders -- --repo sync/repo sync/a sync/b
100 | ```
101 | 
102 | Now check `sync/b` it should have same content as `file/a`.
103 | 
104 | For now this **is working**
105 | - sync files: create, delete, update, move
106 | - sync one to many, you can put several paths, it will sync from path1 to all others
107 | 
108 | It **DOESN'T** work
109 | - folders sync: create, delete, rename, will be fixed soon
110 | 
111 | ### Locally
112 | 
113 | #### Getting the sources
114 | 
115 | ```bash
116 | git clone git@github.com:radumarias/syncoxiders.git && cd syncoxiders
117 | ````
118 | 
119 | #### Dependencies
120 | 
121 | ##### Rust
122 | 
123 | To build from source, you need to have Rust installed, you can see more details on how to install
124 | it [here](https://www.rust-lang.org/tools/install).
125 | 
126 | ```bash
127 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
128 | ````
129 | 
130 | Accordingly, it is customary for Rust developers to include this directory in their `PATH` environment variable.
131 | During installation `rustup` will attempt to configure the `PATH`. Because of differences between platforms, command
132 | shells,
133 | and bugs in `rustup`, the modifications to `PATH` may not take effect until the console is restarted, or the user is
134 | logged out, or it may not succeed at all.
135 | 
136 | If, after installation, running `rustc --version` in the console fails, this is the most likely reason.
137 | In that case please add it to the `PATH` manually.
138 | 
139 | Project is setup to use `nightly` toolchain in `rust-toolchain.toml`, on first build you will see it fetch the nightly.
140 | 
141 | Make sure to add this you your `$PATH` too
142 | 
143 | ```bash
144 | export PATH="$PATH::$HOME/.cargo/bin"
145 | ```
146 | 
147 | ##### Other dependencies
148 | 
149 | Also, these deps are required (or based on your distribution):
150 | 
151 | ###### Arch
152 | 
153 | ```bash
154 | sudo pacman -Syu && sudo pacman -S base-devel act
155 | ```
156 | 
157 | ###### Ubuntu
158 | 
159 | ```bash
160 | sudo apt-get update && sudo apt-get install build-essential act
161 | ```
162 | 
163 | ###### Fedora
164 | 
165 | ```bash
166 | sudo dnf update && sudo dnf install && dnf install @development-tools act
167 | ```
168 | 
169 | #### Build for debug
170 | 
171 | ```bash
172 | cargo build
173 | ```
174 | 
175 | #### Build release
176 | 
177 | ```bash
178 | cargo build --release
179 | ```
180 | 
181 | ### Run
182 | 
183 | ```bash
184 | cargo run --release --bin syncoxiders -- --repo REPO A B
185 | ```
186 | 
187 | ### Developing inside a Container
188 | 
189 | See here how to configure for [RustRover](https://www.jetbrains.com/help/rust/connect-to-devcontainer.html) and for [VsCode](https://code.visualstudio.com/docs/devcontainers/containers).
190 | 
191 | You can use the `.devcontainer` directory from the project to start a container with all the necessary tools to build
192 | and run the app.
193 | 
194 | Please see [CONTRIBUTING.md](https://github.com/radumarias/rencfs/blob/main/CONTRIBUTING.md).
195 | 
196 | # Minimum Supported Rust Version (MSRV)
197 | 
198 | The minimum supported version is `1.75`.
199 | 


--------------------------------------------------------------------------------
/_config.yml:
--------------------------------------------------------------------------------
1 | title: SyncOxiders homepage
2 | remote_theme: pages-themes/hacker@v0.2.0
3 | plugins:
4 |   - jekyll-remote-theme
5 |   - jekyll-seo-tag
6 | 


--------------------------------------------------------------------------------
/_layouts/default.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="{{ site.lang | default: "en-US" }}">
 3 |   <head>
 4 |     <link rel="shortcut icon" type="image/png" href="favicon.png">
 5 |     <meta charset="UTF-8">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1">
 7 | 
 8 | {% seo %}
 9 |     <link rel="preconnect" href="https://fonts.gstatic.com">
10 |     <link rel="preload" href="https://fonts.googleapis.com/css?family=Open+Sans:400,700&display=swap" as="style" type="text/css" crossorigin>
11 |     <meta name="viewport" content="width=device-width, initial-scale=1">
12 |     <meta name="theme-color" content="#157878">
13 |     <meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
14 |     <link rel="stylesheet" href="{{ '/assets/css/style.css?v=' | append: site.github.build_revision | relative_url }}">
15 |     {% include head-custom.html %}
16 |   </head>
17 |   <body style="margin: auto; padding: 10px; max-width: 1000px;">
18 |     <header class="page-header" role="banner">
19 |       <h1 class="project-name">{{ page.title | default: site.title | default: site.github.repository_name }}</h1>
20 |       <h2 class="project-tagline">{{ page.description | default: site.description | default: site.github.project_tagline }}</h2>
21 |       {% if site.github.is_project_page %}
22 |         <a href="{{ site.github.repository_url }}" class="btn">View on GitHub</a>
23 |       {% endif %}
24 |     </header>
25 |     <main id="content" role="main">
26 |       <div class="w3-container">
27 |         <p>{{ content }}</p>
28 |       </div>
29 |       
30 |       <footer class="site-footer">
31 |         {% if site.github.is_project_page %}
32 |           <span class="site-footer-owner"><a href="{{ site.github.repository_url }}">{{ site.github.repository_name }}</a> is maintained by <a href="{{ site.github.owner_url }}">{{ site.github.owner_name }}</a>.</span>
33 |         {% endif %}
34 |         <span class="site-footer-credits">This page was generated by <a href="https://pages.github.com">GitHub Pages</a>.</span>
35 |       </footer>      
36 |     </main>
37 |   </body>
38 | </html>
39 | 


--------------------------------------------------------------------------------
/about.markdown:
--------------------------------------------------------------------------------
 1 | ---
 2 | layout: page
 3 | title: About
 4 | permalink: /about/
 5 | ---
 6 | 
 7 | Cloud file and email Sync, file Sharing, Backup and Encryption solution written in Rust.
 8 | 
 9 | The purpose of this project is to offer an easy and reliable way to sync files and emails
10 | between multiple providers and share files between multiple storage providers (like Google Drive, Dropbox, S3, SFTP servers, ...)
11 | and local files. Also simple way for backup of your files and emails and encryption.
12 | It offers real time sync (from simple Copy One-way to Two-way Sync) all handled in the cloud, without the explicit need of local clients.


--------------------------------------------------------------------------------
/favicon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/favicon.png


--------------------------------------------------------------------------------
/file-change-consumer/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "file-change-consumer"
 3 | description.workspace = true
 4 | version.workspace = true
 5 | edition.workspace = true
 6 | license.workspace = true
 7 | authors.workspace = true
 8 | homepage.workspace = true
 9 | repository.workspace = true
10 | readme.workspace = true
11 | keywords.workspace = true
12 | categories.workspace = true
13 | documentation.workspace = true
14 | exclude.workspace = true
15 | 
16 | [dependencies]
17 | 


--------------------------------------------------------------------------------
/file-change-consumer/src/main.rs:
--------------------------------------------------------------------------------
1 | fn main() {
2 |     println!("Hello, world!");
3 | }
4 | 


--------------------------------------------------------------------------------
/file-change-router/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "file-change-router"
 3 | description.workspace = true
 4 | version.workspace = true
 5 | edition.workspace = true
 6 | license.workspace = true
 7 | authors.workspace = true
 8 | homepage.workspace = true
 9 | repository.workspace = true
10 | readme.workspace = true
11 | keywords.workspace = true
12 | categories.workspace = true
13 | documentation.workspace = true
14 | exclude.workspace = true
15 | 
16 | [dependencies]
17 | 


--------------------------------------------------------------------------------
/file-change-router/src/main.rs:
--------------------------------------------------------------------------------
1 | fn main() {
2 |     println!("Hello, world!");
3 | }
4 | 


--------------------------------------------------------------------------------
/file-tree-merge/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "file-tree-merge"
 3 | description.workspace = true
 4 | version.workspace = true
 5 | edition.workspace = true
 6 | license.workspace = true
 7 | authors.workspace = true
 8 | homepage.workspace = true
 9 | repository.workspace = true
10 | readme.workspace = true
11 | keywords.workspace = true
12 | categories.workspace = true
13 | documentation.workspace = true
14 | exclude.workspace = true
15 | 
16 | [[bin]]
17 | name = "syncoxiders"
18 | path = "src/main.rs"
19 | 
20 | [dependencies]
21 | colored.workspace = true
22 | rayon.workspace = true
23 | 
24 | walkdir = "2.5.0"
25 | chksum-md5 = "0.0.0"
26 | git2 = "0.19.0"
27 | slab_tree = "0.3.2"
28 | tap = "1.0.1"
29 | regex = "1"
30 | clap = { version = "4", features = ["derive"] }
31 | crc32fast = "1.3"
32 | anyhow.workspace = true
33 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/apply_change.rs:
--------------------------------------------------------------------------------
  1 | use std::fs::File;
  2 | use std::path::Path;
  3 | use std::sync::atomic::{AtomicU64, Ordering};
  4 | use std::sync::{Arc, Mutex};
  5 | use std::{fs, io};
  6 | 
  7 | use anyhow::Result;
  8 | use colored::*;
  9 | use rayon::iter::ParallelIterator;
 10 | use rayon::prelude::IntoParallelRefIterator;
 11 | 
 12 | use crate::change_tree::Change;
 13 | use crate::change_tree_merge::{DstItems, HashKind, MergedChanges};
 14 | use crate::tree_creator::Item;
 15 | use crate::{crc_eq, file_hash, git_commit, retry};
 16 | 
 17 | pub fn apply(
 18 |     changes: MergedChanges,
 19 |     path1: &Path,
 20 |     path2: &Path,
 21 |     repo1: &Path,
 22 |     repo2: &Path,
 23 |     dry_run: bool,
 24 |     checksum: bool,
 25 |     crc: bool,
 26 |     print_all_changes: bool,
 27 | ) -> Result<()> {
 28 |     let (changes, (_, items_path1), items_path2) = changes;
 29 |     if changes.is_empty() {
 30 |         return Ok(());
 31 |     }
 32 |     if !path1.exists() {
 33 |         println!("{}", "path1 does not exist".red().bold());
 34 |         anyhow::bail!("path1 does not exist");
 35 |     }
 36 |     if !path2.exists() {
 37 |         println!("{}", "path2 does not exist".red().bold());
 38 |         anyhow::bail!("path2 does not exist");
 39 |     }
 40 |     println!(
 41 |         "{}",
 42 |         format!("Applying {} changes ...", changes.len()).cyan()
 43 |     );
 44 |     let total = AtomicU64::new(0);
 45 |     let synced = AtomicU64::new(0);
 46 |     let applied_size_since_commit = AtomicU64::new(0);
 47 |     let items_path1 = Arc::new(Mutex::new(items_path1));
 48 |     let items_path2 = Arc::new(Mutex::new(items_path2));
 49 |     let mut to_process = vec![];
 50 |     let process_batch_size = 1000;
 51 |     let print_batch_size = 100;
 52 |     let commit_after_size_bytes = 64 * 1024 * 1024;
 53 |     let git_lock = Arc::new(Mutex::new(()));
 54 |     for (change, path) in changes {
 55 |         to_process.push((change.clone(), path.to_string()));
 56 |         if to_process.len() % process_batch_size == 0 {
 57 |             // Process the collection in parallel
 58 |             let res: Vec<Result<()>> = to_process
 59 |                 .par_iter() // Convert the vector into a parallel iterator
 60 |                 .map(|(change, path)| {
 61 |                     process(
 62 |                         change,
 63 |                         path,
 64 |                         items_path1.clone(),
 65 |                         items_path2.clone(),
 66 |                         path1,
 67 |                         path2,
 68 |                         repo1,
 69 |                         repo2,
 70 |                         dry_run,
 71 |                         checksum,
 72 |                         crc,
 73 |                         &total,
 74 |                         git_lock.clone(),
 75 |                         print_batch_size,
 76 |                         &synced,
 77 |                         &applied_size_since_commit,
 78 |                         commit_after_size_bytes,
 79 |                         print_all_changes,
 80 |                     )
 81 |                 })
 82 |                 .collect();
 83 |             for e in res {
 84 |                 if let Err(err) = e {
 85 |                     println!(
 86 |                         "{}",
 87 |                         format!(
 88 |                             "Applied {}/{} changes",
 89 |                             synced.load(Ordering::SeqCst),
 90 |                             total.load(Ordering::SeqCst)
 91 |                         )
 92 |                         .green()
 93 |                         .bold()
 94 |                     );
 95 |                     Err(err)?
 96 |                 }
 97 |             }
 98 |             to_process.clear();
 99 |         }
100 |     }
101 |     // Process the collection in parallel
102 |     let res: Vec<Result<()>> = to_process
103 |         .par_iter() // Convert the vector into a parallel iterator
104 |         .map(|(change, path)| {
105 |             process(
106 |                 change,
107 |                 path,
108 |                 items_path1.clone(),
109 |                 items_path2.clone(),
110 |                 path1,
111 |                 path2,
112 |                 repo1,
113 |                 repo2,
114 |                 dry_run,
115 |                 checksum,
116 |                 crc,
117 |                 &total,
118 |                 git_lock.clone(),
119 |                 print_batch_size,
120 |                 &synced,
121 |                 &applied_size_since_commit,
122 |                 commit_after_size_bytes,
123 |                 print_all_changes,
124 |             )
125 |         })
126 |         .collect();
127 |     for e in res {
128 |         if let Err(err) = e {
129 |             if !dry_run {
130 |                 println!(
131 |                     "{}",
132 |                     format!(
133 |                         "Applied {}/{} changes",
134 |                         synced.load(Ordering::SeqCst),
135 |                         total.load(Ordering::SeqCst)
136 |                     )
137 |                     .green()
138 |                     .bold()
139 |                 );
140 |             }
141 |             Err(err)?
142 |         }
143 |     }
144 |     if !dry_run {
145 |         println!(
146 |             "{}",
147 |             format!(
148 |                 "Applied {}/{} changes",
149 |                 synced.load(Ordering::SeqCst),
150 |                 total.load(Ordering::SeqCst)
151 |             )
152 |             .green()
153 |             .bold()
154 |         );
155 |     }
156 |     retry(
157 |         || {
158 |             // git_add(&repo1.join(TREE_DIR), ".")?;
159 |             git_commit(repo1)?;
160 |             Ok(())
161 |         },
162 |         5,
163 |     )?;
164 |     Ok(())
165 | }
166 | 
167 | fn items_content_eq(
168 |     path1: &&Path,
169 |     a: &Item,
170 |     path2: &&Path,
171 |     b: &Item,
172 |     checksum: bool,
173 | ) -> io::Result<bool> {
174 |     if a.size == b.size && a.mtime == b.mtime {
175 |         if checksum {
176 |             let hash1 = file_hash(&path1.join(&a.path), HashKind::Md5)?;
177 |             let hash2 = file_hash(&path2.join(&b.path), HashKind::Md5)?;
178 |             Ok(hash1.eq(&hash2))
179 |         } else {
180 |             Ok(true)
181 |         }
182 |     } else {
183 |         Ok(false)
184 |     }
185 | }
186 | 
187 | fn process(
188 |     change: &Change,
189 |     path: &String,
190 |     items_path1: Arc<Mutex<DstItems>>,
191 |     path1_item2: Arc<Mutex<DstItems>>,
192 |     path1: &Path,
193 |     path2: &Path,
194 |     _repo1: &Path,
195 |     _repo2: &Path,
196 |     dry_run: bool,
197 |     checksum: bool,
198 |     crc: bool,
199 |     ctr: &AtomicU64,
200 |     _git_lock: Arc<Mutex<()>>,
201 |     batch_size: usize,
202 |     synced: &AtomicU64,
203 |     applied_size_since_commit: &AtomicU64,
204 |     _commit_after_size_bytes: i32,
205 |     print_all_changes: bool,
206 | ) -> Result<()> {
207 |     let dst = path2.join(path);
208 |     ctr.fetch_add(1, Ordering::SeqCst);
209 |     match change.clone() {
210 |         Change::Add | Change::Modify => {
211 |             if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
212 |                 if matches!(change, Change::Add) {
213 |                     println!(
214 |                         "{}",
215 |                         format!("{} '{}'", change.to_string().green(), path.green())
216 |                     );
217 |                 } else {
218 |                     println!(
219 |                         "{}",
220 |                         format!("{} '{}'", change.to_string().blue(), path.blue())
221 |                     );
222 |                 }
223 |             }
224 |             // check if it's the same as in dst
225 |             let mut add = true;
226 |             let guard = items_path1.lock().unwrap();
227 |             let path1_item = guard.get(path).unwrap();
228 |             if let Some(dst_item) = path1_item2.lock().unwrap().get(path) {
229 |                 if items_content_eq(&path1, &path1_item, &path2, &dst_item, checksum)? {
230 |                     add = false;
231 |                 }
232 |             }
233 |             if add {
234 |                 if dry_run {
235 |                     return Ok(());
236 |                 }
237 |                 retry(
238 |                     || {
239 |                         fs::create_dir_all(dst.parent().unwrap())?;
240 |                         fs::copy(path1.join(&path), dst.clone())?;
241 |                         File::set_times(&File::open(dst.clone())?, path1_item.times)?;
242 |                         File::open(dst.clone())?.sync_all()?;
243 |                         File::open(dst.parent().unwrap())?.sync_all()?;
244 |                         Ok(())
245 |                     },
246 |                     5,
247 |                 )?;
248 |                 if crc && !crc_eq(&path1.join(&path), &dst.clone())? {
249 |                     // todo: collect in errors
250 |                     println!(
251 |                         "{}",
252 |                         "   CRC check failed after transfer, aborting".red().bold()
253 |                     );
254 |                     anyhow::bail!("CRC check failed for `{path}` after transfer");
255 |                 }
256 |                 synced.fetch_add(1, Ordering::SeqCst);
257 |                 applied_size_since_commit.fetch_add(path1_item.size, Ordering::SeqCst);
258 |             } else if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
259 |                 println!(
260 |                     "{}",
261 |                     "   skip, already present in path2 with the same content".yellow()
262 |                 );
263 |             }
264 |         }
265 |         Change::Delete => {
266 |             if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
267 |                 println!("{} '{}'", change.to_string().red(), path.red().bold());
268 |             }
269 |             if dst.exists() {
270 |                 if dry_run {
271 |                     return Ok(());
272 |                 }
273 |                 retry(
274 |                     || {
275 |                         fs::remove_file(dst.clone())?;
276 |                         File::open(dst.parent().unwrap())?.sync_all()?;
277 |                         Ok(())
278 |                     },
279 |                     5,
280 |                 )?;
281 |             } else if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
282 |                 println!("{}", "  skip, not present in path2".yellow());
283 |             }
284 |             synced.fetch_add(1, Ordering::SeqCst);
285 |         }
286 |         Change::Rename(old_path) => {
287 |             if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
288 |                 println!("{} '{}'", change.to_string().magenta(), path.magenta());
289 |             }
290 |             let guard = items_path1.lock().unwrap();
291 |             let path1_item = guard.get(path).unwrap();
292 |             // todo: compare if old file hash in src is same as old file hash in dst
293 |             if path2.join(&old_path).exists() {
294 |                 if dry_run {
295 |                     return Ok(());
296 |                 }
297 |                 retry(
298 |                     || {
299 |                         fs::create_dir_all(dst.parent().unwrap())?;
300 |                         fs::rename(path2.join(&old_path), dst.clone())?;
301 |                         File::set_times(&File::open(dst.clone())?, path1_item.times)?;
302 |                         File::open(dst.clone())?.sync_all()?;
303 |                         File::open(dst.parent().unwrap())?.sync_all()?;
304 |                         Ok(())
305 |                     },
306 |                     5,
307 |                 )?;
308 |             } else {
309 |                 println!("{}", format!("  cannot R '{old_path}' -> '{path}', old file not present in path2. Will copy instead from path1 to the new destination").yellow());
310 |                 if dry_run {
311 |                     return Ok(());
312 |                 }
313 |                 retry(
314 |                     || {
315 |                         fs::create_dir_all(path2.join(path).parent().unwrap())?;
316 |                         fs::copy(path1.join(path), dst.clone())?;
317 |                         File::set_times(&File::open(dst.clone())?, path1_item.times)?;
318 |                         File::open(dst.clone())?.sync_all()?;
319 |                         File::open(dst.parent().unwrap())?.sync_all()?;
320 |                         Ok(())
321 |                     },
322 |                     5,
323 |                 )?;
324 |                 if crc && !crc_eq(&path1.join(path), &dst.clone())? {
325 |                     // todo: collect in errors
326 |                     println!(
327 |                         "{}",
328 |                         "   CRC check failed after transfer, aborting".red().bold()
329 |                     );
330 |                     anyhow::bail!("CRC check failed for `{path}` after transfer");
331 |                 }
332 |                 applied_size_since_commit.fetch_add(path1_item.size, Ordering::SeqCst);
333 |             }
334 |             synced.fetch_add(1, Ordering::SeqCst);
335 |         }
336 |         Change::Copy(old_path) => {
337 |             if ctr.load(Ordering::SeqCst) % batch_size as u64 == 0 || print_all_changes {
338 |                 println!("{} '{}'", change.to_string().blue(), path.blue());
339 |             }
340 |             let guard = items_path1.lock().unwrap();
341 |             let path1_item = guard.get(path).unwrap();
342 |             // todo: compare if old file hash in src is same as old file hash in dst
343 |             if path2.join(&old_path).exists() {
344 |                 if dry_run {
345 |                     return Ok(());
346 |                 }
347 |                 retry(
348 |                     || {
349 |                         fs::create_dir_all(dst.clone().parent().unwrap())?;
350 |                         fs::copy(path2.join(&old_path), dst.clone())?;
351 |                         File::set_times(&File::open(dst.clone())?, path1_item.times)?;
352 |                         File::open(dst.clone())?.sync_all()?;
353 |                         File::open(dst.parent().unwrap())?.sync_all()?;
354 |                         Ok(())
355 |                     },
356 |                     5,
357 |                 )?;
358 |             } else {
359 |                 println!("{}", format!("  cannot C '{old_path}' -> '{path}', old file not present in path2. Will copy instead from path1 to the new destination").yellow());
360 |                 if dry_run {
361 |                     return Ok(());
362 |                 }
363 |                 retry(
364 |                     || {
365 |                         fs::create_dir_all(dst.parent().unwrap())?;
366 |                         fs::copy(path1.join(path), dst.clone())?;
367 |                         File::set_times(&File::open(dst.clone())?, path1_item.times)?;
368 |                         File::open(dst.clone())?.sync_all()?;
369 |                         File::open(dst.parent().unwrap())?.sync_all()?;
370 |                         Ok(())
371 |                     },
372 |                     5,
373 |                 )?;
374 |             }
375 |             if crc && !crc_eq(&path1.join(path), &dst.clone())? {
376 |                 // todo: collect in errors
377 |                 println!(
378 |                     "{}",
379 |                     "   CRC check failed after transfer, aborting".red().bold()
380 |                 );
381 |                 anyhow::bail!("CRC check failed for `{path}` after transfer");
382 |             }
383 |             synced.fetch_add(1, Ordering::SeqCst);
384 |             applied_size_since_commit.fetch_add(path1_item.size, Ordering::SeqCst);
385 |         }
386 |     }
387 |     // todo: uncomment when we group changes per src
388 |     // let _guard = git_lock.lock().unwrap();
389 |     // git_add(&repo1.join(TREE_DIR), path)?;
390 |     // if applied_size_since_commit.load(Ordering::SeqCst) > commit_after_size_bytes as u64 {
391 |     //     println!("{}", "Checkpointing applied changes ...".cyan());
392 |     //     git_commit(repo1)?;
393 |     //     applied_size_since_commit.store(0, Ordering::SeqCst);
394 |     // }
395 |     Ok(())
396 | }
397 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/change_tree.rs:
--------------------------------------------------------------------------------
  1 | use std::collections::{BTreeMap, HashMap};
  2 | use std::path::Path;
  3 | use std::{fmt, fs, io};
  4 | 
  5 | use anyhow::Result;
  6 | use git2::{Repository, Status};
  7 | use slab_tree::{NodeId, Tree, TreeBuilder};
  8 | 
  9 | use crate::tree_creator::Item;
 10 | use crate::{git_restore_staged, tree_creator};
 11 | use crate::{git_status, retry};
 12 | 
 13 | pub type PathChanges = (ChangeTree, BTreeMap<String, Item>);
 14 | 
 15 | #[derive(Debug, Clone, PartialEq, Eq)]
 16 | pub enum Change {
 17 |     Add,
 18 |     Modify,
 19 |     Delete,
 20 |     Rename(String),
 21 |     Copy(String),
 22 | }
 23 | 
 24 | impl fmt::Display for Change {
 25 |     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 26 |         match self {
 27 |             Change::Add => write!(f, "A"),
 28 |             Change::Modify => write!(f, "M"),
 29 |             Change::Delete => write!(f, "D"),
 30 |             Change::Rename(name) => write!(f, "R '{}' ->", name),
 31 |             Change::Copy(name) => write!(f, "C '{}' ->", name),
 32 |         }
 33 |     }
 34 | }
 35 | 
 36 | impl From<Status> for Change {
 37 |     fn from(status: Status) -> Self {
 38 |         match status {
 39 |             Status::INDEX_NEW => Change::Add,
 40 |             Status::INDEX_MODIFIED => Change::Modify,
 41 |             Status::INDEX_DELETED => Change::Delete,
 42 |             Status::INDEX_RENAMED => Change::Rename("".to_string()),
 43 |             _ => {
 44 |                 println!("{status:?}");
 45 |                 unreachable!()
 46 |             }
 47 |         }
 48 |     }
 49 | }
 50 | 
 51 | #[derive(Debug)]
 52 | pub struct Node {
 53 |     pub path: String,
 54 |     pub item: Option<Item>,
 55 |     pub change: Option<Change>,
 56 | }
 57 | 
 58 | #[derive(Default)]
 59 | pub struct ChangeTree {
 60 |     pub new_repo: bool,
 61 |     pub tree: Tree<Node>,
 62 |     pub idx: HashMap<String, NodeId>,
 63 | }
 64 | 
 65 | pub fn build(items: Vec<Item>, repo: &Path) -> Result<PathChanges> {
 66 |     use crate::TREE_DIR;
 67 | 
 68 |     if repo.exists() && !repo.is_dir() {
 69 |         anyhow::bail!("Destination is not a directory");
 70 |     }
 71 | 
 72 |     retry(|| Ok(()), 5)?;
 73 | 
 74 |     if !repo.exists() {
 75 |         fs::create_dir_all(repo)?;
 76 |     }
 77 |     let mut new_repo = false;
 78 |     let init_repo = |_| {
 79 |         new_repo = true;
 80 |         Repository::init(repo)
 81 |     };
 82 |     let repo = match Repository::discover(repo).or_else(init_repo) {
 83 |         Ok(repo) => repo,
 84 |         Err(e) => Err(io::Error::new(io::ErrorKind::Other, e))?,
 85 |     };
 86 | 
 87 |     crate::git_add(repo.workdir().unwrap(), ".")?;
 88 |     let items_map: BTreeMap<_, _> = items
 89 |         .into_iter()
 90 |         .map(|data| (data.path.clone(), data))
 91 |         .collect();
 92 |     // let iter = repo
 93 |     //     .statuses(Some(
 94 |     //         git2::StatusOptions::new()
 95 |     //             .include_ignored(false)
 96 |     //             .recurse_untracked_dirs(true)
 97 |     //             .sort_case_insensitively(true)
 98 |     //             .include_untracked(true)
 99 |     //             .update_index(true),
100 |     //     ))
101 |     //     .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
102 |     // let mut statuses: Vec<_> = iter.iter().collect();
103 |     // statuses.sort_unstable_by_key(|status| status.path().unwrap().to_string());
104 |     let mut nodes_idx = HashMap::new();
105 |     let tree_builder = TreeBuilder::new();
106 |     // if statuses.is_empty() {
107 |     //     return Ok((
108 |     //         ChangeTree {
109 |     //             new_repo,
110 |     //             tree: tree_builder.build(),
111 |     //             idx: nodes_idx,
112 |     //         },
113 |     //         items_map,
114 |     //     ));
115 |     // }
116 |     let mut tree = tree_builder
117 |         .with_root(Node {
118 |             path: "".to_string(),
119 |             item: None,
120 |             change: None,
121 |         })
122 |         .build();
123 |     let root_id = tree.root_id().unwrap();
124 |     let root = tree.get_mut(root_id).unwrap();
125 |     nodes_idx.insert(
126 |         root.as_ref()
127 |             .data()
128 |             .item
129 |             .as_ref()
130 |             .map_or("".to_string(), |x| x.path.clone()),
131 |         root_id,
132 |     );
133 |     for line in git_status(&repo.workdir().unwrap().join(TREE_DIR))?.lines() {
134 |         let change = line.chars().take(1).collect::<String>();
135 |         let mut path = line.chars().skip(3).collect::<String>();
136 |         let change = match change.as_str() {
137 |             "M" => Change::Modify,
138 |             "A" => Change::Add,
139 |             "D" => Change::Delete,
140 |             "R" | "C" => {
141 |                 let capture = path
142 |                     .split(" -> ")
143 |                     .map(|x| x.to_string())
144 |                     .collect::<Vec<String>>();
145 |                 // let re = Regex::new(r"^(\S+)\s+->\s+(\S+)").expect("Failed to create regex");
146 |                 // let capture = re.captures(path.as_str()).expect("Failed to match");
147 |                 let mut old_path = capture[0].clone();
148 |                 if old_path.starts_with("\\\"") {
149 |                     old_path = old_path
150 |                         .strip_prefix("\\\"")
151 |                         .unwrap()
152 |                         .strip_suffix("\\\"")
153 |                         .unwrap()
154 |                         .to_string();
155 |                 } else if old_path.starts_with('\"') {
156 |                     old_path = old_path
157 |                         .strip_prefix('"')
158 |                         .unwrap()
159 |                         .strip_suffix('\"')
160 |                         .unwrap()
161 |                         .to_string();
162 |                 }
163 |                 path = capture[1].to_string();
164 |                 match change.as_str() {
165 |                     "R" => Change::Rename(old_path),
166 |                     "C" => Change::Copy(old_path),
167 |                     _ => unreachable!(),
168 |                 }
169 |             }
170 |             _ => unreachable!(),
171 |         };
172 |         // println!("{} {}", change, path);
173 |         if path.starts_with("\\\"") {
174 |             path = path
175 |                 .strip_prefix("\\\"")
176 |                 .unwrap()
177 |                 .strip_suffix("\\\"")
178 |                 .unwrap()
179 |                 .to_string();
180 |         } else if path.starts_with('\"') {
181 |             path = path
182 |                 .strip_prefix('"')
183 |                 .unwrap()
184 |                 .strip_suffix('\"')
185 |                 .unwrap()
186 |                 .to_string();
187 |         }
188 |         let parent_node_id = get_parent(&path, &mut tree, &mut nodes_idx);
189 |         let mut parent_node = tree.get_mut(parent_node_id).unwrap();
190 |         if let Some(child_node_id) = nodes_idx.get(&path) {
191 |             let mut child_node = tree.get_mut(*child_node_id).unwrap();
192 |             child_node.data().item = items_map.get(&path).cloned();
193 |             child_node.data().change = Some(change);
194 |             continue;
195 |         }
196 |         let child_id = parent_node
197 |             .append(Node {
198 |                 path: path.clone(),
199 |                 item: items_map.get(&path).cloned(),
200 |                 change: Some(change),
201 |             })
202 |             .node_id();
203 |         nodes_idx.insert(path, child_id);
204 |     }
205 | 
206 |     git_restore_staged(repo.workdir().unwrap(), ".")?;
207 | 
208 |     Ok((
209 |         ChangeTree {
210 |             new_repo,
211 |             tree,
212 |             idx: nodes_idx,
213 |         },
214 |         items_map,
215 |     ))
216 | }
217 | 
218 | fn get_parent(path: &str, tree: &mut Tree<Node>, idx: &mut HashMap<String, NodeId>) -> NodeId {
219 |     let parent = path
220 |         .find(tree_creator::PATH_SEPARATOR)
221 |         .map_or("", |i| &path[..i]);
222 |     if parent.is_empty() {
223 |         return tree.root_id().unwrap();
224 |     }
225 |     if let Some(parent_node) = idx.get(parent) {
226 |         *parent_node
227 |     } else {
228 |         let parent_node_id = get_parent(parent, tree, idx);
229 |         let mut parent_node = tree.get_mut(parent_node_id).unwrap();
230 |         let child_id = parent_node
231 |             .append(Node {
232 |                 path: parent.to_string(),
233 |                 item: None,
234 |                 change: None,
235 |             })
236 |             .node_id();
237 |         idx.insert(path.to_string(), child_id);
238 | 
239 |         parent_node_id
240 |     }
241 | }
242 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/change_tree_merge.rs:
--------------------------------------------------------------------------------
 1 | use std::collections::BTreeMap;
 2 | 
 3 | use crate::change_tree::{Change, PathChanges};
 4 | use crate::tree_creator::Item;
 5 | use anyhow::{anyhow, Result};
 6 | 
 7 | pub enum MergeStrategy {
 8 |     Copy,
 9 |     Move,
10 |     OneWay,
11 |     TwoWay,
12 | }
13 | 
14 | pub type Changes = Vec<(Change, String)>;
15 | pub type SrcItems = (usize, BTreeMap<String, Item>);
16 | pub type DstItems = BTreeMap<String, Item>;
17 | pub type MergedChanges = (Changes, SrcItems, DstItems);
18 | 
19 | pub fn merge(
20 |     path_changes: Vec<PathChanges>,
21 |     strategy: MergeStrategy,
22 | ) -> Result<Vec<MergedChanges>> {
23 |     if path_changes.len() < 2 {
24 |         return Err(anyhow!("Min 2 path_changes required for merge"));
25 |     }
26 |     match strategy {
27 |         MergeStrategy::Copy => unimplemented!(),
28 |         MergeStrategy::Move => unimplemented!(),
29 |         MergeStrategy::OneWay => do_one_way(path_changes),
30 |         MergeStrategy::TwoWay => unimplemented!(),
31 |     }
32 | }
33 | 
34 | fn do_one_way(mut path_changes: Vec<PathChanges>) -> Result<Vec<MergedChanges>> {
35 |     let mut merged_changes = vec![];
36 |     let (changes_path1, items_path1) = path_changes.remove(0);
37 |     merged_changes.push((
38 |         Default::default(),
39 |         (0, Default::default()),
40 |         Default::default(),
41 |     ));
42 |     let mut changes = vec![];
43 |     if changes_path1.tree.root().unwrap().first_child().is_some() {
44 |         let root = changes_path1.tree.root().unwrap();
45 |         root.traverse_pre_order().for_each(|node| {
46 |             if node.node_id() == changes_path1.tree.root_id().unwrap() {
47 |                 return;
48 |             }
49 |             let change = node.data().change.as_ref().unwrap();
50 |             let path = node.data().path.clone();
51 |             // todo: merge changes similar to what we do on apply
52 |             changes.push((change.clone(), path));
53 |         });
54 |     }
55 |     while !path_changes.is_empty() {
56 |         merged_changes.push((
57 |             changes.clone(),
58 |             // todo: use Arc
59 |             (0, items_path1.clone()),
60 |             path_changes.remove(0).1,
61 |         ));
62 |     }
63 | 
64 |     Ok(merged_changes)
65 | }
66 | 
67 | #[derive(Debug, Clone, PartialEq, Eq)]
68 | pub enum HashKind {
69 |     Md5,
70 |     Sha1,
71 |     Sha256,
72 | }
73 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/lib.rs:
--------------------------------------------------------------------------------
  1 | #![cfg_attr(not(debug_assertions), deny(warnings))]
  2 | #![doc(html_playground_url = "https://play.rust-lang.org")]
  3 | #![deny(clippy::all)]
  4 | #![deny(clippy::correctness)]
  5 | #![deny(clippy::suspicious)]
  6 | #![deny(clippy::complexity)]
  7 | #![deny(clippy::perf)]
  8 | #![deny(clippy::style)]
  9 | #![deny(clippy::pedantic)]
 10 | #![deny(clippy::nursery)]
 11 | #![deny(clippy::cargo)]
 12 | // #![deny(missing_docs)]
 13 | #![allow(clippy::similar_names)]
 14 | #![allow(clippy::too_many_arguments)]
 15 | #![allow(clippy::significant_drop_tightening)]
 16 | #![allow(clippy::redundant_closure)]
 17 | #![allow(clippy::missing_errors_doc)]
 18 | use anyhow::Result;
 19 | use change_tree_merge::HashKind;
 20 | use crc32fast::Hasher;
 21 | use std::fs::File;
 22 | use std::io::{BufReader, Read};
 23 | use std::path::Path;
 24 | use std::process::Command;
 25 | use std::time::Duration;
 26 | use std::{io, thread};
 27 | 
 28 | pub mod apply_change;
 29 | pub mod change_tree;
 30 | pub mod change_tree_merge;
 31 | pub mod path_walker;
 32 | pub mod tree_creator;
 33 | 
 34 | pub const TREE_DIR: &str = "tree";
 35 | 
 36 | pub trait IterRef {
 37 |     /// The type of the elements being iterated over.
 38 |     type Item;
 39 | 
 40 |     /// Which kind of iterator are we turning this into?
 41 |     type Iter: Iterator<Item = Self::Item>;
 42 | 
 43 |     /// Creates an iterator from a value.
 44 |     fn iter(&self) -> Self::Iter;
 45 | }
 46 | 
 47 | pub fn crc_eq(path1: &Path, path2: &Path) -> Result<bool> {
 48 |     let src_crc = crc(path1)?;
 49 |     let dst_crc = crc(path2)?;
 50 |     Ok(src_crc == dst_crc)
 51 | }
 52 | 
 53 | pub fn crc(path: &Path) -> Result<u32> {
 54 |     // Open the file
 55 |     let file = File::open(path)?;
 56 |     let mut reader = BufReader::new(file);
 57 | 
 58 |     // Create a CRC32 hasher
 59 |     let mut hasher = Hasher::new();
 60 |     let mut buffer = [0u8; 4096];
 61 | 
 62 |     // Read the file in chunks and update the hasher
 63 |     loop {
 64 |         let bytes_read = reader.read(&mut buffer)?;
 65 |         if bytes_read == 0 {
 66 |             break;
 67 |         }
 68 |         hasher.update(&buffer[..bytes_read]);
 69 |     }
 70 | 
 71 |     // Finalize the hasher to get the checksum
 72 |     let checksum = hasher.finalize();
 73 | 
 74 |     Ok(checksum)
 75 | }
 76 | 
 77 | pub fn file_hash(path: &Path, kind: HashKind) -> io::Result<String> {
 78 |     match kind {
 79 |         HashKind::Md5 => Ok(chksum_md5::chksum(File::open(path)?)
 80 |             .unwrap()
 81 |             .to_hex_lowercase()),
 82 |         HashKind::Sha1 => {
 83 |             unimplemented!("sha1")
 84 |         }
 85 |         HashKind::Sha256 => {
 86 |             unimplemented!("sha256")
 87 |         }
 88 |     }
 89 | }
 90 | 
 91 | pub fn git_status(repo: &Path) -> Result<String> {
 92 |     command("git", vec!["status", "-s"], repo)
 93 | }
 94 | 
 95 | pub fn git_add(repo: &Path, path_specs: &str) -> Result<()> {
 96 |     command("git", vec!["add", path_specs], repo)?;
 97 |     Ok(())
 98 | }
 99 | 
100 | pub fn command(command: &str, args: Vec<&str>, dir: &Path) -> Result<String> {
101 |     let mut c = Command::new(command);
102 |     let c = c.current_dir(dir);
103 |     let c = args.iter().fold(c, |c, arg| c.arg(arg));
104 |     let output = c.output().expect("Failed to execute command");
105 |     if output.status.success() {
106 |         Ok(String::from_utf8_lossy(&output.stdout).to_string())
107 |     } else {
108 |         anyhow::bail!(String::from_utf8_lossy(&output.stderr).to_string())
109 |     }
110 | }
111 | 
112 | pub fn git_restore_staged(repo: &Path, path_specs: &str) -> Result<()> {
113 |     command(
114 |         "git",
115 |         vec!["rm", "--cached", "--ignore-unmatch", "-r", path_specs],
116 |         repo,
117 |     )?;
118 |     let _ = command("git", vec!["restore", "--staged", path_specs], repo);
119 |     Ok(())
120 | }
121 | 
122 | pub fn git_commit(repo: &Path) -> Result<()> {
123 |     // repo.commit(
124 |     //     Some("HEAD"),
125 |     //     &repo.signature().unwrap(),
126 |     //     &repo.signature().unwrap(),
127 |     //     if new_repo { "Initial commit" } else { "Update" },
128 |     //     &repo
129 |     //         .find_tree(repo.index().unwrap().write_tree().unwrap())
130 |     //         .unwrap(),
131 |     //     &[&repo.head().unwrap().peel_to_commit().unwrap()],
132 |     // )
133 |     // .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
134 |     command(
135 |         "git",
136 |         vec!["commit", "--allow-empty", "-m", "\"changes\""],
137 |         repo,
138 |     )?;
139 |     Ok(())
140 | }
141 | 
142 | pub fn git_delete_history(repo: &Path) -> Result<()> {
143 |     command("git", vec!["checkout", "--orphan", "latest_branch"], repo)?;
144 |     git_add(repo, "-A")?;
145 |     git_commit(repo)?;
146 |     command("git", vec!["branch", "-D", "master"], repo)?;
147 |     command("git", vec!["branch", "-m", "master"], repo)?;
148 |     Ok(())
149 | }
150 | 
151 | fn retry<F, R>(f: F, count: usize) -> Result<R>
152 | where
153 |     F: Fn() -> Result<R>,
154 | {
155 |     let mut retries = 0;
156 |     loop {
157 |         match f() {
158 |             Ok(r) => return Ok(r),
159 |             Err(e) => {
160 |                 if retries > count {
161 |                     return Err(e);
162 |                 }
163 |                 thread::sleep(Duration::from_secs(2_u64.pow(retries as u32)));
164 |                 retries += 1;
165 |             }
166 |         }
167 |     }
168 | }
169 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/main.rs:
--------------------------------------------------------------------------------
  1 | use std::path::{Path, PathBuf};
  2 | use std::{io, thread};
  3 | 
  4 | use anyhow::Result;
  5 | use clap::Parser;
  6 | use colored::Colorize;
  7 | use tap::Pipe;
  8 | 
  9 | use file_tree_merge::change_tree::PathChanges;
 10 | use file_tree_merge::change_tree_merge::MergeStrategy;
 11 | use file_tree_merge::path_walker::PathWalker;
 12 | use file_tree_merge::tree_creator::TreeCreator;
 13 | use file_tree_merge::{
 14 |     apply_change, change_tree, change_tree_merge, git_add, git_commit, git_delete_history, TREE_DIR,
 15 | };
 16 | 
 17 | #[derive(Parser, Debug)]
 18 | #[command(version, about, long_about = None)]
 19 | struct Args {
 20 |     #[arg(
 21 |         short,
 22 |         long,
 23 |         short = 'r',
 24 |         help = "A directory where we'll keep a git repo to detect changes. Should persist between runs. MUST NOT BE ON ANY OF THE ENDPOINTS"
 25 |     )]
 26 |     repo: PathBuf,
 27 | 
 28 |     #[arg(
 29 |         short,
 30 |         long,
 31 |         default_value_t = false,
 32 |         help = "This simulates the sync. Will not apply any operations to the files, will just print the operations that would have normally be applied to endpoints"
 33 |     )]
 34 |     dry_run: bool,
 35 | 
 36 |     #[arg(
 37 |         short,
 38 |         long,
 39 |         short = 't',
 40 |         default_value_t = false,
 41 |         help = "If specified it will calculate MD5 hash for files when comparing file in path1 with the file in path2 when applying Add and Modify operations between endpoints. It will be considerably slower when activated"
 42 |     )]
 43 |     checksum: bool,
 44 | 
 45 |     #[arg(
 46 |         short,
 47 |         long,
 48 |         short = 'x',
 49 |         default_value_t = false,
 50 |         help = "If specified it will skip CRC check after file was transferred. Without this it compares the CRC of the file in path1 before transfer with the CRC of the file in path2 after transferred. This ensures the transfer was successful. Checking CRC is highly recommend if any of the endpoints are accessed over the network"
 51 |     )]
 52 |     no_crc: bool,
 53 | 
 54 |     #[arg(
 55 |         short,
 56 |         long,
 57 |         short = 'p',
 58 |         default_value_t = false,
 59 |         help = "Will print logs for each sync operation it applying"
 60 |     )]
 61 |     print_all_changes: bool,
 62 | 
 63 |     #[arg(help = "Endpoints where data to be synced resides")]
 64 |     inputs: Vec<PathBuf>,
 65 | }
 66 | 
 67 | fn main() -> Result<()> {
 68 |     let args = Args::parse();
 69 | 
 70 |     if args.no_crc {
 71 |         println!("{}", "CRC check after transfer is disabled. Make sure this is what you want because like that file integrity cannot be ensured. If you didn't intended this, remove --no-crc from args!".red().bold());
 72 |         thread::sleep(std::time::Duration::from_secs(10));
 73 |     }
 74 | 
 75 |     if args.checksum {
 76 |         println!(
 77 |             "{}",
 78 |             "Checksum mode enabled, it will be quite slow!"
 79 |                 .yellow()
 80 |                 .bold()
 81 |         );
 82 |     }
 83 | 
 84 |     if args.dry_run {
 85 |         println!("{}", "Dry-run mode enabled, it will not touch create, modify ot delete any files, will just print the changes!".yellow().bold());
 86 |     }
 87 | 
 88 |     if args.inputs.len() < 2 {
 89 |         println!(
 90 |             "{}",
 91 |             "You need to specify at least two paths to sync!"
 92 |                 .red()
 93 |                 .bold()
 94 |         );
 95 |         anyhow::bail!("You need to specify at least two paths to sync!");
 96 |     } else {
 97 |         std::fs::create_dir_all(&args.repo)?;
 98 |     }
 99 | 
100 |     for path in &args.inputs {
101 |         if !path.exists() {
102 |             println!(
103 |                 "{}",
104 |                 format!("Path '{}' does not exist", path.display()).red()
105 |             );
106 |             anyhow::bail!("Path '{}' does not exist", path.display());
107 |         }
108 |     }
109 | 
110 |     println!("{}", "Building changes trees ...".cyan());
111 |     let mut changes = vec![];
112 |     let mut errors = vec![];
113 |     for (idx, path) in args.inputs.iter().enumerate() {
114 |         let (changes_tree, err) =
115 |             changes_tree(PathWalker::new(path)?, &args.repo.join(idx.to_string()))?;
116 |         changes.push(changes_tree);
117 |         errors.push(err);
118 |     }
119 | 
120 |     println!("{}", "Merging changes trees ...".cyan());
121 |     change_tree_merge::merge(changes, MergeStrategy::OneWay)?.pipe(|x| {
122 |         if x.iter().map(|x| x.0.len()).sum::<usize>() == 0 {
123 |             println!("{}", "No changes".green());
124 |             return Ok::<(), anyhow::Error>(());
125 |         }
126 |         if !args.dry_run {
127 |             println!("{}", "Applying changes ...".cyan());
128 |         }
129 |         for (idx, path_change_to_apply) in x.into_iter().enumerate() {
130 |             let src_repo_idx = path_change_to_apply.1 .0;
131 |             println!(
132 |                 "{}",
133 |                 format!("-> '{}' ...", args.inputs[idx].to_string_lossy()).cyan()
134 |             );
135 |             if path_change_to_apply.0.is_empty() {
136 |                 println!("{}", "    no changes".green());
137 |                 continue;
138 |             }
139 |             apply_change::apply(
140 |                 path_change_to_apply,
141 |                 &args.inputs[src_repo_idx],
142 |                 &args.inputs[idx],
143 |                 &args.repo.join(src_repo_idx.to_string()),
144 |                 &args.repo.join(idx.to_string()),
145 |                 args.dry_run,
146 |                 args.checksum,
147 |                 !args.no_crc,
148 |                 args.print_all_changes,
149 |             )?;
150 |             println!("{}", "Cleaning up repo ...".cyan());
151 |             git_add(&args.repo.join(idx.to_string()), ".")?;
152 |             git_commit(&args.repo.join(idx.to_string()))?;
153 |             git_delete_history(&args.repo.join(idx.to_string()))?;
154 |         }
155 |         // todo: remove when we group changes per src
156 |         git_add(&args.repo.join("0"), ".")?;
157 |         git_commit(&args.repo.join("0"))?;
158 |         git_delete_history(&args.repo.join("0"))?;
159 |         Ok(())
160 |     })?;
161 | 
162 |     for (idx, err) in errors.iter().enumerate() {
163 |         if !err.is_empty() {
164 |             println!(
165 |                 "{}",
166 |                 format!("Errors reading from path{}:", idx).red().bold()
167 |             );
168 |             for e in err {
169 |                 println!("{}", e.to_string().red().bold());
170 |             }
171 |         }
172 |     }
173 | 
174 |     Ok(())
175 | }
176 | 
177 | fn changes_tree(iter: PathWalker, repo: &Path) -> Result<(PathChanges, Vec<io::Error>)> {
178 |     iter.pipe(TreeCreator::new)
179 |         .pipe(|x| x.create(&repo.join(TREE_DIR)))?
180 |         .pipe(|x| Ok((change_tree::build(x.0, repo)?, x.1)))
181 | }
182 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/path_walker.rs:
--------------------------------------------------------------------------------
  1 | use anyhow::Result;
  2 | use colored::Colorize;
  3 | use std::fs::FileTimes;
  4 | use std::path::{Path, PathBuf};
  5 | use std::time::Duration;
  6 | use std::{io, thread};
  7 | use walkdir::{DirEntry, WalkDir};
  8 | 
  9 | use crate::{tree_creator, IterRef};
 10 | 
 11 | pub struct PathWalker {
 12 |     path: PathBuf,
 13 | }
 14 | 
 15 | impl PathWalker {
 16 |     pub fn new(path: &Path) -> Result<Self> {
 17 |         if !path.exists() {
 18 |             println!(
 19 |                 "{}",
 20 |                 format!("Path '{}' does not exist", path.display()).red()
 21 |             );
 22 |             anyhow::bail!("Path '{}' does not exist", path.display())
 23 |         }
 24 |         Ok(Self {
 25 |             path: path.to_path_buf(),
 26 |         })
 27 |     }
 28 | }
 29 | 
 30 | pub struct Iter {
 31 |     it: Box<dyn Iterator<Item = walkdir::Result<DirEntry>>>,
 32 |     path: PathBuf,
 33 |     ctr: u64,
 34 | }
 35 | 
 36 | impl IterRef for PathWalker {
 37 |     type Item = io::Result<tree_creator::Item>;
 38 |     type Iter = Iter;
 39 | 
 40 |     fn iter(&self) -> Self::Iter {
 41 |         Iter {
 42 |             it: Box::new(
 43 |                 WalkDir::new(self.path.clone())
 44 |                     .follow_links(true)
 45 |                     .into_iter(),
 46 |             ),
 47 |             path: self.path.clone(),
 48 |             ctr: 0,
 49 |         }
 50 |     }
 51 | }
 52 | 
 53 | impl Iterator for Iter {
 54 |     type Item = io::Result<tree_creator::Item>;
 55 | 
 56 |     fn next(&mut self) -> Option<Self::Item> {
 57 |         let next = {
 58 |             let mut retries = 0;
 59 |             loop {
 60 |                 let n = self.it.next();
 61 |                 if n.is_none() {
 62 |                     return None;
 63 |                 } else if n.as_ref().unwrap().is_ok() {
 64 |                     break n;
 65 |                 } else {
 66 |                     retries += 1;
 67 |                     thread::sleep(Duration::from_millis(2_u64.pow(retries)));
 68 |                 }
 69 |                 if retries > 5 {
 70 |                     return None;
 71 |                 }
 72 |             }
 73 |         };
 74 |         next.map(|entry| {
 75 |             entry
 76 |                 .map(|entry| {
 77 |                     let path = entry.path();
 78 |                     if self.ctr % 100 == 0 {
 79 |                         println!(
 80 |                             "{}",
 81 |                             format!("Checking '{}'", path.to_string_lossy()).cyan()
 82 |                         );
 83 |                     }
 84 |                     self.ctr += 1;
 85 |                     let path_rel = path.strip_prefix(&self.path).unwrap();
 86 |                     let atime = entry.metadata().unwrap().accessed().unwrap();
 87 |                     let mtime = entry.metadata().unwrap().modified().unwrap();
 88 |                     let times = FileTimes::new().set_accessed(atime).set_modified(mtime);
 89 |                     let size = entry.metadata().unwrap().len();
 90 |                     let is_dir = entry.metadata().unwrap().is_dir();
 91 |                     tree_creator::Item {
 92 |                         path: path_rel.to_string_lossy().to_string(),
 93 |                         times,
 94 |                         atime,
 95 |                         mtime,
 96 |                         size,
 97 |                         is_dir,
 98 |                     }
 99 |                 })
100 |                 .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
101 |         })
102 |     }
103 | }
104 | 


--------------------------------------------------------------------------------
/file-tree-merge/src/tree_creator.rs:
--------------------------------------------------------------------------------
  1 | use std::fmt::{Debug, Formatter};
  2 | use std::fs::{File, FileTimes};
  3 | use std::io::Write;
  4 | use std::path::Path;
  5 | use std::sync::{Arc, Mutex};
  6 | use std::time::{SystemTime, UNIX_EPOCH};
  7 | use std::{fs, io};
  8 | 
  9 | use anyhow::Result;
 10 | use colored::Colorize;
 11 | use rayon::iter::ParallelIterator;
 12 | use rayon::prelude::IntoParallelRefIterator;
 13 | 
 14 | use crate::{retry, IterRef};
 15 | 
 16 | // pub(crate) const PATH_SEPARATOR: &str = "／";
 17 | pub(crate) const PATH_SEPARATOR: &str = "|";
 18 | 
 19 | pub struct Item {
 20 |     pub path: String,
 21 |     pub times: FileTimes,
 22 |     pub atime: SystemTime,
 23 |     pub mtime: SystemTime,
 24 |     pub size: u64,
 25 |     pub is_dir: bool,
 26 | }
 27 | 
 28 | impl Clone for Item {
 29 |     fn clone(&self) -> Self {
 30 |         Self {
 31 |             path: self.path.clone(),
 32 |             times: self.times.clone(),
 33 |             mtime: self.atime.clone(),
 34 |             atime: self.mtime.clone(),
 35 |             size: self.size,
 36 |             is_dir: self.is_dir,
 37 |         }
 38 |     }
 39 | }
 40 | 
 41 | impl Debug for Item {
 42 |     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
 43 |         f.debug_struct("Item")
 44 |             .field("path", &self.path)
 45 |             .field("times", &self.times)
 46 |             .field("atime", &self.atime)
 47 |             .field("mtime", &self.mtime)
 48 |             .field("size", &self.size)
 49 |             .field("is_dir", &self.is_dir)
 50 |             .finish()
 51 |     }
 52 | }
 53 | 
 54 | pub struct TreeCreator<I: Iterator<Item = io::Result<Item>>, Iter: IterRef<Iter = I>> {
 55 |     iter: Iter,
 56 | }
 57 | 
 58 | impl<I: Iterator<Item = io::Result<Item>>, Iter: IterRef<Iter = I>> TreeCreator<I, Iter> {
 59 |     pub fn new(iter: Iter) -> Self {
 60 |         Self { iter }
 61 |     }
 62 | 
 63 |     pub fn create(&self, tree_dir: &Path) -> Result<(Vec<Item>, Vec<io::Error>)> {
 64 |         let dst = tree_dir.to_path_buf();
 65 |         retry(
 66 |             || {
 67 |                 if dst.exists() {
 68 |                     if !dst.is_dir() {
 69 |                         anyhow::bail!("Destination is not a directory");
 70 |                     }
 71 |                     remove_all_from_dir(&dst)?;
 72 |                 } else {
 73 |                     fs::create_dir_all(&dst)?;
 74 |                 }
 75 |                 Ok(())
 76 |             },
 77 |             5,
 78 |         )?;
 79 | 
 80 |         let mut first = true;
 81 |         let items = Arc::new(Mutex::new(vec![]));
 82 |         let errors = Arc::new(Mutex::new(vec![]));
 83 |         let mut to_process = vec![];
 84 |         let batch_size = 1000;
 85 | 
 86 |         for item in self.iter.iter() {
 87 |             if first {
 88 |                 // skip root
 89 |                 first = false;
 90 |                 continue;
 91 |             }
 92 |             if let Err(err) = item {
 93 |                 println!("{}", format!("Error reading file: {:?}", err).red().bold());
 94 |                 errors.lock().unwrap().push(err);
 95 |                 continue;
 96 |             }
 97 |             let item = item?;
 98 |             to_process.push(item);
 99 | 
100 |             if to_process.len() % batch_size == 0 {
101 |                 // Process the collection in parallel
102 |                 let res: Vec<Result<()>> = to_process
103 |                     .par_iter() // Convert the vector into a parallel iterator
104 |                     .map(|item| process(item.clone(), &dst))
105 |                     .map(|item| {
106 |                         items.lock().unwrap().push(item?);
107 |                         Ok(())
108 |                     })
109 |                     .collect();
110 |                 for e in res {
111 |                     if let Err(err) = e {
112 |                         Err(err)?
113 |                     }
114 |                 }
115 |                 to_process.clear();
116 |             };
117 |         }
118 |         // Process the collection in parallel
119 |         let res: Vec<Result<()>> = to_process
120 |             .par_iter() // Convert the vector into a parallel iterator
121 |             .map(|item| process(item.clone(), &dst))
122 |             .map(|item| {
123 |                 items.lock().unwrap().push(item?);
124 |                 Ok(())
125 |             })
126 |             .collect();
127 |         for e in res {
128 |             if let Err(err) = e {
129 |                 Err(err)?
130 |             }
131 |         }
132 | 
133 |         Ok((
134 |             Arc::into_inner(items)
135 |                 .unwrap()
136 |                 .into_inner()
137 |                 .unwrap()
138 |                 .clone(),
139 |             Arc::into_inner(errors)
140 |                 .unwrap()
141 |                 .into_inner()
142 |                 .unwrap()
143 |                 .into_iter()
144 |                 .map(|e| io::Error::new(io::ErrorKind::Other, e))
145 |                 .collect(),
146 |         ))
147 |     }
148 | }
149 | 
150 | fn remove_all_from_dir(dir: &Path) -> io::Result<()> {
151 |     for entry in fs::read_dir(dir)? {
152 |         let entry = entry?;
153 |         let path = entry.path();
154 |         if path.is_dir() {
155 |             fs::remove_dir_all(path)?;
156 |         } else {
157 |             fs::remove_file(&path)?;
158 |         }
159 |     }
160 |     Ok(())
161 | }
162 | 
163 | fn get_time_bytes(time: &SystemTime) -> Vec<u8> {
164 |     // Convert to a duration since the Unix epoch
165 |     let duration_since_epoch = time.duration_since(UNIX_EPOCH).unwrap();
166 | 
167 |     // Convert the duration to seconds and nanoseconds
168 |     let secs = duration_since_epoch.as_secs();
169 |     let nanos = duration_since_epoch.subsec_nanos();
170 | 
171 |     // Convert seconds and nanoseconds to byte arrays
172 |     let secs_bytes = secs.to_be_bytes();
173 |     let nanos_bytes = nanos.to_be_bytes();
174 | 
175 |     // Combine the byte arrays
176 |     let mut bytes = [0u8; 12];
177 |     bytes[..8].copy_from_slice(&secs_bytes);
178 |     bytes[8..].copy_from_slice(&nanos_bytes);
179 | 
180 |     bytes.to_vec()
181 | }
182 | 
183 | fn process(item: Item, dst: &Path) -> Result<Item> {
184 |     // let path_rel = item.path.replace('/', PATH_SEPARATOR);
185 |     // let path = dst.join(path_rel);
186 |     let path = dst.join(item.path.clone());
187 |     retry(
188 |         || {
189 |             if item.is_dir {
190 |                 // println!("Creating dir: {:?}", path);
191 |                 fs::create_dir_all(&path)?;
192 |             } else {
193 |                 // println!("Creating file: {:?}", path);
194 |                 fs::create_dir_all(path.parent().unwrap())?;
195 |                 let mut file = File::create(&path)?;
196 |                 Path::new(&path)
197 |                     .parent()
198 |                     .map_or(Ok(()), fs::create_dir_all)?;
199 |                 file.write_all(&item.size.to_le_bytes())?;
200 |                 file.write_all(&get_time_bytes(&item.mtime))?;
201 |                 file.flush()?;
202 |                 file.sync_all()?;
203 |                 File::set_times(&file, item.times)?;
204 |                 File::open(&path)?.sync_all()?;
205 |                 File::open(path.parent().unwrap())?.sync_all()?;
206 |             }
207 |             Ok(())
208 |         },
209 |         5,
210 |     )?;
211 |     Ok(item)
212 | }
213 | 


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/cleanup.sh:
--------------------------------------------------------------------------------
1 | rm -rf "$setup_base_path"


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/common.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | Green="\e[0;32m"
 6 | Yellow="\e[0;33m"
 7 | Red="\033[0;31m"
 8 | NC="\e[0m" # No Color
 9 | 
10 | # Function to handle errors
11 | error_handler() {
12 |     echo "Error on line $1"
13 | }
14 | 
15 | # Function to generate a random size between given ranges in MB
16 | generate_random_size() {
17 |   local min_mb=$1
18 |   local max_mb=$2
19 |   local min_bytes=$((min_mb * 1048576))
20 |   local max_bytes=$((max_mb * 1048576))
21 |   shuf -i ${min_bytes}-${max_bytes} -n 1
22 | }
23 | 


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/setup.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | # Trap ERR signal and call error_handler function
 6 | trap 'error_handler $LINENO' ERR
 7 | 
 8 | rm -rf "$setup_base_path"
 9 | 
10 | # repo
11 | mkdir -p "$setup_base_path/repo"
12 | 
13 | # paths
14 | if [ "$setup_num_paths" -gt 0 ]; then
15 |   for (( i=1; i<=setup_num_paths; i++ )); do
16 |     rm -rf "$setup_base_path/path$i"
17 |     mkdir -p "$setup_base_path/path$i"
18 |   done
19 | fi
20 | 
21 | # Create a function to generate files with random content
22 | generate_files() {
23 |   if [ "$setup_num_initial_files" -gt 0 ]; then
24 |     for i in $(seq 1 $setup_num_initial_files); do
25 |       size=$(generate_random_size 1 100)
26 |       head -c $size </dev/urandom > $setup_base_path/path1/file
27 |     done
28 |   fi
29 | }
30 | 
31 | # Generate initial files in path1
32 | generate_files
33 | 


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | Green="\e[0;32m"
 6 | Yellow="\e[0;33m"
 7 | Red="\033[0;31m"
 8 | NC="\e[0m" # No Color
 9 | 
10 | ignore_marker='#[test(ignore)]'
11 | 
12 | find tests -type f -name "*.sh" | while read -r file; do
13 |   absolute_path=$(realpath "$file")
14 |   basename=$(basename "$file")
15 | 
16 |   parent_dir=$(dirname "$file")
17 |   # Change to the parent directory
18 |   echo "parent dir $parent_dir"
19 |   cd "$parent_dir" || exit
20 |   
21 |   first_parent=$(basename "$parent_dir")
22 |   
23 |   if grep -Fq "$ignore_marker" "$absolute_path"; then
24 |     echo -e "test $first_parent/$basename ... ${Yellow}ignored${NC}"
25 |     # Change back to the original directory (optional)
26 |     cd - >/dev/null || exit
27 |     continue
28 |   else
29 |     echo -e "test $first_parent/$basename ..."
30 |     sh "$absolute_path"
31 |     # Capture the exit code
32 |     exit_code=$?
33 |     # Check the exit code and handle it
34 |     if [ $exit_code -eq 0 ]; then
35 |         echo -e "${Green}ok${NC}"
36 |     else
37 |         echo -e "${Red}failed${NC}"
38 |         exit $exit_code
39 |     fi
40 |   fi
41 | 
42 |   # Change back to the original directory (optional)
43 |   cd - >/dev/null || exit
44 | 
45 |   echo -e ""
46 | done


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/add.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | # Add new files in path1
15 | for i in {6..10}; do
16 |   size=$(generate_random_size 1 100)
17 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
18 |   touch $setup_base_path/path1/file$i
19 | done
20 | 
21 | # Run sync
22 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
23 | 
24 | # Verify new files in path2
25 | for i in {6..10}; do
26 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
27 |     echo -e "${Red}file$i addition sync failed"
28 |     exit 1
29 |   fi
30 | done
31 | 
32 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/add_modify_delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | for i in {1..10}; do
15 |   size=$(generate_random_size 1 100)
16 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
17 |   touch $setup_base_path/path1/file$i
18 |   cp $setup_base_path/path1/file$i $setup_base_path/path2/file$i
19 |   touch -r $setup_base_path/path1/file$i $setup_base_path/path2/file$i
20 | done
21 | 
22 | # Run sync
23 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
24 | 
25 | # Add files in path1
26 | for i in {11..15}; do
27 |   size=$(generate_random_size 1 100)
28 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
29 |   touch $setup_base_path/path1/file$i
30 | done
31 | 
32 | # Change existing files in path1
33 | for i in {1..5}; do
34 |   size=$(generate_random_size 1 100)
35 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
36 |   touch $setup_base_path/path1/file$i
37 | done
38 | 
39 | # Delete some files in path1
40 | for i in {6..10}; do
41 |   rm $setup_base_path/path1/file$i
42 | done
43 | 
44 | # Run sync
45 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
46 | 
47 | # Verify new files in path2
48 | for i in {11..15}; do
49 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
50 |     echo -e "${Red}file$i addition sync failed"
51 |     exit 1
52 |   fi
53 | done
54 | 
55 | # Verify modified files in path2
56 | for i in {1..5}; do
57 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
58 |     echo -e "${Red}file$i modification sync failed"
59 |     exit 1
60 |   fi
61 | done
62 | 
63 | # Verify deleted files in path2
64 | for i in {6..10}; do
65 |   if [ -f $setup_base_path/path2/file$i ]; then
66 |     echo -e "${Red}file$i deletion sync failed"
67 |     exit 1
68 |   fi
69 | done
70 | 
71 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/add_modify_delete_multiple_paths.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=5
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | for i in {1..10}; do
15 |   size=$(generate_random_size 1 100)
16 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
17 |   touch $setup_base_path/path1/file$i
18 |   for j in $(seq 2 $setup_num_paths); do
19 |     cp $setup_base_path/path1/file$i $setup_base_path/path$j/file$i
20 |     touch -r $setup_base_path/path1/file$i $setup_base_path/path$j/file$i
21 |   done
22 | done
23 | 
24 | # Run sync
25 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
26 | 
27 | # Add files in path1
28 | for i in {11..15}; do
29 |   size=$(generate_random_size 1 100)
30 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
31 |   touch $setup_base_path/path1/file$i
32 | done
33 | 
34 | # Change existing files in path1
35 | for i in {1..5}; do
36 |   size=$(generate_random_size 1 100)
37 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
38 |   touch $setup_base_path/path1/file$i
39 | done
40 | 
41 | # Delete some files in path1
42 | for i in {6..10}; do
43 |   rm $setup_base_path/path1/file$i
44 | done
45 | 
46 | # Run sync
47 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2 $setup_base_path/path3 $setup_base_path/path4 $setup_base_path/path5
48 | 
49 | # Verify new files in path2
50 | for i in {11..15}; do
51 |   for j in $(seq 2 $setup_num_paths); do
52 |     if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path$j/file$i; then
53 |       echo -e "${Red}file$i addition sync failed to path$j"
54 |       exit 1
55 |     fi
56 |   done
57 | done
58 | 
59 | # Verify modified files in path2
60 | for i in {1..5}; do
61 |   for j in $(seq 2 $setup_num_paths); do
62 |     if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path$j/file$i; then
63 |       echo -e "${Red}file$i modification sync failed to path$j"
64 |       exit 1
65 |     fi
66 |   done
67 | done
68 | 
69 | # Verify deleted files in path2
70 | for i in {6..10}; do
71 |   for j in $(seq 2 $setup_num_paths); do
72 |     if [ -f $setup_base_path/path$j/file$i ]; then
73 |       echo -e "${Red}file$i deletion sync failed to path$j"
74 |       exit 1
75 |     fi
76 |   done
77 | done
78 | 
79 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/add_more.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | # Add new files in path1
15 | for i in {6..10}; do
16 |   size=$(generate_random_size 1 100)
17 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
18 |   touch $setup_base_path/path1/file$i
19 | done
20 | 
21 | # Run sync
22 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
23 | 
24 | # Verify new files in path2
25 | for i in {6..10}; do
26 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
27 |     echo -e "${Red}file$i addition sync failed"
28 |     exit 1
29 |   fi
30 | done
31 | 
32 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | size=$(generate_random_size 1 100)
15 | head -c $size </dev/urandom > $setup_base_path/path1/file1
16 | 
17 | # Run sync
18 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
19 | 
20 | # Delete a file in path1
21 | rm $setup_base_path/path1/file1
22 | 
23 | # Run sync
24 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
25 | 
26 | # Verify deletion in path2
27 | if [ -f $setup_base_path/path2/file1 ]; then
28 |   echo "file1 deletion sync failed"
29 |   exit 1
30 | fi
31 | 
32 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/delete_many.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | for i in {1..5}; do
15 |   size=$(generate_random_size 1 100)
16 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
17 |   touch $setup_base_path/path1/file$i
18 | done
19 | 
20 | # Run sync
21 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
22 | 
23 | # Delete files in path1
24 | for i in {1..5}; do
25 |   rm $setup_base_path/path1/file$i
26 | done
27 | 
28 | # Run sync
29 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
30 | 
31 | # Verify files are deleted in path2
32 | for i in {1..5}; do
33 |   if [ -f $setup_base_path/path2/file$i ]; then
34 |     echo -e "${Red}file$i deletion sync failed"
35 |     exit 1
36 |   fi
37 | done
38 | 
39 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/dry-run.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | # Add new files in path1
15 | for i in {1..5}; do
16 |   size=$(generate_random_size 1 100)
17 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
18 |   touch $setup_base_path/path1/file$i
19 | done
20 | 
21 | # Run sync
22 | $BIN --dry-run --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
23 | 
24 | # Verify no files in path2
25 | for i in {1..5}; do
26 |   if [ -f $setup_base_path/path2/file$i ]; then
27 |     echo "Dry-run for file$i failed"
28 |     exit 1
29 |   fi
30 | done
31 | 
32 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/large/add_modify_delete.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | #[test(ignore)]
 4 | 
 5 | export setup_base_path=~/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../.../target/release/syncoxiders"
10 | 
11 | source ../../common.sh
12 | source ../../setup.sh
13 | 
14 | set -e
15 | 
16 | echo "Setting up files ..."
17 | 
18 | # Add large files in path1
19 | for i in {21..25}; do
20 |     size=$(generate_random_size 1000 10000)
21 |     head -c $size </dev/urandom > $setup_base_path/path1/file$i
22 |     touch $setup_base_path/path1/file$i
23 | done
24 | 
25 | # Change existing files in path1 with large content
26 | for i in {1..5}; do
27 |     size=$(generate_random_size 1000 10000)
28 |     head -c $size </dev/urandom > $setup_base_path/path1/file$i
29 |     touch $setup_base_path/path1/file$i
30 | done
31 | 
32 | # Delete some files in path1
33 | for i in {11..15}; do
34 |   rm /tmp/syncoxiders_test/path1/file$i
35 | done
36 | 
37 | # Run sync
38 | $BIN --repo $setup_base_path/repo setup_base_path/path1 $setup_base_path/path2
39 | 
40 | # Verify new large files in path2
41 | for i in {21..25}; do
42 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
43 |     echo -e "${Red}file$i addition sync failed"
44 |     exit 1
45 |   fi
46 | done
47 | 
48 | # Verify modified large files in path2
49 | for i in {1..5}; do
50 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
51 |     echo -e "${Red}file$i modification sync failed"
52 |     exit 1
53 |   fi
54 | done
55 | 
56 | # Verify deleted files in path2
57 | for i in {11..15}; do
58 |   if [ -f $setup_base_path/path2/file$i ]; then
59 |     echo -e "${Red}file$i deletion sync failed"
60 |     exit 1
61 |   fi
62 | done
63 | 
64 | source ../../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/modify.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | size=$(generate_random_size 1 100)
15 | head -c $size </dev/urandom > $setup_base_path/path1/file1
16 | cp $setup_base_path/path1/file1 $setup_base_path/path2/file1
17 | 
18 | # Modify a file in path1
19 | head -c $size </dev/urandom > $setup_base_path/path1/file1
20 | touch $setup_base_path/path1/file1
21 | 
22 | # Run sync
23 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
24 | 
25 | # Verify modified file in path2
26 | if ! cmp -s $setup_base_path/path1/file1 $setup_base_path/path2/file1; then
27 |   echo -e "${Red}file1 modify sync failed"
28 |   exit 1
29 | fi
30 | 
31 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/modify_many.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | for i in {1..5}; do
15 |   size=$(generate_random_size 1 100)
16 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
17 |   touch $setup_base_path/path1/file$i
18 |   cp $setup_base_path/path1/file$i $setup_base_path/path2/file$i
19 | done
20 | 
21 | # Change existing files in path1
22 | for i in {1..5}; do
23 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
24 |   touch $setup_base_path/path1/file$i
25 | done
26 | 
27 | # Run sync
28 | $BIN --repo $setup_base_path/repo $setup_base_path/path1 $setup_base_path/path2
29 | 
30 | # Verify changed files in path2
31 | for i in {1..5}; do
32 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
33 |     echo -e "${Red}file$i modification sync failed"
34 |     exit 1
35 |   fi
36 | done
37 | 
38 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-tree-merge/tests/scripts/tests/no-crc.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -e
 4 | 
 5 | export setup_base_path=/tmp/syncoxiders_test
 6 | export setup_num_paths=2
 7 | export setup_num_initial_files=0
 8 | 
 9 | BIN="$(dirname $0)/../../../../target/release/syncoxiders"
10 | 
11 | source ../common.sh
12 | source ../setup.sh
13 | 
14 | # Add new files in path1
15 | for i in {1..5}; do
16 |   size=$(generate_random_size 1 100)
17 |   head -c $size </dev/urandom > $setup_base_path/path1/file$i
18 |   touch $setup_base_path/path1/file$i
19 | done
20 | 
21 | # Run sync with no CRC check
22 | $BIN --repo $setup_base_path/repo --no-crc $setup_base_path/path1 $setup_base_path/path2
23 | 
24 | # Verify files in path2
25 | for i in {1..5}; do
26 |   if ! cmp -s $setup_base_path/path1/file$i $setup_base_path/path2/file$i; then
27 |     echo -e "${Red}file$i sync with no CRC check failed"
28 |     exit 1
29 |   fi
30 | done
31 | 
32 | source ../cleanup.sh


--------------------------------------------------------------------------------
/file-watcher/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "file-watcher"
 3 | description.workspace = true
 4 | version.workspace = true
 5 | edition.workspace = true
 6 | license.workspace = true
 7 | authors.workspace = true
 8 | homepage.workspace = true
 9 | repository.workspace = true
10 | readme.workspace = true
11 | keywords.workspace = true
12 | categories.workspace = true
13 | documentation.workspace = true
14 | exclude.workspace = true
15 | 
16 | [dependencies]
17 | 


--------------------------------------------------------------------------------
/file-watcher/src/main.rs:
--------------------------------------------------------------------------------
1 | fn main() {
2 |     println!("Hello, world!");
3 | }
4 | 


--------------------------------------------------------------------------------
/p2p-transfer/.cargo/config.toml:
--------------------------------------------------------------------------------
1 | # It's recommended to set the flag on a per-target basis:
2 | [target.wasm32-unknown-unknown]
3 | rustflags = ['--cfg', 'getrandom_backend="wasm_js"']


--------------------------------------------------------------------------------
/p2p-transfer/.gitignore:
--------------------------------------------------------------------------------
 1 | # Mac stuff:
 2 | .DS_Store
 3 | 
 4 | # trunk output folder
 5 | dist
 6 | 
 7 | # Rust compile target directories:
 8 | target
 9 | target_ra
10 | target_wasm
11 | 
12 | # https://github.com/lycheeverse/lychee
13 | .lycheecache
14 | .idea
15 | Cargo.lock


--------------------------------------------------------------------------------
/p2p-transfer/.typos.toml:
--------------------------------------------------------------------------------
1 | # https://github.com/crate-ci/typos
2 | # install:  cargo install typos-cli
3 | # run:      typos
4 | 
5 | [default.extend-words]
6 | egui = "egui" # Example for how to ignore a false positive
7 | 


--------------------------------------------------------------------------------
/p2p-transfer/Cargo.toml:
--------------------------------------------------------------------------------
 1 | [package]
 2 | name = "p2p-transfer"
 3 | description.workspace = true
 4 | version.workspace = true
 5 | edition.workspace = true
 6 | license.workspace = true
 7 | authors.workspace = true
 8 | homepage.workspace = true
 9 | repository.workspace = true
10 | readme.workspace = true
11 | keywords.workspace = true
12 | categories.workspace = true
13 | documentation.workspace = true
14 | exclude.workspace = true
15 | 
16 | [package.metadata.docs.rs]
17 | all-features = true
18 | targets = ["x86_64-unknown-linux-gnu", "wasm32-unknown-unknown"]
19 | 
20 | [dependencies]
21 | egui = "0.31"
22 | eframe = { version = "0.31", default-features = false, features = [
23 |     "accesskit",     # Make egui compatible with screen readers. NOTE: adds a lot of dependencies.
24 |     "default_fonts", # Embed the default egui fonts.
25 |     "glow",          # Use the glow rendering backend. Alternative: "wgpu".
26 |     "persistence",   # Enable restoring app state when restarting the app.
27 |     "wayland",       # To support Linux (and CI)
28 |     "x11",           # To support older Linux distributions (restores one of the default features)
29 | ] }
30 | log = "0.4.27"
31 | 
32 | # You only need serde if you want app persistence:
33 | serde = { version = "1.0.219", features = ["derive"] }
34 | rfd = "0.15.3"
35 | gloo = "0.11.0"
36 | wasm-bindgen = "0.2.100"
37 | web-sys = "0.3.77"
38 | tracing-wasm = "0.2.1"
39 | console_error_panic_hook = "0.1.7"
40 | tokio = "1.45.0"
41 | js-sys = "0.3.77"
42 | wasm-bindgen-futures = "0.4.50"
43 | iroh = "0.35.0"
44 | n0-future = "0.1.3"
45 | 
46 | # Fix for getrandom WASM support
47 | getrandom = { version = "0.3", features = ["wasm_js"] }
48 | async-channel = "2.3.1"
49 | anyhow.workspace = true
50 | 
51 | # native:
52 | [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
53 | env_logger = "0.11.8"
54 | 
55 | # web:
56 | [target.'cfg(target_arch = "wasm32")'.dependencies]
57 | wasm-bindgen-futures = "0.4.50"
58 | web-sys = "0.3.70"              # to access the DOM (to hide the loading text)
59 | 
60 | [profile.release]
61 | opt-level = 2 # fast and small wasm
62 | 
63 | # Optimize all dependencies even in debug builds:
64 | [profile.dev.package."*"]
65 | opt-level = 2
66 | 
67 | 
68 | [patch.crates-io]
69 | 
70 | # If you want to use the bleeding edge version of egui and eframe:
71 | # egui = { git = "https://github.com/emilk/egui", branch = "master" }
72 | # eframe = { git = "https://github.com/emilk/egui", branch = "master" }
73 | 
74 | # If you fork https://github.com/emilk/egui you can test with:
75 | # egui = { path = "../egui/crates/egui" }
76 | # eframe = { path = "../egui/crates/eframe" }


--------------------------------------------------------------------------------
/p2p-transfer/LICENSE-APACHE:
--------------------------------------------------------------------------------
  1 |                               Apache License
  2 |                         Version 2.0, January 2004
  3 |                      http://www.apache.org/licenses/
  4 | 
  5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 | 1. Definitions.
  8 | 
  9 |    "License" shall mean the terms and conditions for use, reproduction,
 10 |    and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |    "Licensor" shall mean the copyright owner or entity authorized by
 13 |    the copyright owner that is granting the License.
 14 | 
 15 |    "Legal Entity" shall mean the union of the acting entity and all
 16 |    other entities that control, are controlled by, or are under common
 17 |    control with that entity. For the purposes of this definition,
 18 |    "control" means (i) the power, direct or indirect, to cause the
 19 |    direction or management of such entity, whether by contract or
 20 |    otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |    outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |    "You" (or "Your") shall mean an individual or Legal Entity
 24 |    exercising permissions granted by this License.
 25 | 
 26 |    "Source" form shall mean the preferred form for making modifications,
 27 |    including but not limited to software source code, documentation
 28 |    source, and configuration files.
 29 | 
 30 |    "Object" form shall mean any form resulting from mechanical
 31 |    transformation or translation of a Source form, including but
 32 |    not limited to compiled object code, generated documentation,
 33 |    and conversions to other media types.
 34 | 
 35 |    "Work" shall mean the work of authorship, whether in Source or
 36 |    Object form, made available under the License, as indicated by a
 37 |    copyright notice that is included in or attached to the work
 38 |    (an example is provided in the Appendix below).
 39 | 
 40 |    "Derivative Works" shall mean any work, whether in Source or Object
 41 |    form, that is based on (or derived from) the Work and for which the
 42 |    editorial revisions, annotations, elaborations, or other modifications
 43 |    represent, as a whole, an original work of authorship. For the purposes
 44 |    of this License, Derivative Works shall not include works that remain
 45 |    separable from, or merely link (or bind by name) to the interfaces of,
 46 |    the Work and Derivative Works thereof.
 47 | 
 48 |    "Contribution" shall mean any work of authorship, including
 49 |    the original version of the Work and any modifications or additions
 50 |    to that Work or Derivative Works thereof, that is intentionally
 51 |    submitted to Licensor for inclusion in the Work by the copyright owner
 52 |    or by an individual or Legal Entity authorized to submit on behalf of
 53 |    the copyright owner. For the purposes of this definition, "submitted"
 54 |    means any form of electronic, verbal, or written communication sent
 55 |    to the Licensor or its representatives, including but not limited to
 56 |    communication on electronic mailing lists, source code control systems,
 57 |    and issue tracking systems that are managed by, or on behalf of, the
 58 |    Licensor for the purpose of discussing and improving the Work, but
 59 |    excluding communication that is conspicuously marked or otherwise
 60 |    designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |    "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |    on behalf of whom a Contribution has been received by Licensor and
 64 |    subsequently incorporated within the Work.
 65 | 
 66 | 2. Grant of Copyright License. Subject to the terms and conditions of
 67 |    this License, each Contributor hereby grants to You a perpetual,
 68 |    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |    copyright license to reproduce, prepare Derivative Works of,
 70 |    publicly display, publicly perform, sublicense, and distribute the
 71 |    Work and such Derivative Works in Source or Object form.
 72 | 
 73 | 3. Grant of Patent License. Subject to the terms and conditions of
 74 |    this License, each Contributor hereby grants to You a perpetual,
 75 |    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |    (except as stated in this section) patent license to make, have made,
 77 |    use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |    where such license applies only to those patent claims licensable
 79 |    by such Contributor that are necessarily infringed by their
 80 |    Contribution(s) alone or by combination of their Contribution(s)
 81 |    with the Work to which such Contribution(s) was submitted. If You
 82 |    institute patent litigation against any entity (including a
 83 |    cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |    or a Contribution incorporated within the Work constitutes direct
 85 |    or contributory patent infringement, then any patent licenses
 86 |    granted to You under this License for that Work shall terminate
 87 |    as of the date such litigation is filed.
 88 | 
 89 | 4. Redistribution. You may reproduce and distribute copies of the
 90 |    Work or Derivative Works thereof in any medium, with or without
 91 |    modifications, and in Source or Object form, provided that You
 92 |    meet the following conditions:
 93 | 
 94 |    (a) You must give any other recipients of the Work or
 95 |        Derivative Works a copy of this License; and
 96 | 
 97 |    (b) You must cause any modified files to carry prominent notices
 98 |        stating that You changed the files; and
 99 | 
100 |    (c) You must retain, in the Source form of any Derivative Works
101 |        that You distribute, all copyright, patent, trademark, and
102 |        attribution notices from the Source form of the Work,
103 |        excluding those notices that do not pertain to any part of
104 |        the Derivative Works; and
105 | 
106 |    (d) If the Work includes a "NOTICE" text file as part of its
107 |        distribution, then any Derivative Works that You distribute must
108 |        include a readable copy of the attribution notices contained
109 |        within such NOTICE file, excluding those notices that do not
110 |        pertain to any part of the Derivative Works, in at least one
111 |        of the following places: within a NOTICE text file distributed
112 |        as part of the Derivative Works; within the Source form or
113 |        documentation, if provided along with the Derivative Works; or,
114 |        within a display generated by the Derivative Works, if and
115 |        wherever such third-party notices normally appear. The contents
116 |        of the NOTICE file are for informational purposes only and
117 |        do not modify the License. You may add Your own attribution
118 |        notices within Derivative Works that You distribute, alongside
119 |        or as an addendum to the NOTICE text from the Work, provided
120 |        that such additional attribution notices cannot be construed
121 |        as modifying the License.
122 | 
123 |    You may add Your own copyright statement to Your modifications and
124 |    may provide additional or different license terms and conditions
125 |    for use, reproduction, or distribution of Your modifications, or
126 |    for any such Derivative Works as a whole, provided Your use,
127 |    reproduction, and distribution of the Work otherwise complies with
128 |    the conditions stated in this License.
129 | 
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 |    any Contribution intentionally submitted for inclusion in the Work
132 |    by You to the Licensor shall be under the terms and conditions of
133 |    this License, without any additional terms or conditions.
134 |    Notwithstanding the above, nothing herein shall supersede or modify
135 |    the terms of any separate license agreement you may have executed
136 |    with Licensor regarding such Contributions.
137 | 
138 | 6. Trademarks. This License does not grant permission to use the trade
139 |    names, trademarks, service marks, or product names of the Licensor,
140 |    except as required for reasonable and customary use in describing the
141 |    origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 |    agreed to in writing, Licensor provides the Work (and each
145 |    Contributor provides its Contributions) on an "AS IS" BASIS,
146 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |    implied, including, without limitation, any warranties or conditions
148 |    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |    PARTICULAR PURPOSE. You are solely responsible for determining the
150 |    appropriateness of using or redistributing the Work and assume any
151 |    risks associated with Your exercise of permissions under this License.
152 | 
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 |    whether in tort (including negligence), contract, or otherwise,
155 |    unless required by applicable law (such as deliberate and grossly
156 |    negligent acts) or agreed to in writing, shall any Contributor be
157 |    liable to You for damages, including any direct, indirect, special,
158 |    incidental, or consequential damages of any character arising as a
159 |    result of this License or out of the use or inability to use the
160 |    Work (including but not limited to damages for loss of goodwill,
161 |    work stoppage, computer failure or malfunction, or any and all
162 |    other commercial damages or losses), even if such Contributor
163 |    has been advised of the possibility of such damages.
164 | 
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 |    the Work or Derivative Works thereof, You may choose to offer,
167 |    and charge a fee for, acceptance of support, warranty, indemnity,
168 |    or other liability obligations and/or rights consistent with this
169 |    License. However, in accepting such obligations, You may act only
170 |    on Your own behalf and on Your sole responsibility, not on behalf
171 |    of any other Contributor, and only if You agree to indemnify,
172 |    defend, and hold each Contributor harmless for any liability
173 |    incurred by, or claims asserted against, such Contributor by reason
174 |    of your accepting any such warranty or additional liability.
175 | 
176 | END OF TERMS AND CONDITIONS
177 | 
178 | APPENDIX: How to apply the Apache License to your work.
179 | 
180 |    To apply the Apache License to your work, attach the following
181 |    boilerplate notice, with the fields enclosed by brackets "[]"
182 |    replaced with your own identifying information. (Don't include
183 |    the brackets!)  The text should be enclosed in the appropriate
184 |    comment syntax for the file format. We also recommend that a
185 |    file or class name and description of purpose be included on the
186 |    same "printed page" as the copyright notice for easier
187 |    identification within third-party archives.
188 | 
189 | Copyright [yyyy] [name of copyright owner]
190 | 
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 | 
195 |     http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 | 


--------------------------------------------------------------------------------
/p2p-transfer/LICENSE-MIT:
--------------------------------------------------------------------------------
 1 | Permission is hereby granted, free of charge, to any
 2 | person obtaining a copy of this software and associated
 3 | documentation files (the "Software"), to deal in the
 4 | Software without restriction, including without
 5 | limitation the rights to use, copy, modify, merge,
 6 | publish, distribute, sublicense, and/or sell copies of
 7 | the Software, and to permit persons to whom the Software
 8 | is furnished to do so, subject to the following
 9 | conditions:
10 | 
11 | The above copyright notice and this permission notice
12 | shall be included in all copies or substantial portions
13 | of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
23 | DEALINGS IN THE SOFTWARE.
24 | 


--------------------------------------------------------------------------------
/p2p-transfer/README.md:
--------------------------------------------------------------------------------
 1 | # eframe template
 2 | 
 3 | [![dependency status](https://deps.rs/repo/github/emilk/eframe_template/status.svg)](https://deps.rs/repo/github/emilk/eframe_template)
 4 | [![Build Status](https://github.com/emilk/eframe_template/workflows/CI/badge.svg)](https://github.com/emilk/eframe_template/actions?workflow=CI)
 5 | 
 6 | This is a template repo for [eframe](https://github.com/emilk/egui/tree/master/crates/eframe), a framework for writing apps using [egui](https://github.com/emilk/egui/).
 7 | 
 8 | The goal is for this to be the simplest way to get started writing a GUI app in Rust.
 9 | 
10 | You can compile your app natively or for the web, and share it using Github Pages.
11 | 
12 | ## Getting started
13 | 
14 | Start by clicking "Use this template" at https://github.com/emilk/eframe_template/ or follow [these instructions](https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/creating-a-repository-from-a-template).
15 | 
16 | Change the name of the crate: Choose a good name for your project, and change the name to it in:
17 | * `Cargo.toml`
18 |     * Change the `package.name` from `eframe_template` to `your_crate`.
19 |     * Change the `package.authors`
20 | * `main.rs`
21 |     * Change `eframe_template::TemplateApp` to `your_crate::TemplateApp`
22 | * `index.html`
23 |     * Change the `<title>eframe template</title>` to `<title>your_crate</title>`. optional.
24 | * `assets/sw.js`
25 |   * Change the `'./eframe_template.js'` to `./your_crate.js` (in `filesToCache` array)
26 |   * Change the `'./eframe_template_bg.wasm'` to `./your_crate_bg.wasm` (in `filesToCache` array)
27 | 
28 | Alternatively, you can run `fill_template.sh` which will ask for the needed names and email and perform the above patches for you. This is particularly useful if you clone this repository outside GitHub and hence cannot make use of its
29 | templating function.
30 | 
31 | ### Learning about egui
32 | 
33 | `src/app.rs` contains a simple example app. This is just to give some inspiration - most of it can be removed if you like.
34 | 
35 | The official egui docs are at <https://docs.rs/egui>. If you prefer watching a video introduction, check out <https://www.youtube.com/watch?v=NtUkr_z7l84>. For inspiration, check out the [the egui web demo](https://emilk.github.io/egui/index.html) and follow the links in it to its source code.
36 | 
37 | ### Testing locally
38 | 
39 | `cargo run --release`
40 | 
41 | On Linux you need to first run:
42 | 
43 | `sudo apt-get install libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxkbcommon-dev libssl-dev`
44 | 
45 | On Fedora Rawhide you need to run:
46 | 
47 | `dnf install clang clang-devel clang-tools-extra libxkbcommon-devel pkg-config openssl-devel libxcb-devel gtk3-devel atk fontconfig-devel`
48 | 
49 | ### Web Locally
50 | 
51 | You can compile your app to [WASM](https://en.wikipedia.org/wiki/WebAssembly) and publish it as a web page.
52 | 
53 | We use [Trunk](https://trunkrs.dev/) to build for web target.
54 | 1. Install the required target with `rustup target add wasm32-unknown-unknown`.
55 | 2. Install Trunk with `cargo install --locked trunk`.
56 | 3. Run `trunk serve` to build and serve on `http://127.0.0.1:8080`. Trunk will rebuild automatically if you edit the project.
57 | 4. Open `http://127.0.0.1:8080/index.html#dev` in a browser. See the warning below.
58 | 
59 | > `assets/sw.js` script will try to cache our app, and loads the cached version when it cannot connect to server allowing your app to work offline (like PWA).
60 | > appending `#dev` to `index.html` will skip this caching, allowing us to load the latest builds during development.
61 | 
62 | ### Web Deploy
63 | 1. Just run `trunk build --release`.
64 | 2. It will generate a `dist` directory as a "static html" website
65 | 3. Upload the `dist` directory to any of the numerous free hosting websites including [GitHub Pages](https://docs.github.com/en/free-pro-team@latest/github/working-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site).
66 | 4. we already provide a workflow that auto-deploys our app to GitHub pages if you enable it.
67 | > To enable Github Pages, you need to go to Repository -> Settings -> Pages -> Source -> set to `gh-pages` branch and `/` (root).
68 | >
69 | > If `gh-pages` is not available in `Source`, just create and push a branch called `gh-pages` and it should be available.
70 | >
71 | > If you renamed the `main` branch to something else (say you re-initialized the repository with `master` as the initial branch), be sure to edit the github workflows `.github/workflows/pages.yml` file to reflect the change
72 | > ```yml
73 | > on:
74 | >   push:
75 | >     branches:
76 | >       - <branch name>
77 | > ```
78 | 
79 | You can test the template app at <https://emilk.github.io/eframe_template/>.
80 | 
81 | ## Updating egui
82 | 
83 | As of 2023, egui is in active development with frequent releases with breaking changes. [eframe_template](https://github.com/emilk/eframe_template/) will be updated in lock-step to always use the latest version of egui.
84 | 
85 | When updating `egui` and `eframe` it is recommended you do so one version at the time, and read about the changes in [the egui changelog](https://github.com/emilk/egui/blob/master/CHANGELOG.md) and [eframe changelog](https://github.com/emilk/egui/blob/master/crates/eframe/CHANGELOG.md).
86 | 


--------------------------------------------------------------------------------
/p2p-transfer/Trunk.toml:
--------------------------------------------------------------------------------
1 | [build]
2 | filehash = false
3 | 


--------------------------------------------------------------------------------
/p2p-transfer/assets/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/p2p-transfer/assets/favicon.ico


--------------------------------------------------------------------------------
/p2p-transfer/assets/icon-1024.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/p2p-transfer/assets/icon-1024.png


--------------------------------------------------------------------------------
/p2p-transfer/assets/icon-256.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/p2p-transfer/assets/icon-256.png


--------------------------------------------------------------------------------
/p2p-transfer/assets/icon_ios_touch_192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/p2p-transfer/assets/icon_ios_touch_192.png


--------------------------------------------------------------------------------
/p2p-transfer/assets/manifest.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "egui Template PWA",
 3 |   "short_name": "egui-template-pwa",
 4 |   "icons": [
 5 |     {
 6 |       "src": "./assets/icon-256.png",
 7 |       "sizes": "256x256",
 8 |       "type": "image/png"
 9 |     },
10 |     {
11 |       "src": "./assets/maskable_icon_x512.png",
12 |       "sizes": "512x512",
13 |       "type": "image/png",
14 |       "purpose": "any maskable"
15 |     },
16 |     {
17 |       "src": "./assets/icon-1024.png",
18 |       "sizes": "1024x1024",
19 |       "type": "image/png"
20 |     }
21 |   ],
22 |   "lang": "en-US",
23 |   "id": "/index.html",
24 |   "start_url": "./index.html",
25 |   "display": "standalone",
26 |   "background_color": "white",
27 |   "theme_color": "white"
28 | }
29 | 


--------------------------------------------------------------------------------
/p2p-transfer/assets/maskable_icon_x512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/p2p-transfer/assets/maskable_icon_x512.png


--------------------------------------------------------------------------------
/p2p-transfer/assets/sw.js:
--------------------------------------------------------------------------------
 1 | var cacheName = 'egui-template-pwa';
 2 | var filesToCache = [
 3 |   './',
 4 |   './index.html',
 5 |   './p2p-transfer.js',
 6 |   './p2p-transfer_bg.wasm',
 7 | ];
 8 | 
 9 | /* Start the service worker and cache all of the app's content */
10 | self.addEventListener('install', function (e) {
11 |   e.waitUntil(
12 |     caches.open(cacheName).then(function (cache) {
13 |       return cache.addAll(filesToCache);
14 |     })
15 |   );
16 | });
17 | 
18 | /* Serve cached content when offline */
19 | self.addEventListener('fetch', function (e) {
20 |   e.respondWith(
21 |     caches.match(e.request).then(function (response) {
22 |       return response || fetch(e.request);
23 |     })
24 |   );
25 | });
26 | 


--------------------------------------------------------------------------------
/p2p-transfer/check.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | # This scripts runs various CI-like checks in a convenient way.
 3 | set -eux
 4 | 
 5 | cargo check --quiet --workspace --all-targets
 6 | cargo check --quiet --workspace --all-features --lib --target wasm32-unknown-unknown
 7 | cargo fmt --all -- --check
 8 | cargo clippy --quiet --workspace --all-targets --all-features --  -D warnings -W clippy::all
 9 | cargo test --quiet --workspace --all-targets --all-features
10 | cargo test --quiet --workspace --doc
11 | trunk build
12 | 


--------------------------------------------------------------------------------
/p2p-transfer/fill_template.ps1:
--------------------------------------------------------------------------------
 1 | $crate = Read-Host "To fill the template, tell me your egui project crate name: "
 2 | $name = Read-Host "To fill the template, tell me your name (for author in Cargo.toml): "
 3 | $email = Read-Host "To fill the template, tell me your e-mail address (also for Cargo.toml): "
 4 | 
 5 | Write-Host "Patching files..."
 6 | 
 7 | (Get-Content "Cargo.toml") -replace "eframe_template", $crate | Set-Content "Cargo.toml"
 8 | (Get-Content "src\main.rs") -replace "eframe_template", $crate | Set-Content "src\main.rs"
 9 | (Get-Content "index.html") -replace "eframe template", $crate -replace "eframe_template", $crate | Set-Content "index.html"
10 | (Get-Content "assets\sw.js") -replace "eframe_template", $crate | Set-Content "assets\sw.js"
11 | (Get-Content "Cargo.toml") -replace "Emil Ernerfeldt", $name -replace "emil.ernerfeldt@gmail.com", $email | Set-Content "Cargo.toml"
12 | 
13 | Write-Host "Done."


--------------------------------------------------------------------------------
/p2p-transfer/fill_template.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env sh
 2 | 
 3 | set -e
 4 | 
 5 | echo "To fill the template tell me your egui project crate name: "
 6 | read crate
 7 | 
 8 | echo "To fill the template tell me your name (for author in Cargo.toml): "
 9 | read name
10 | 
11 | echo "To fill the template tell me your e-mail address (also for Cargo.toml): "
12 | read email
13 | 
14 | echo "Patching files..."
15 | 
16 | # Determine the correct sed syntax based on OS
17 | if [ "$(uname)" = "Darwin" ]; then
18 |   # macOS sed
19 |   sed_inplace() {
20 |     sed -i '' "$@"
21 |   }
22 | else
23 |   # GNU sed (Linux)
24 |   sed_inplace() {
25 |     sed -i "$@"
26 |   }
27 | fi
28 | 
29 | sed_inplace "s/eframe_template/$crate/g" Cargo.toml
30 | sed_inplace "s/eframe_template/$crate/g" src/main.rs
31 | sed_inplace "s/eframe template/$crate/g" index.html
32 | sed_inplace "s/eframe_template/$crate/g" assets/sw.js
33 | sed_inplace "s/Emil Ernerfeldt/$name/g" Cargo.toml
34 | sed_inplace "s/emil.ernerfeldt@gmail.com/$email/g" Cargo.toml
35 | 
36 | echo "Done."
37 | 


--------------------------------------------------------------------------------
/p2p-transfer/flake.nix:
--------------------------------------------------------------------------------
 1 | {
 2 |   description = "eframe devShell";
 3 | 
 4 |   inputs = {
 5 |     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
 6 |     rust-overlay.url = "github:oxalica/rust-overlay";
 7 |     flake-utils.url = "github:numtide/flake-utils";
 8 |   };
 9 | 
10 |   outputs = { self, nixpkgs, rust-overlay, flake-utils, ... }:
11 |     flake-utils.lib.eachDefaultSystem (system:
12 |       let
13 |         overlays = [ (import rust-overlay) ];
14 |         pkgs = import nixpkgs { inherit system overlays; };
15 |       in with pkgs; {
16 |         devShells.default = mkShell rec {
17 |           buildInputs = [
18 |             # Rust
19 |             rust-bin.stable.latest.default
20 |             trunk
21 | 
22 |             # misc. libraries
23 |             openssl
24 |             pkg-config
25 | 
26 |             # GUI libs
27 |             libxkbcommon
28 |             libGL
29 |             fontconfig
30 | 
31 |             # wayland libraries
32 |             wayland
33 | 
34 |             # x11 libraries
35 |             xorg.libXcursor
36 |             xorg.libXrandr
37 |             xorg.libXi
38 |             xorg.libX11
39 | 
40 |           ];
41 | 
42 |           LD_LIBRARY_PATH = "${lib.makeLibraryPath buildInputs}";
43 |         };
44 |       });
45 | }
46 | 


--------------------------------------------------------------------------------
/p2p-transfer/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html>
  3 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  4 | 
  5 | <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
  6 | 
  7 | <head>
  8 |     <title>p2p-transfer</title>
  9 | 
 10 |     <link data-trunk rel="rust" data-wasm-opt="2" />
 11 |     <base data-trunk-public-url />
 12 | 
 13 |     <link data-trunk rel="icon" href="assets/favicon.ico">
 14 | 
 15 | 
 16 |     <link data-trunk rel="copy-file" href="assets/sw.js"/>
 17 |     <link data-trunk rel="copy-file" href="assets/manifest.json"/>
 18 |     <link data-trunk rel="copy-file" href="assets/icon-1024.png" data-target-path="assets"/>
 19 |     <link data-trunk rel="copy-file" href="assets/icon-256.png" data-target-path="assets"/>
 20 |     <link data-trunk rel="copy-file" href="assets/icon_ios_touch_192.png" data-target-path="assets"/>
 21 |     <link data-trunk rel="copy-file" href="assets/maskable_icon_x512.png" data-target-path="assets"/>
 22 | 
 23 | 
 24 |     <link rel="manifest" href="assets/manifest.json">
 25 |     <link rel="apple-touch-icon" href="assets/icon_ios_touch_192.png">
 26 |     <meta name="theme-color" media="(prefers-color-scheme: light)" content="white">
 27 |     <meta name="theme-color" media="(prefers-color-scheme: dark)" content="#404040">
 28 | 
 29 | 
 30 |     <style>
 31 |         html {
 32 |             touch-action: manipulation;
 33 |         }
 34 | 
 35 |         body {
 36 | 
 37 |             background: #909090;
 38 |         }
 39 | 
 40 |         @media (prefers-color-scheme: dark) {
 41 |             body {
 42 |                 background: #404040;
 43 |             }
 44 |         }
 45 | 
 46 |         /* Allow canvas to fill entire web page: */
 47 |         html,
 48 |         body {
 49 |             overflow: hidden;
 50 |             margin: 0 !important;
 51 |             padding: 0 !important;
 52 |             height: 100%;
 53 |             width: 100%;
 54 |         }
 55 | 
 56 |         canvas {
 57 |             margin-right: auto;
 58 |             margin-left: auto;
 59 |             display: block;
 60 |             position: absolute;
 61 |             top: 0;
 62 |             left: 0;
 63 |             width: 100%;
 64 |             height: 100%;
 65 |         }
 66 | 
 67 |         .centered {
 68 |             margin-right: auto;
 69 |             margin-left: auto;
 70 |             display: block;
 71 |             position: absolute;
 72 |             top: 50%;
 73 |             left: 50%;
 74 |             transform: translate(-50%, -50%);
 75 |             color: #f0f0f0;
 76 |             font-size: 24px;
 77 |             font-family: Ubuntu-Light, Helvetica, sans-serif;
 78 |             text-align: center;
 79 |         }
 80 | 
 81 |         .lds-dual-ring {
 82 |             display: inline-block;
 83 |             width: 24px;
 84 |             height: 24px;
 85 |         }
 86 | 
 87 |         .lds-dual-ring:after {
 88 |             content: " ";
 89 |             display: block;
 90 |             width: 24px;
 91 |             height: 24px;
 92 |             margin: 0px;
 93 |             border-radius: 50%;
 94 |             border: 3px solid #fff;
 95 |             border-color: #fff transparent #fff transparent;
 96 |             animation: lds-dual-ring 1.2s linear infinite;
 97 |         }
 98 | 
 99 |         @keyframes lds-dual-ring {
100 |             0% {
101 |                 transform: rotate(0deg);
102 |             }
103 | 
104 |             100% {
105 |                 transform: rotate(360deg);
106 |             }
107 |         }
108 |     </style>
109 | </head>
110 | 
111 | <body>
112 |     <canvas id="the_canvas_id"></canvas>
113 |     <div class="centered" id="loading_text">
114 |         <p style="font-size:16px">
115 |             Loading…
116 |         </p>
117 |         <div class="lds-dual-ring"></div>
118 |     </div>
119 | 
120 |     <script src="https://cdn.jsdelivr.net/npm/webtorrent@latest/webtorrent.min.js">
121 | 
122 |     if ('serviceWorker' in navigator && window.location.hash !== "#dev") {
123 |     window.addEventListener('load', function () {
124 |                 navigator.serviceWorker.register('sw.js');
125 |             });
126 |         }
127 |     </script>
128 | </body>
129 | 
130 | </html>
131 | 


--------------------------------------------------------------------------------
/p2p-transfer/rust-toolchain:
--------------------------------------------------------------------------------
 1 | # If you see this, run "rustup self update" to get rustup 1.23 or newer.
 2 | 
 3 | # NOTE: above comment is for older `rustup` (before TOML support was added),
 4 | # which will treat the first line as the toolchain name, and therefore show it
 5 | # to the user in the error, instead of "error: invalid channel name '[toolchain]'".
 6 | 
 7 | [toolchain]
 8 | channel = "1.82"  # Avoid specifying a patch version here; see https://github.com/emilk/eframe_template/issues/145
 9 | components = [ "rustfmt", "clippy" ]
10 | targets = [ "wasm32-unknown-unknown" ]
11 | 


--------------------------------------------------------------------------------
/p2p-transfer/src/app.rs:
--------------------------------------------------------------------------------
  1 | use eframe::egui;
  2 | use egui::{Button, Color32, Grid, Label, RichText, TextStyle, Ui, Vec2};
  3 | use serde::{Deserialize, Serialize};
  4 | use wasm_bindgen::{JsCast, JsValue};
  5 | 
  6 | #[derive(Debug, Default, Clone, Serialize, Deserialize)]
  7 | struct TorrentInfo{
  8 |     magnet_uri : Option<String>,
  9 |     download_progress: f32,
 10 |     peers_count: usize,
 11 |     is_download: bool,
 12 |     is_seeding: bool,
 13 |     download_complete: bool
 14 | }
 15 | 
 16 | #[derive(Deserialize, Serialize)]
 17 | #[serde(default)]
 18 | pub struct P2PTransfer {
 19 |     #[serde(skip)]
 20 |     value: f32,
 21 |     #[cfg(target_arch = "wasm32")]
 22 |     #[serde(skip)]
 23 |     file_input_closure: Option<wasm_bindgen::closure::Closure<dyn FnMut(web_sys::Event)>>,
 24 |     #[serde(skip)]
 25 |     picked_file_name: std::sync::Arc<std::sync::Mutex<Option<String>>>,
 26 |     #[serde(skip)]
 27 |     picked_file_path: std::sync::Arc<std::sync::Mutex<Option<String>>>,
 28 |     #[serde(skip)]
 29 |     picked_file_size: std::sync::Arc<std::sync::Mutex<Option<u64>>>,
 30 |     #[serde(skip)]
 31 |     torrent_info: std::sync::Arc<std::sync::Mutex<TorrentInfo>>,
 32 |     #[serde(skip)]
 33 |     magnet_input: String
 34 | 
 35 | }
 36 | 
 37 | impl Default for P2PTransfer {
 38 |     fn default() -> Self {
 39 |         Self {
 40 |             value: 0.0,
 41 |             #[cfg(target_arch = "wasm32")]
 42 |             file_input_closure: None,
 43 |             picked_file_name: std::sync::Arc::new(std::sync::Mutex::new(None)),
 44 |             picked_file_path: std::sync::Arc::new(std::sync::Mutex::new(None)),
 45 |             picked_file_size: std::sync::Arc::new(std::sync::Mutex::new(None)),
 46 |             torrent_info: std::sync::Arc::new(std::sync::Mutex::new(TorrentInfo::default())),
 47 |             magnet_input: String::new(),
 48 |         }
 49 |     }
 50 | }
 51 | 
 52 | impl P2PTransfer {
 53 |     pub fn new(cc: &eframe::CreationContext<'_>) -> Self {
 54 |         // Load previous app state (if any)
 55 |         if let Some(storage) = cc.storage {
 56 |             return eframe::get_value(storage, eframe::APP_KEY).unwrap_or_default();
 57 |         }
 58 |         Default::default()
 59 |     }
 60 | 
 61 |     #[cfg(not(target_arch = "wasm32"))]
 62 |     fn pick_file(&mut self) {
 63 |         if let Some(path) = rfd::FileDialog::new().pick_file() {
 64 |             let file_name = path.file_name().unwrap_or_default().to_string_lossy().to_string();
 65 |             let file_path = path.display().to_string();
 66 |             let file_size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
 67 | 
 68 |             if let Ok(mut filename) = self.picked_file_name.lock() {
 69 |                 *filename = Some(file_name);
 70 |             }
 71 |             if let Ok(mut filepath) = self.picked_file_path.lock() {
 72 |                 *filepath = Some(file_path);
 73 |             }
 74 |             if let Ok(mut filesize) = self.picked_file_size.lock() {
 75 |                 *filesize = Some(file_size);
 76 |             }
 77 |         }
 78 |     }
 79 | 
 80 |     #[cfg(target_arch = "wasm32")]
 81 |     fn pick_file(&mut self, ctx: &egui::Context) {
 82 |         use wasm_bindgen::JsCast;
 83 |         use web_sys::{Event, HtmlInputElement};
 84 | 
 85 |         self.file_input_closure = None;
 86 | 
 87 |         let document = web_sys::window().unwrap().document().unwrap();
 88 | 
 89 |         let input = document.create_element("input").unwrap().dyn_into::<HtmlInputElement>().unwrap();
 90 |         input.set_attribute("type", "file").unwrap();
 91 | 
 92 |         let ctx_clone = ctx.clone();
 93 |         let shared_filename = self.picked_file_name.clone();
 94 |         let shared_filepath = self.picked_file_path.clone();
 95 |         let shared_filesize = self.picked_file_size.clone();
 96 | 
 97 |         let closure = wasm_bindgen::closure::Closure::wrap(Box::new(move |event: Event| {
 98 |             let input = event.target().unwrap().dyn_into::<HtmlInputElement>().unwrap();
 99 | 
100 |             if let Some(files) = input.files() {
101 |                 if let Some(file) = files.get(0) {
102 |                     let name = file.name();
103 |                     let size = file.size() as u64;
104 |                     // In web, path is not fully accessible for security reasons, but we can use the name
105 |                     let path = name.clone();
106 | 
107 |                     web_sys::console::log_1(&format!("Picked file: {}", name).into());
108 | 
109 |                     if let Some(window) = web_sys::window() {
110 |                         if let Some(local_storage) = window.local_storage().ok().flatten() {
111 |                             let _ = local_storage.set_item("picked", name.as_str());
112 | 
113 |                             // Update the shared states
114 |                             if let Ok(mut filename) = shared_filename.lock() {
115 |                                 *filename = Some(name);
116 |                             }
117 |                             if let Ok(mut filepath) = shared_filepath.lock() {
118 |                                 *filepath = Some(path);
119 |                             }
120 |                             if let Ok(mut filesize) = shared_filesize.lock() {
121 |                                 *filesize = Some(size);
122 |                             }
123 |                         }
124 |                     }
125 | 
126 |                     ctx_clone.request_repaint();
127 |                 }
128 |             }
129 |         }) as Box<dyn FnMut(_)>);
130 | 
131 |         input.set_onchange(Some(closure.as_ref().unchecked_ref()));
132 |         self.file_input_closure = Some(closure);
133 |         input.click();
134 |     }
135 | 
136 |     fn format_size(&self, size_bytes: u64) -> String {
137 |         const KB: u64 = 1024;
138 |         const MB: u64 = 1024 * KB;
139 |         const GB: u64 = 1024 * MB;
140 | 
141 |         if size_bytes < KB {
142 |             format!("{} bytes", size_bytes)
143 |         } else if size_bytes < MB {
144 |             format!("{:.2} KB", size_bytes as f64 / KB as f64)
145 |         } else if size_bytes < GB {
146 |             format!("{:.2} MB", size_bytes as f64 / MB as f64)
147 |         } else {
148 |             format!("{:.2} GB", size_bytes as f64 / GB as f64)
149 |         }
150 |     }
151 | 
152 |     // Added this method to fix the missing generate_magnet_uri error
153 | 
154 | 
155 |     fn show_file_info(&mut self, ui: &mut Ui) {
156 |         // Scope the mutex locks to extract data and drop guards early
157 |         let (name, path, size) = {
158 |             let file_name_binding = self.picked_file_name.lock().ok();
159 |             let file_path_binding = self.picked_file_path.lock().ok();
160 |             let file_size_binding = self.picked_file_size.lock().ok();
161 | 
162 |             match (
163 |                 file_name_binding.as_ref().map(|f| f.as_ref().cloned()),
164 |                 file_path_binding.as_ref().map(|f| f.as_ref().cloned()),
165 |                 file_size_binding.as_ref().map(|f| f.as_ref().cloned()),
166 |             ) {
167 |                 (Some(Some(name)), Some(Some(path)), Some(Some(size))) => (name, path, size),
168 |                 _ => {
169 |                     // No file selected, display message and return
170 |                     ui.add_space(15.0);
171 |                     ui.horizontal(|ui| {
172 |                         ui.spacing_mut().item_spacing.x = 8.0;
173 |                         ui.label(RichText::new("📄").color(Color32::GRAY));
174 |                         ui.label(RichText::new("No file selected").color(Color32::GRAY));
175 |                     });
176 |                     return;
177 |                 }
178 |             }
179 |         };
180 | 
181 |         // Generate magnet URI before entering the closure
182 |         // if let Some(magnet_uri) = self.generate_magnet_uri(&path) {
183 |         //     self.magnet_input = magnet_uri;
184 |         // }
185 | 
186 |         ui.add_space(15.0);
187 |         ui.group(|ui| {
188 |             ui.set_width(ui.available_width());
189 |             ui.vertical(|ui| {
190 |                 // Header with icon
191 |                 ui.horizontal(|ui| {
192 |                     ui.add(Label::new(RichText::new("📁").heading()));
193 |                     ui.heading(RichText::new("Selected File").color(Color32::from_rgb(50, 150, 200)));
194 |                 });
195 | 
196 |                 ui.add_space(8.0);
197 |                 ui.separator();
198 |                 ui.add_space(8.0);
199 | 
200 |                 // File info in a more compact layout
201 |                 Grid::new("file_info_grid")
202 |                     .num_columns(2)
203 |                     .spacing([10.0, 4.0])
204 |                     .show(ui, |ui| {
205 |                         // File name
206 |                         ui.strong("Name:");
207 |                         ui.label(&name);
208 |                         ui.end_row();
209 | 
210 |                         // File path
211 |                         ui.strong("Path:");
212 |                         ui.label(&path);
213 |                         ui.end_row();
214 | 
215 |                         // File size
216 |                         ui.strong("Size:");
217 |                         #[cfg(target_arch = "wasm32")]
218 |                         web_sys::console::log_1(&format!("============{:?}", size).into());
219 |                         ui.label(self.format_size(size));
220 |                         ui.end_row();
221 |                     });
222 | 
223 |                 ui.add_space(10.0);
224 |                 ui.separator();
225 |                 ui.add_space(10.0);
226 | 
227 |                 // Action buttons
228 |                 ui.horizontal(|ui| {
229 |                     // Share button with icon
230 |                     let share_btn = ui.add(
231 |                         Button::new(RichText::new("🔗 Share").text_style(TextStyle::Button))
232 |                     );
233 |                     share_btn.clone().on_hover_text("Generate magnet URI for this file");
234 | 
235 |                     if share_btn.clicked() {
236 |                         // if let Some(magnet_uri) = self.generate_magnet_uri(&path) {
237 |                         //     self.magnet_input = magnet_uri;
238 |                         // }
239 |                     }
240 |                 });
241 | 
242 |                 // Display generated magnet URI if available
243 |                 if !self.magnet_input.is_empty() {
244 |                     ui.add_space(10.0);
245 |                     ui.group(|ui| {
246 |                         ui.vertical(|ui| {
247 |                             ui.label(RichText::new("Magnet URI:").strong());
248 |                             ui.add_space(5.0);
249 |                             ui.label(&self.magnet_input);
250 |                             ui.add_space(5.0);
251 | 
252 |                             if ui.button("📋 Copy").clicked() {
253 |                                 ui.ctx().copy_text(self.magnet_input.clone());
254 |                             }
255 |                         });
256 |                     });
257 |                 }
258 |             });
259 |         });
260 |     }
261 | }
262 | 
263 | impl eframe::App for P2PTransfer {
264 |     fn save(&mut self, storage: &mut dyn eframe::Storage) {
265 |         eframe::set_value(storage, eframe::APP_KEY, self);
266 |     }
267 | 
268 |     fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
269 |         let frame = egui::containers::Frame::new()
270 |             .fill(ctx.style().visuals.window_fill)
271 |             .inner_margin(20.0)
272 |             .stroke(ctx.style().visuals.widgets.noninteractive.bg_stroke);
273 | 
274 |         egui::TopBottomPanel::top("top_panel")
275 |             .frame(frame.clone())
276 |             .show(ctx, |ui| {
277 |                 ui.add_space(4.0);
278 |                 egui::menu::bar(ui, |ui| {
279 |                     ui.heading(RichText::new("Syncoxiders").strong());
280 |                     ui.add_space(16.0);
281 | 
282 |                     let is_web = cfg!(target_arch = "wasm32");
283 |                     if !is_web {
284 |                         ui.menu_button("File", |ui| {
285 |                             if ui.button("Quit").clicked() {
286 |                                 ctx.send_viewport_cmd(egui::ViewportCommand::Close);
287 |                             }
288 |                         });
289 |                         ui.add_space(16.0);
290 |                     }
291 | 
292 |                     ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
293 |                         egui::widgets::global_theme_preference_buttons(ui);
294 |                     });
295 |                 });
296 |                 ui.add_space(4.0);
297 |             });
298 | 
299 |         egui::CentralPanel::default()
300 |             .frame(frame)
301 |             .show(ctx, |ui| {
302 |                 ui.vertical_centered(|ui| {
303 |                     ui.add_space(20.0);
304 |                     ui.heading(RichText::new("P2P File Sharing").size(24.0));
305 |                     ui.add_space(5.0);
306 |                     ui.label("Easily share files with a secure peer-to-peer connection");
307 |                     ui.add_space(20.0);
308 |                 });
309 | 
310 |                 ui.horizontal_centered(|ui| {
311 |                     let btn = ui.add_sized(
312 |                         Vec2::new(200.0, 40.0),
313 |                         egui::Button::new(RichText::new("Choose File").size(16.0))
314 |                             .fill(Color32::from_rgb(50, 150, 200))
315 |                     );
316 | 
317 |                     if btn.clicked() {
318 |                         #[cfg(target_arch = "wasm32")]
319 |                         self.pick_file(ctx);
320 |                         #[cfg(not(target_arch = "wasm32"))]
321 |                         self.pick_file();
322 |                     }
323 |                 });
324 | 
325 |                 self.show_file_info(ui);
326 | 
327 |                 ui.with_layout(egui::Layout::bottom_up(egui::Align::LEFT), |ui| {
328 |                     ui.horizontal(|ui| {
329 |                         ui.spacing_mut().item_spacing.x = 0.0;
330 |                         ui.label("Powered by ");
331 |                         ui.hyperlink_to("Syncoxiders", "https://github.com/emilk/eframe_template");
332 |                         ui.label(" • ");
333 |                         ui.hyperlink_to(
334 |                             "Source code",
335 |                             "https://github.com/emilk/eframe_template/blob/main/",
336 |                         );
337 |                     });
338 |                     egui::warn_if_debug_build(ui);
339 |                 });
340 |             });
341 | 
342 |         // egui::CentralPanel::default()
343 |         // .frame(frame)
344 |         // .show(ctx, |ui| {
345 |         //     ui.label("Iroh node working");
346 |         // });
347 | 
348 |     }
349 | }


--------------------------------------------------------------------------------
/p2p-transfer/src/lib.rs:
--------------------------------------------------------------------------------
1 | #![warn(clippy::all, rust_2018_idioms)]
2 | 
3 | mod app;
4 | mod node;
5 | 
6 | pub use app::P2PTransfer;
7 | 


--------------------------------------------------------------------------------
/p2p-transfer/src/main.rs:
--------------------------------------------------------------------------------
 1 | #![warn(clippy::all, rust_2018_idioms)]
 2 | #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] // hide console window on Windows in release
 3 | 
 4 | // When compiling natively:
 5 | #[cfg(not(target_arch = "wasm32"))]
 6 | fn main() -> eframe::Result {
 7 |     env_logger::init(); // Log to stderr (if you run with `RUST_LOG=debug`).
 8 | 
 9 |     let native_options = eframe::NativeOptions {
10 |         viewport: egui::ViewportBuilder::default()
11 |             .with_inner_size([400.0, 300.0])
12 |             .with_min_inner_size([300.0, 220.0])
13 |             .with_icon(
14 |                 // NOTE: Adding an icon is optional
15 |                 eframe::icon_data::from_png_bytes(&include_bytes!("../assets/icon-256.png")[..])
16 |                     .expect("Failed to load icon"),
17 |             ),
18 |         ..Default::default()
19 |     };
20 |     eframe::run_native(
21 |         "Syncoxider App",
22 |         native_options,
23 |         Box::new(|cc| Ok(Box::new(p2p_transfer::P2PTransfer::new(cc)))),
24 |     )
25 | }
26 | 
27 | // When compiling to web using trunk:
28 | #[cfg(target_arch = "wasm32")]
29 | fn main() {
30 |     use eframe::wasm_bindgen::JsCast as _;
31 |     console_error_panic_hook::set_once();
32 |     tracing_wasm::set_as_global_default();
33 | 
34 |     // Redirect `log` message to `console.log` and friends:
35 |     eframe::WebLogger::init(log::LevelFilter::Debug).ok();
36 | 
37 |     let web_options = eframe::WebOptions::default();
38 | 
39 |     wasm_bindgen_futures::spawn_local(async {
40 |         let document = web_sys::window()
41 |             .expect("No window")
42 |             .document()
43 |             .expect("No document");
44 | 
45 |         let canvas = document
46 |             .get_element_by_id("the_canvas_id")
47 |             .expect("Failed to find the_canvas_id")
48 |             .dyn_into::<web_sys::HtmlCanvasElement>()
49 |             .expect("the_canvas_id was not a HtmlCanvasElement");
50 | 
51 |         let start_result = eframe::WebRunner::new()
52 |             .start(
53 |                 canvas,
54 |                 web_options,
55 |                 Box::new(|cc| Ok(Box::new(p2p_transfer::P2PTransfer::new(cc)))),
56 |             )
57 |             .await;
58 | 
59 |         // Remove the loading text and spinner:
60 |         if let Some(loading_text) = document.get_element_by_id("loading_text") {
61 |             match start_result {
62 |                 Ok(_) => {
63 |                     loading_text.remove();
64 |                 }
65 |                 Err(e) => {
66 |                     loading_text.set_inner_html(
67 |                         "<p> The app has crashed. See the developer console for details. </p>",
68 |                     );
69 |                     panic!("Failed to start eframe: {e:?}");
70 |                 }
71 |             }
72 |         }
73 |     });
74 | }
75 | 


--------------------------------------------------------------------------------
/p2p-transfer/src/node.rs:
--------------------------------------------------------------------------------
  1 | use iroh::endpoint::{Accept, Connection};
  2 | use iroh::{Endpoint, NodeId};
  3 | use iroh::protocol::{ProtocolHandler, Router};
  4 | use tokio::sync::broadcast;
  5 | use anyhow::Result;
  6 | use async_channel::Sender;
  7 | use log::info;
  8 | use n0_future::boxed::BoxFuture;
  9 | use n0_future::future::Boxed;
 10 | use n0_future::{task, Stream};
 11 | use serde::{Deserialize, Serialize};
 12 | 
 13 | pub struct EchoNode{
 14 |     router: Router,
 15 |     accept_events: broadcast::Sender<AcceptEvent>,
 16 | }
 17 | 
 18 | impl EchoNode {
 19 |     pub async fn spawn() -> Result<Self> {
 20 | 
 21 |         let endpoint = Endpoint::builder().discovery_n0().alpns(vec![Echo::ALPN.to_vec()]).bind().await?;
 22 |         let (event_sender, _event_receiver) = broadcast::channel(128);
 23 |         let echo = Echo::new(event_sender.clone());
 24 |         let router = Router::builder(endpoint)
 25 |             .accept(Echo::ALPN, echo)
 26 |             .spawn();
 27 |         Ok(Self { router, accept_events: event_sender })
 28 | 
 29 | 
 30 |     }
 31 | 
 32 |     pub fn endpoint(&self) -> &Endpoint {
 33 |         self.router.endpoint()
 34 |     }
 35 | 
 36 |     pub fn connect(
 37 |         &self,
 38 |         node_id: NodeId,
 39 |         payload: String
 40 |     ) -> impl Stream<Item = ConnectEvent> + Unpin {
 41 | 
 42 |         let (event_sender, event_receiver) = async_channel::bounded(16);
 43 |         let endpoint = self.router.endpoint().clone();
 44 |         task::spawn(async move {
 45 |             let res = connect(&endpoint, node_id, payload, event_sender.clone()).await;
 46 |             let error = res.as_ref().err().map(|e| e.to_string());
 47 |             event_sender.send(ConnectEvent::Closed {error}).await.ok();
 48 |         });
 49 |         Box::pin(event_receiver)
 50 |     }
 51 | }
 52 | 
 53 | #[derive(Debug, Clone , Serialize, Deserialize)]
 54 | #[serde(tag="type", rename_all = "camelCase")]
 55 | pub enum ConnectEvent {
 56 |     Connected,
 57 |     Sent {bytes_sent: u64},
 58 |     Received {bytes_received: u64},
 59 |     Closed {error: Option<String>}
 60 | }
 61 | 
 62 | #[derive(Debug, Clone, Serialize, Deserialize)]
 63 | #[serde(tag="type", rename_all = "camelCase")]
 64 | pub enum AcceptEvent {
 65 | 
 66 |     Accepted {
 67 |         node_id: NodeId,
 68 |     },
 69 |     Echoed {
 70 |         node_id: NodeId,
 71 |         bytes_sent: u64
 72 |     },
 73 |     Closed {
 74 |         node_id: NodeId,
 75 |         error: Option<String>
 76 |     }
 77 | }
 78 | 
 79 | #[derive(Debug, Clone)]
 80 | pub struct Echo{
 81 |     event_sender: broadcast::Sender<AcceptEvent>
 82 | }
 83 | 
 84 | impl Echo{
 85 |     pub const ALPN: &[u8] = b"iroh/example-browser-echo/0";
 86 |     pub fn new(event_sender: broadcast::Sender<AcceptEvent>) -> Self {
 87 | 
 88 |         Self { event_sender }
 89 | 
 90 |     }
 91 | }
 92 | 
 93 | impl Echo {
 94 |     async fn handle_connection(self, connection: Connection) -> Result<()> {
 95 | 
 96 |         let node_id  = connection.remote_node_id()?;
 97 |         self.event_sender.send(AcceptEvent::Accepted {node_id }).ok();
 98 |         let res = self.handle_connection_0(&connection).await;
 99 |         let error = res.as_ref().err().map(|err| err.to_string());
100 |         self.event_sender.send(AcceptEvent::Closed {node_id, error}).ok();
101 |         res
102 | 
103 | 
104 |     }
105 | 
106 |     async fn handle_connection_0(&self, connection: &Connection) -> Result<()> {
107 | 
108 |         let node_id = connection.remote_node_id()?;
109 |         info!("Accepted connection from {}", node_id);
110 | 
111 |         let (mut send, mut recv) = connection.accept_bi().await?;
112 | 
113 |         //Echo any bytes received
114 |         let bytes_sent = tokio::io::copy(&mut recv, &mut send).await?;
115 | 
116 |         info!("Copied over {bytes_sent} byte(s)");
117 | 
118 |         self.event_sender.send(AcceptEvent::Echoed {node_id, bytes_sent}).ok();
119 | 
120 |         send.finish()?;
121 |         connection.closed().await;
122 |         Ok(())
123 | 
124 | 
125 |     }
126 | }
127 | 
128 | impl ProtocolHandler for Echo{
129 |     fn accept(&self, connection: Connection) -> BoxFuture<Result<()>> {
130 |         Box::pin(self.clone().handle_connection(connection))
131 |     }
132 | }
133 | 
134 | async fn connect(
135 |     endpoint: &Endpoint,
136 |     node_id: NodeId,
137 |     payload: String,
138 |     event_sender: Sender<ConnectEvent>
139 | ) -> Result<()>{
140 | 
141 |     let connection = endpoint.connect(node_id, Echo::ALPN).await?;
142 |     event_sender.send(ConnectEvent::Connected).await?;
143 |     let (mut send_stream , mut recv_stream) = connection.open_bi().await?;
144 |     let event_sender_clone = event_sender.clone();
145 |     let send_task = task::spawn(async move {
146 |         let event_sender = event_sender_clone.clone();
147 |         async move {
148 |             let bytes_sent = payload.len();
149 |             send_stream.write_all(payload.as_bytes()).await?;
150 |             event_sender.send(ConnectEvent::Sent {
151 |                 bytes_sent: bytes_sent as u64,
152 |             })
153 |                 .await?;
154 |             anyhow::Ok(())
155 |         }
156 |     });
157 |     let n = tokio::io::copy(&mut recv_stream, &mut tokio::io::sink()).await?;
158 |     connection.close(1u8.into(), b"done");
159 |     event_sender.send(ConnectEvent::Received {
160 |         bytes_received: n as u64,
161 |     }).await?;
162 |     send_task.await?.await?;
163 |     Ok(())
164 | 
165 | }


--------------------------------------------------------------------------------
/website/pages/compare.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../../README.md#what-separates-it-from-other-products)
 2 | 
 3 | # What separates it from others
 4 | 
 5 | ## What other solutions are there
 6 | 
 7 | MultCloud, S3 Drive, rclone, odrive, Resilio Sync.  
 8 | Some of them are hard to use especially for non-technical users and also they lack multiple features.
 9 | 
10 | - `MultCloud` seems to be very similar for Sync and Backup but lacks sync with local files. Share is only read only, the files are not kept in sync. Also it’s very buggy, login with Google doesn’t work when you access the shared link, reset pass doesn’t work well, their Google Drive app is still under review and for some reason their Android app is not in store but distributed as APK on their site. Also it lacks encryption
11 | - `S3 Drive` is a bit harder to setup as it requires some manual config in some cases providing access tokens manually. Also lacks many features like Share, Backup, Encryption. The sync is also quite slow
12 | - `rclone` is great but it’s mostly for geeks, you need to do some manual config and start it, mostly from the console. For Google Drive they have a problem, it is not quite real time Sync from remote to local. It has delay for auto-sync (even if you are listing the content of a folder it doesn’t immediately pickup changes) and does sync on some specific operations
13 | - `odrive` doesn’t have sync between providers but just offers a unified view between all your providers. For sharing it offers the other person just a view of the shared files in odrive apps, they cannot add the files to their provider for accessing in there and keeping them in sync, or save them locally and keep them in sync
14 | - `Resilio Sync` is also great but is not of the scale I'm proposing, it only handles your local files sync, not between storage providers and also doesn't have such an easy sharing
15 | 
16 | **What all are missing is:**
17 | - **An easy way to send a file to someone with just the browser**
18 | - **Share between storage providers**
19 | 
20 | ## What we offer better or additionally
21 | 
22 | **Main ones are:**
23 | - **True real-time `bi-directional Sync` and especially `Share` between storage providers (but also for local files)**
24 | - **Simple and quick way to `Share` files with someone using minimal tools, ideally only a browser**
25 | - **Simple and secure inter-cloud Encryption**
26 | 
27 | Additionally:
28 | - Receiving files, like S3 presigned URLs but it creates a dest folder where others can upload more files and even folders
29 | - Can combine both local files for Sync and Share and files on storage providers
30 | - Encryption for stored files
31 | - Backup solution with borgbackup and repo
32 | - Performant, efficient, and secure (implemented in RUST and other modern technologies)
33 | - Search and analytics
34 | - WAL (Write-Ahead Logging) to ensure file integrity
35 | - Handles very large files efficiently with concurrent and resumable transfers
36 | - Unified browser, desktop and mobile apps
37 | - Extensive cilents and access, libs, CLI app, REST API and gRPC service
38 | 


--------------------------------------------------------------------------------
/website/pages/features.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../../README.md#features)
 2 | 
 3 | # Features
 4 | 
 5 | All sync operations and changes are applied with WAL (Write-Ahead Logging) to ensure file integrity on crash or power loss. That is, any changes are first written to WAL and then applied to the file. On crash or power loss, the next time the process starts it will apply remaining changes, this repeats until all changes are successfully applied.
 6 | 
 7 | MD5, SHA1 hashes are checked at all times for file integrity. After we transfer the files we will compare local hash with remote hash to ensure data integrity.
 8 | 
 9 | All transfers are multi-threaded and resumable. I can handles very large files efficiently with parallel and resumable downloads.
10 | 
11 | - [Sync between providers, Copy, Sync, Move, Two-way sync](features/sync-providers.md#sync-between-providers)
12 | - [Conflict resolution](features/sync-providers.md#conflict-resolution)
13 | - [Sync local files between multiple authenticated devices](features/sync-local.md)
14 | - [MD5, SHA1 hashes are checked at all times for file integrity](features.md#features)
15 | - Multi-threaded transfers
16 | - [Make sync (and changes) with WAL (Write-Ahead Logging) to ensure file integrity](features.md#features)
17 | - [Share between multiple providers like Google Drive, Dropbox, S3, ...](features/share-providers.md)
18 | - [Share local files](features/share-local.md)
19 | - [Easy way to share files with external users, via a link or notification](features/share-external.md)
20 | - [Receiving files, like S3 presigned URLs but create a dest folder where others can upload more files and even folders](features/receive.md)
21 |   - deadline, late uploads, password
22 | - [Encrypted files and folders, with files saved in provider or with local files](features/encrypt.md)
23 |   - also encrypted sharing with PGP
24 | - [Browser app built in WebAssembly compatible with all major browsers](features/browser-app.md)
25 | - [Local app for desktop and mobile, access files via FUSE on Linux and macOS, WinFSP (or others) on Windows and file picker on mobile](features/local-app.md)
26 |   - encrypted cache, full copy kept in sync, notifications
27 | - [Handles very large files efficiently with concurrent and resumable transfers](features/large-files.md)
28 | - [Local views: My Drive, Computers, Shared, Shared with me, Recents, Starred, Trash](features/local-view.md)
29 | - [Global view from all providers and local files, My Drive, Computers, Shared, Shared with me, Recents, Starred, Trash](features/global-view.md)
30 | - [Internal links to other files](features/links.md)
31 | - [Search capabilities](features/search.md)
32 | - [Analytics](features/analytics.md)
33 | - [File history and versioning](features/versioning.md)
34 | - [Cleanup storage](features/cleanup.md)
35 | - [Sync, share status, storage overview](features/status.md)
36 | - [Backups with Borg, encrypted, deduplicated and compressed](features/backup.md)
37 | - [Keep emails in sync between multiple providers](features/emails.md)
38 | - [Backup your emails locally or on our Borg repo](features/emails.md)
39 | - [Anti-Virus scanning](features/antivirus.md)
40 | - [Automation, convert to PDF, convert image, unzip, convert audio/video, watermark files](features/automation.md)
41 | - [Photo manager](features/photos.md)
42 | - [REST API, gRPC, CLI clients and client libs in multiple languages](features/clients.md)
43 | - [Many supported providers](features/supported-providers.md)
44 | - [Integration with other systems](features/integrations.md)
45 | 


--------------------------------------------------------------------------------
/website/pages/features/analytics.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Analytics
 4 | 
 5 | We offer various analytics like:
 6 | - stats on types of files, images, videos, ...
 7 | - metrics on files metadata, files size, changed date, changed by, ...
 8 | - stats per provider, space used, free, transferred MB per month, in total, ...
 9 | - metrics on files activity, how often a file is changed, who did the most/recent change, most changed/used files, unused files ...
10 | - duplicate files between providers
11 | 
12 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/frame-analytics.png?raw=true)
13 | 


--------------------------------------------------------------------------------
/website/pages/features/antivirus.md:
--------------------------------------------------------------------------------
1 | # Anti-Virus scanning
2 | 
3 | When you upload a file it will be automatically scanned for viruses with [ClamAV](https://www.clamav.net) and reported to you. Files with viruses will not be accepted or will be kept in a quarantine folder that you can act upon.
4 | 


--------------------------------------------------------------------------------
/website/pages/features/automation.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Automation
 4 | 
 5 | You will be able to define some automation actions for folders. When files are added or changed in them the following actions could be performed:
 6 | - rename files
 7 | - convert to PDF
 8 | - convert image
 9 | - convert audio/video
10 | - unzip
11 | - watermark files
12 | 
13 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/automation.png?raw=true)
14 | 


--------------------------------------------------------------------------------
/website/pages/features/backup.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Backups with BorgBackup, encrypted, deduplicated and compressed
 4 | 
 5 | We’re using [BorgBackup](https://www.borgbackup.org) which handles encryption, deduplication and compression. We offer BorgBackup repos that can be used to backup data. Subscription is separate from the Sync and Share services.
 6 | 
 7 | There are 2 scopes of backups:
 8 | - **local data**: using the local app or browser app you can setup backup schedule for local files, this will be done with cron jobs, scheduled tasks on Windows, termux-job-scheduler on Android and ibimobiledevice in iOS
 9 | - **provider files**: from browser app or local app you can schedule backups from provider files. The service will need to read all files and changes from the provider and will backup on our repo. All the process is handled by the service, you don’t need the local app running for this.
10 | 
11 | When you want to restore some data you can use the local app or browser app, you’ll select the archive to restore from, you can then access files in the app and copy them locally or to a provider. In the local app you can also mount it in the OS from where you can copy the files. You can also use BorgBackup CLI or `Vorta` for GUI if you want, setup will be provided for you in the local app, browser app and on our website.
12 | 


--------------------------------------------------------------------------------
/website/pages/features/browser-app.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Browser app built in WebAssembly
 4 | 
 5 | We have a browser app as a file manager built in WebAssembly that is compatible with all major browsers. You can perform all operations from it and it handles the traffic also. Concurrent and resumable transfers.
 6 | 
 7 | It supports notifications that you receive for several events and changes, like when someone is sharing a file with you, someone accepted your share, someone changed a shared file and others.
 8 | 
 9 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/app.png?raw=true)  
10 | Image is copyrighted to `SpaceDrive`.
11 | 


--------------------------------------------------------------------------------
/website/pages/features/cleanup.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Cleanup storage
4 | 
5 | Based on the analytics overview you can choose what files to delete when you need to make more space on provider or locally.
6 | 


--------------------------------------------------------------------------------
/website/pages/features/clients.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # REST API, gRPC, CLI clients and client libs in multiple languages
4 | 
5 | We will expose a `REST API` and `gRPC` service and will have client libs in multiple languages that users can use to build their own apps.
6 | Also we will have a CLI that can be used for example to automate various flows.
7 | 
8 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/clients.png?raw=true)
9 | 


--------------------------------------------------------------------------------
/website/pages/features/emails.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Keep emails in sync between multiple providers
 4 | 
 5 | You can define sync rules between multiple email addresses. This extends mail forwarding as it’s `Two-way` sync and you can define specific filters like from what emails, what words, specific actions like where to sync it, which provider has priority, if we should mark it as read or archive it.
 6 | 
 7 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-emails.png?raw=true)
 8 | 
 9 | # Backup your emails locally or on our BorgBackup repo
10 | 
11 | From the browser app or local app you can define backup rules and schedule and app will run on those intervals and backup your emails locally.
12 | 
13 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-emails-with-service.png?raw=true)
14 | 
15 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-emails-locally.png?raw=true)
16 | 


--------------------------------------------------------------------------------
/website/pages/features/encrypt.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Encrypt
 4 | 
 5 | In transit data (between our apps, service, provider, browser, torrent clients) is always encrypted using TLS 1.3 when possible. But you may decide to keep your files encrypted (at rest), for privacy reasons. Your provider will not have access to the content of your files, nor metadata. You will need the local app or browser app to access the files. Works with local files too.
 6 | 
 7 | You can choose to encrypt the whole provider content or just selected folders.
 8 | 
 9 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-encrypted.png?raw=true)
10 | 
11 | ## Encrypted share
12 | 
13 | It uses `PGP` to handle encryption. There are 2 options:
14 | - **share with users in our service**: a session key will be generated then encrypted with destination’s user public key. The encrypted key will be sent along when sharing. If user is adding the share to providers or local app the decryption and encryption will be handled automatically. The file will be decryted in memory on demand using the session key (decrypting only the chunks that are read). On the disk or in provider it will be kept encrypted all the time
15 | - **share with external users**: if the other user is not using our service or they download the file with browser or torrent, before they can download they will need to upload their public key. After that the file is downloaded as encrypted along with the encrypted session key. Users can then decrypt the session key with their private key and then decrypt the file, this can be handled with any PGP client like GPG
16 | 
17 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-encryptyed-with-service-user.png?raw=true)
18 | 
19 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-encrypte-with-external-user.png?raw=true)
20 | 
21 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-encrypt-share.png?raw=true)
22 | 


--------------------------------------------------------------------------------
/website/pages/features/global-view.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Global view from all providers and local files, My Drive, Computers, Shared, Shared with me, Recents, Starred, Trash
4 | 
5 | Like local views but gives you an overview of files from all providers and local folders accessed based on some basic categories. Corresponds to All folder from the Local view frame. You can perform all operations on them, create new files, copy between them, share, search, ...
6 | 
7 | There is one more special folder, Shared with me at the root level, where you can keep files that are shared with you which you don’t want to add to a provider or save locally. This is accessible only from our apps and when you go there you see latest version of those files and any changes you and others make will be synced between all of you.
8 | 


--------------------------------------------------------------------------------
/website/pages/features/integrations.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Integration with other systems
 4 | 
 5 | We consider providing integration with:
 6 | - Google Workspace
 7 | - Microsoft
 8 | - Slack
 9 | - AWS
10 | - HubSpot
11 | - Autodesk
12 | - Canva
13 | - Figma
14 | - Miro
15 | 


--------------------------------------------------------------------------------
/website/pages/features/large-files.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Handles very large files efficiently with concurrent and resumable transfers for the browser app, local app and shared files
4 | 
5 | All transfers are made concurrently and are resumable. This is true for browser app, local app (if they crash or closed while transferring the next time they star the transfer will continue) and for shared files (browser transfers and with torrent clients).
6 | 
7 | When possible we use `BitTorrent` with `QUIC` or `Apache Arrow` format to sync files, this is true between local files for ex, but for provider files we use the supported provider protocols.
8 | 


--------------------------------------------------------------------------------
/website/pages/features/links.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Internal links to other files
4 | 
5 | Links to files inside same provider or local files. This is normally dependent on the underlying remote support, but in cases it’s not supported we do our best to implement inks which then you can only manage and access with our apps.
6 | 


--------------------------------------------------------------------------------
/website/pages/features/local-app.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Local app for desktop and mobile
 4 | 
 5 | We have local apps for all major desktop OSs and on mobile for Android and iOS.
 6 | 
 7 | The desktop app is very similar to browser app, in fact they share the same code. The same, will handle all operations and transfers with the service and with other P2P apps. Will expose files with FUSE on Linux and macOS, WinFSP (or others) on Windows and file picker on mobile. Concurrent and resumable transfers.
 8 | 
 9 | It operates in 2 modes:
10 | - **encrypted cache (default)**: it downloads the files on demand in real-time and keep them in a local encrypted cache with a maximum size
11 | - **full copy kept in sync**: it will download all files for each provider and keeps them in `Two-way` sync
12 | 
13 | It supports notifications that you receive for several events and changes, like when someone is sharing a file with you, someone accepted your share, someone changed a shared file and others.
14 | 
15 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/app.png?raw=true)  
16 | Image is copyrighted to `SpaceDrive`.
17 | 


--------------------------------------------------------------------------------
/website/pages/features/local-view.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Local views: My Drive, Computers, Shared, Shared with me, Recents, Starred, Trash
4 | 
5 | You will have a dedicated entry point folder for each provider and also for local folders. Inside each folder you there will be some basic categories. You can perform all operations on them, create new files, copy between provider or local folders, share, search, ...
6 | 
7 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/frame-local-view.png?raw=true)
8 | 


--------------------------------------------------------------------------------
/website/pages/features/photos.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Photo manager
4 | 
5 | We will offer a basic browser based and local app photo manager. It can manage photos from all providers, or local photos, offers basic edit operations, share photos, albums, stats, 
6 | 


--------------------------------------------------------------------------------
/website/pages/features/receive.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Receiving files
 4 | 
 5 | Someone might want to send you a file. For that you first setup a Request Files link, send it to them, and we’ll handle how the file gets back to you.
 6 | 
 7 | This is similar to [S3 presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) but create a dest folder where others can upload more files and even folders.
 8 | Useful for `One-way` one time transfers. After upload the files will not be kept in sync.
 9 | 
10 | You can create such an URL from a provider folder or local folder. For local folders the browser app or local app need to be running when the user upload the files.
11 | 
12 | We’ll notify the user with the link, or you can send it to them. You will be notified when they uploaded new files.
13 | 
14 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/frame-receive.png?raw=true)
15 | 
16 | You can define these:
17 | - title and description
18 | - destination folder: in your provider or locally
19 | - deadline: until the link is valid
20 | - late uploads: if users are allowed to upload after this date
21 | - password: a password user need to use to upload files
22 | 
23 | When user opens the link it will be presented with:
24 | - **upload with browser**: will use the browser to upload multiple files and folders
25 | - **upload with torrent**: based on a link the user’s torrent client will be started and they can upload by adding files or folders to their local folder linked to the torrent
26 | - **upload with browser app**: will start the browser file manager app (they can create an account if they want) from where they can choose to upload files from a provider (if they have an account on our service), `Shared with me` folder or local files
27 | - **upload with local app**: they can install and/or use the local app from where they can choose to upload a files from a provider (if they have an account on our service), `Shared with me` folder or local files
28 |     - a QR code will also be presented if they want to use the mobile apps
29 | 
30 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/receive-file-with-provider.png?raw=true)
31 | 
32 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/receive-local-file.png?raw=true)
33 | 
34 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-receive.png?raw=true)
35 | 


--------------------------------------------------------------------------------
/website/pages/features/search.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Search capabilities
4 | 
5 | - **metadata fields**: like name, size, modified date
6 | - **advanced search*: like shared with, last modified by
7 | - **file content**: search inside file content, this is dependent on the underlying remote support. You might opt-in for us analyzing the files and offer content search capabilities but this would require us to read all files
8 | 


--------------------------------------------------------------------------------
/website/pages/features/share-external.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Share with external users
 4 | 
 5 | You can share a file from your providers with someone not using our service or any other storage providers. We’ll generate a link, we’ll email to them, or you can send them, and they can get it from browser, with a torrent client or sync it with local our client.
 6 | 
 7 | You can initiated the share in 2 ways:
 8 | - **from browser app**: using our file manager app in browser you will pick-up a file from a provider
 9 | - **from local app**: you will initiate the share from local app
10 | 
11 | Sync mode (`Copy`, `Move`, `One-way`, `Two-way`), `compare-mode` and `conflict resolution` are handled as per above.
12 | 
13 | You can define these options when sharing:
14 | - password
15 | - expiry date
16 | 
17 | You always have the option to stop sharing.
18 | 
19 | You will be notified when they accepted or change something and they will be notified when you make changes.
20 | 
21 | When they access the link will have these options:
22 | - **download with browser**: download will be handled by their browser, it’s a `One-way` one time transfer
23 | - **download with torrent**: they will use a torrent client to get the file. We will handle the torrent tracker. This is a `One-way` transfer
24 | - **use browser app**: will start the file manager browser app (they can create an account if they want) from where they can add the files to a provider (if they have an account on our service), save it as link in `Shared with me` folder or save them locally. Files will be kept in sync between the two of you
25 | - **use local app**: they can install and/or use the local app from where they can add the files to a provider (if they have an account on our service), save it as link in `Shared with me` folder or sync them locally. Files will be kept in sync between the two of you
26 |     - a QR code will also be presented if they want to use the mobile apps
27 | - **preview**: for some types we offer preview, like for images, videos, audios, text files, PDFs, ...
28 | - 
29 | Sync mode (`Copy`, `Move`, `One-way`, `Two-way`), `compare-mode` and `conflict resolution` are handled as per above.
30 | 
31 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-share-external.png?raw=true)
32 | 


--------------------------------------------------------------------------------
/website/pages/features/share-local.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Share Local files
 4 | 
 5 | You can quickly share a file directly with someone via a link, they can get it from browser, with a torrent client or sync it with our apps.
 6 | 
 7 | If both of you are using any of our apps you can share it directly from the app and files will be kept in sync.
 8 | 
 9 | The file transfer can be handled in 2 ways:
10 | - **P2P**: in a `peer-to-peer` manner, your can use our file manager app in browser (uses `WebRTC`) or local app (uses `HTTP`, `QUIC` and `BitTorrent`). In both cases the apps need to be running for the user to download the files
11 | - **Upload to our service**: with the browser app or local app you can first upload the local files to us and user will download them directly from us. The browser app or local app doesn't need to be running for download to happen
12 | 
13 | Sync mode (`Copy`, `Move`, `One-way`, `Two-way`), `compare-mode` and `conflict resolution` are handled as per above.
14 | 
15 | You can define these options when sharing:
16 | - password
17 | - expiry date
18 | 
19 | You always have the option to stop sharing.
20 | 
21 | You will be notified when they accepted or change something and they will be notified when you make changes.
22 | 
23 | When they access the link they have the same options as above for sharing between other service users or with external users.
24 | 
25 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-local.png?raw=true)
26 | 
27 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-share-local.png?raw=true)
28 | 


--------------------------------------------------------------------------------
/website/pages/features/share-providers.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Share between service users
 4 | 
 5 | You may share a file from your provider with another person, using our service. After they accept it it will be accessible in their provider, both of you can change the file and the changes are kept in sync.
 6 | 
 7 | The service will notify the other user and he will select where to save the files, what provider and folder. You will be notified back when they accepted and also when they changed something.
 8 | 
 9 | The user may decide to not add the shared files to their providers but keep it as a link (reference), in that case the shared files will be visible to him in a Shared with me folder in the browser or local app. Any changes he makes in there will synced.
10 | 
11 | On sharing on the same provider if it supports sharing it will use its build-in sharing functionality when possible.
12 | 
13 | In other cases, it will copy the file to destination provider and then keep it in sync between the two of you. Both of you need to use our service with the providers setup in our service for this to work. Sync mode (Copy, Move, One-way, Two-way), compare-mode and conflict resolution are handled as per above.
14 | 
15 | The service will notify the user. You will be notified back when they accepted and when they changed something, they will be notified when you changed something.
16 | 
17 | When they access the link will have these options:
18 | - **download with browser**: download will be handled by their browser, it’s a `One-way` one time transfer
19 | - **download with torrent**: they will use a torrent client to get the file. We will act as the torrent tracker. This is a One-way transfer
20 | - **use browser app**: will start the file manager browser app from where they can add the files to a provider, save it as link in `Shared with me` folder or save them locally. Files will be kept in sync between the two of you
21 | - **use local app**: they can install and/or use the local app from where they can add the files to a provider, save it as link in `Shared with me` folder or save them locally. Files will be kept in sync between the two of you
22 |     - a QR code will also be presented if they want to use the mobile apps
23 | - **preview**: for some types we offer preview, like for images, videos, audios, text files, PDFs, ...
24 | 
25 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-providers.png?raw=true)
26 | 
27 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-share-provider.png?raw=true)
28 | 


--------------------------------------------------------------------------------
/website/pages/features/status.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Sync, share status, storage overview
 4 | 
 5 | We offer some health overview for you like:
 6 | - **connection status**: between your local apps, our service and providers
 7 | - **sync status**: when was the last sync, if was successful or it failed, pending changes to be synced locally, pending local changes to be synced to providers, ...
 8 | - **share**: pending shared files to be accepted, pending Request Files, ...
 9 | - **storage stats**: basic stats like used/free space per provider, type of files, ...
10 | 


--------------------------------------------------------------------------------
/website/pages/features/supported-providers.md:
--------------------------------------------------------------------------------
1 | [⟵ Back](../features.md#features)
2 | 
3 | # Many supported providers
4 | 
5 | Initially we will use [rclone](https://rclone.org#providers) on the service side and we will support all their supported providers.
6 | 
7 | In the future we plan to implement our own clients for several providers which might offer more granular and efficient sync. It helps us for example to catch remote changes almost in real-time (as soon as we are notified by the remote), and propagate the changes to other destinations, without needing to use file watchers or rely on the sync logic of rclone provider implementation.
8 | 


--------------------------------------------------------------------------------
/website/pages/features/sync-local.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Sync local files
 4 | 
 5 | Useful to sync local files between multiple devices with no remote providers involved. It will do it in a `P2P` manner using `QUIC` (will fallback to `TCP/IP` if firewalls blocks `UDP`). Your local apps need to be running for the sync to happen. This is similar to `Resilio Sync`.
 6 | 
 7 | All devices are authenticated using web of trust to make sure are trusted by you. When you setup a sync an invite link or QR code is be generated. When you open the other app based on these it will send a handshake for the device to join the sync group and you need to confirm it from any of the other devices in the group. This will give him a cryptographic key synced between all devices and only then other devices will accept sync operations with it.
 8 | 
 9 | Sync mode (`Copy`, `Move`, `One-way`, `Two-way`), `compare-mode` and `conflict resolution` are handled as per above.
10 | 
11 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-local.png?raw=true)
12 | 
13 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-sync-local.png?raw=true)
14 | 


--------------------------------------------------------------------------------
/website/pages/features/sync-providers.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # Sync between providers
 4 | 
 5 | After you setup the providers and Sync rules the sync process is automatically handled in the cloud. All your files will stay in sync in real-time and you can access them directly on providers solutions (local client or in browser) or on our local app or browser app.
 6 | 
 7 | We also offer a cloud storage solution based on S3 for now, you can setup our service as a provider and use the local app to sync the files. Then you can take full advantage of syncing those files between providers.
 8 | 
 9 | This is useful when you want to keep files in sync between multiple and/or different providers.
10 | 
11 | ## Sync modes
12 | 
13 | There are 4 modes to sync the files:
14 | 1. `Copy`: it will copy new and changes files from source to destination. No deletes or renames will occur on dst. In case the file is changed on destination also (see how this is determined based on **compare-mode** below) it will resolve the conflict using `keep-mode: path1` (see below)
15 | 2. `Move`: like `Copy` but will move files from source to destination, deleting them from source after transferred
16 | 3. `One-way`: like Copy but will also delete and rename files on destination. Conflicts are handled based on conflict resolution
17 | 4. `Two-way`: will propagate changes in both directions. In case of changes on both sides conflicts are handled based on conflict resolution
18 | 
19 | ## compare-mode
20 | 
21 | Takes a comma-separated list, with the currently supported values being `size`, `modtime`, and `hash`. For example, you could compare `size` and `modtime` but not the `hash`.
22 | 
23 | ## Conflict resolution
24 | 
25 | If configured first it will try to `auto-merge` (see below). This is disabled by default as it needs to download the remote file to perform the merge. if it doesn’t succeed it will keep the content based on `keep-mode` and will copy the other file to a new file with suffix in name like `conflict-<other>-date` indicating the other identifier and the date when the change was made.
26 | 
27 | ## auto-merge
28 | 
29 | - `text files`: it will try to merge the changes similar to Git. If there are changes on different lines the files will be merged, if the changes are on the same line it’s a conflict and will not merge anything. This is disabled by default as it needs to download the remote file
30 | - `binary files`: it will not do any auto-merge
31 | 
32 | ## keep-mode
33 | - `newer (default)`: the newer file (by modtime) is considered the winner, regardless of which side it came from. This may result in having a mix of some winners from `Path1`, and some winners from `Path2`
34 | - `older`: same as newer, except the older file is considered the winner
35 | - `bigger`: the bigger file (by `size`) is considered the winner (regardless of `modtime`, if any). This can be a useful option for remotes without `modtime` support, or with the kinds of files (such as logs) that tend to grow but not shrink, over time
36 | - `smaller`: the smaller file (by `size`) is considered the winner (regardless of `modtime`, if any)
37 | - `path1`: the version from `Path1` is unconditionally considered the winner (regardless of `modtime` and `size`, if any). This can be useful if one side is more trusted or up-to-date than the other
38 | - `path2`: same as `path1`, except the `path2` version is considered the winner
39 | 
40 | If either of the underlying remotes lacks support for the chosen method, it will be ignored and will fall back to the default of `newer`. If `modtime` is not supported either by the remote it will fallback to `path1`.
41 | 
42 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-providers.png?raw=true)
43 | 
44 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/diagram-sync-providers.png?raw=true)
45 | 


--------------------------------------------------------------------------------
/website/pages/features/versioning.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../features.md#features)
 2 | 
 3 | # File history and versioning
 4 | 
 5 | ## History
 6 | 
 7 | Many providers offer a history of changes and operations for each file. Where supported we will use that, where not we will try to implement and make it available in apps and clients. The same will offer this for local files.
 8 | 
 9 | This mostly refers to name change and activity history, not convent history, see versioning for that.
10 | 
11 | ## Versioning
12 | 
13 | Many providers support having multiple versions of the same file, as you copy a new file with same name or change it, it will be a new version. When supported by providers we will use it, when not we will try to implement it our own and make it available in apps and clients. Works also for local files.
14 | 
15 | Implementation note: this could be a good use case of git repos with LFS. Git objects are compressed and deduplicated inside same object.
16 | 


--------------------------------------------------------------------------------
/website/pages/how-it-works.md:
--------------------------------------------------------------------------------
  1 | [⟵ Back](../../README.md#how-it-works)
  2 | 
  3 | # How it works
  4 | There are several ways to interact with our service:
  5 | - **browser app**: we expose an app built for WebAssembly than can manage all operations and transfers
  6 | - **local app**: we also have a local app which is very similar to the browser app, in fact they share the same code. The same, will handle all operations and transfers with the service and with other P2P apps. Will expose files with FUSE or other technologies
  7 | - **mobile app**: similar to local app but for Android and iOS
  8 | - **clients**:
  9 |   - **CLI app**: a command line interface to interact with our service. It can also expose files with FUSE or other technologies
 10 |   - **libs**: we have libs in several languages. They can also expose files with FUSE or other technologies
 11 | - **API**: we expose a REST API and gRPC service that you can use to manage all operations and transfers. It uses WebSockets to notify you about changes
 12 | 
 13 | ![clients](https://github.com/radumarias/syncoxiders/blob/main/website/resources/clients.png?raw=true)
 14 | 
 15 | ## Sync
 16 | 
 17 | ### Files
 18 | 
 19 | #### Between providers
 20 | 
 21 | - You setup your providers on our site, you login and grant access to all providers that you need
 22 | - Define sync rules, like just Copy from one provider to another or Two-way sync between them
 23 | - From this point on we’ll handle syncing all changes between providers (no local app needed for this, it’s all happening in the cloud)
 24 | - We also offer a cloud storage solution, you can setup our service as a provider and use a local app to sync the files. Then you can take full advantage of syncing those files between providers
 25 | 
 26 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-providers.png?raw=true)
 27 | 
 28 | #### Local files
 29 | 
 30 | - In case you have some local files you want to keep in sync between multiple devices, without any file storage providers, you can use our local app on each device which will handle sync in P2P manner using QUIC. Apps need to be running for this. This is similar to Resilio Sync
 31 | - All devices are authenticated using web of trust to make sure are trusted by you
 32 | 
 33 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-local-files.png?raw=true)
 34 | 
 35 | #### Emails
 36 | 
 37 | You can define sync rules between multiple email addresses. This extends mail forwarding as it’s Two-Way sync and you can define specific filters like from what emails, what words, specific actions like where to sync it, which provider has priority, if we should mark it as read or archive it.
 38 | 
 39 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-emails.png?raw=true)
 40 | 
 41 | ## Share files
 42 | 
 43 | ### Between service users
 44 | 
 45 | - You may share a file from your provider with another person, using our service. After they accept it it will be accessible in their provider, both of you can change the file and the changes are kept in sync
 46 | - The service will notify the other user and he will select where to save the files, what provider and folder. You will be notified back when they accepted and also when they changed something
 47 | - The user may decide to not add the shared files to their providers but keep it as a link (reference), in that case the shared files will be visible to him in a Shared with me folder in the browser or local app. Any changes he makes in there will synced
 48 | - On sharing on the same provider if it supports sharing it will use their build-in sharing functionality. On other cases it will copy the file to destination provider and then keep it in sync between the two of you
 49 | 
 50 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-providers.png?raw=true)
 51 | 
 52 | ### With external users
 53 | 
 54 | - You can share a file from your providers with someone not using our service or any other storage providers. We’ll generate a link, we’ll email to them, or you can send them, and they can get it from browser, with a torrent client or sync it with our local client
 55 | - If they are using our browser or local app you will be notified when they changed something and they will be notified when you make changes
 56 | 
 57 | ### Local files
 58 | 
 59 | - You can quickly share a file directly with someone via a link, they can get it from browser, with a torrent client or sync it with our browser or local app
 60 | - If both of you are using the local app you can share directly from the app and files will be kept in sync
 61 | - The share can be made in 2 ways:
 62 |     - **P2P**: in a peer-to-peer manner, your can use our file manager app in browser (uses WebRTC) or local app (uses HTTP, QUIC and BitTorrent), in both cases they need to be running for the user to download the files
 63 |     - **Upload to our service**: with the browser app or local app you can first upload the local file to us and user will download it directly from us. The browser app or local app don’t need to be running for the download to happen
 64 | 
 65 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-local-file.png?raw=true)
 66 | 
 67 | ### Receiving files into provider
 68 | 
 69 | Someone might want to send you a file, you setup a Request Files link, send it to them, and we’ll handle how the files gets back to you.
 70 | 
 71 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/receive-file-with-provider.png?raw=true)
 72 | 
 73 | ### Receiving files locally
 74 | 
 75 | User might upload the file to our service and we’ll save it to your provider, or it can send it directly to you via P2P.
 76 | 
 77 | You can mix between these, for example you can share a local file and the other person who save it on their provider, both files will be kept in sync.
 78 | Or you can create a Request Files link based on a provider folder (or local folder) and others can send you files from their provider or local files using their browser, torrent client or local app.
 79 | 
 80 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/receive-local-file.png?raw=true)
 81 | 
 82 | ## Encrypt
 83 | 
 84 | You may want to keep your files encrypted for privacy. Your provider will not have access to the content of your files, nor metadata. You will need the local app or browser app to access the files. It works with local files too.
 85 | 
 86 | Encrypted share: it uses PGP to handle encryption. There are 2 options:
 87 | - **share with users on our service**: a session key will be generated then encrypted with destination’s user’s public key. The encrypted key will be sent along with the file when sharing. If user is adding the shared file to provider or local app the file will be decryted on demand using the session key. On the disk or on provider it will be kept encrypted all the time
 88 | - **share with external user**: if user is not using our service or downloads the file with browser or torrent, before downloading the file they will need to upload their public key. After that the file is downloaded as encrypted along with the encrypted session key. User can then decrypt the session key with his private key and then decrypt the file, this can be handled with any PGP client like GPG (instructions will be provided on the download page)
 89 | 
 90 | ### Sync encrypted between providers
 91 | 
 92 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/sync-encrypted.png?raw=true)
 93 | 
 94 | ### Share encrypted with users on our service
 95 | 
 96 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-encryptyed-with-service-user.png?raw=true)
 97 | 
 98 | ### Share encrypted with external user
 99 | 
100 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/share-encrypte-with-external-user.png?raw=true)
101 | 
102 | ## Backup
103 | 
104 | We’re using BorgBackup which handles encryption, deduplication and compression. We offer BorgBackup repos that can be used to backup data. Subscription is separate from the Sync and Share services.
105 | 
106 | ### Files
107 | 
108 | There are 2 sources of backups:
109 | - **files from provider**: from browser app or local app you can schedule backups from provider files. The service will need to read all files and changes from the provider and will backup on our repo. Everything is handled by the service, you don’t need the local app running for this
110 | - **local files**: using the local app you can setup backups schedule for local files and the local app will handle the backup process
111 | 
112 | When you want to restore some data you can use the local app, you’ll select the archive to restore from and it will be mounted in OS from where you can copy the files. You can also use BorgBackup CLI or Vorta for GUI if you want, setup will be provided for you in the local app, browser app and on our website
113 | 
114 | ### Backup files from provider
115 | 
116 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-provider.png?raw=true)
117 | 
118 | #### Backup local files
119 | 
120 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-local-files.png?raw=true)
121 | 
122 | When you want to restore some data you can use the local app, you’ll select the archive to restore from and it will be mounted in OS from where you can copy the files. You can also use borg CLI or Vorta for GUI if you want, setup will be provided for you in the local app, browser app and on our website.
123 | 
124 | ### Emails
125 | 
126 | From the browser app or local app you can define backup rules and schedule. There are 2 modes:
127 | - **backup by our service**: running backup in scheduled time will be handled by our service, all emails will be read by us from your email provider and saved to our BorgBackup repo
128 | - **backup locally**: local app will run on defined intervals, read the mails directly from your email provider and backup to our BorgBackup repo
129 | 
130 | #### Backup by our service
131 | 
132 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-emails-with-service.png?raw=true)
133 | 
134 | #### Backup locally
135 | 
136 | ![](https://github.com/radumarias/syncoxiders/blob/main/website/resources/backup-emails-locally.png?raw=true)
137 | 


--------------------------------------------------------------------------------
/website/pages/name.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../../README.md#whats-with-the-name)
 2 | 
 3 | # What's with the name
 4 | 
 5 | `SyncOxiders` :) Well:
 6 | - in Rust community the term `oxide` or `oxidize` it's often used, as it relates to `rust` as a substance
 7 | - `rs` it's short for `rust` and it's the Rust files extension
 8 | - many Rust related domains tend to be `.rs`
 9 | 
10 | So the name is `Sync` obvious, `Oxide` as mentioned before and `rs` for the domain. So we have the website [syncoxide.rs](https://syncoxide.rs/)
11 | 


--------------------------------------------------------------------------------
/website/pages/poc-demo.md:
--------------------------------------------------------------------------------
  1 | [< Back](../../README.md#poc)
  2 | 
  3 | # How it works for now
  4 | 
  5 | `One-way` `One-to-Many` sync for `Add`, `Modify`, `Delete`, `Rename` operations. You can see here a short video demo:  
  6 | [![Watch the video](https://img.youtube.com/vi/JHQC1XpCzQw/0.jpg)](https://www.youtube.com/watch?v=JHQC1XpCzQw)
  7 | 
  8 | We'll exemplify for 2 paths, from `path1` to `path2` but it works with multiple paths, it will Sync from `path1` to
  9 | others.
 10 | 
 11 | **`One-way` sync:**
 12 | 
 13 | - have 2 mounted folders with rclone (`path1`, `path2`)
 14 | - build changes tree for `path1`
 15 | - apply changes from `path1` to `path2` for these operations:
 16 |     - `Add`, `Modify`, `Delete`, `Rename`
 17 | 
 18 | We use `git` to catch the changes, how it works:
 19 | 
 20 | - we have a special directory `repo` shared for both endpoints. This will be used to create a git repo for each path and
 21 |   tracks changes. It should persist between runs
 22 | - inside the `repo` for each path we create a `tree` directory and create the tree structure from `path` in there
 23 | - in the files content we keep `size` and `mtime` or `MD5 hash` if enabled
 24 | - we do `git add .`, then `git status -s` shows what's changed, we use `git2` crate to interact with git
 25 | - after we have the changes tree we apply them to `path2`
 26 |     - on `Add` and `Modify` we check if the file is already present in `path2` and if it's the same as in `path1` we
 27 |       skip it
 28 |     - comparison between the files is made using `size`, `mtime` or `MD5 hash`, if enabled
 29 |     - on `Rename` if the `old` file is not present in the `path2` to move it, we copy it from `path1`
 30 | - changes are applied wth WAL logic, we use git changes as WAL
 31 |     - after we build the changes tree we unstage all changes
 32 |     - after we applied a change for a file we stage that file in git
 33 |     - after each `64MB` we write we are checkpointing (we do `git commit`)
 34 |     - after applying all changes we commit remainig staged ones
 35 |     - then we delete the history and keep just an index of all files so e can catch new changes
 36 |     - like this if the process is suddenly interrupted the next time it runs it will see there are changes and will
 37 |       apply them
 38 |         - this hapens until all pending changes are applied
 39 | 
 40 | # Using CLI
 41 | 
 42 | **For now you need to have the `git` client installed locally.**
 43 | 
 44 | You can find the binaries [here](https://github.com/radumarias/syncoxiders/actions/workflows/ci.yml).
 45 | Select the last successful run and at the bottom you should have binaries for multiple OSs.
 46 | 
 47 | For other targets you could clone the repo and build it, see below how.
 48 | 
 49 | You can run `syncoxiders -h` to see all args. The basic usage is like this:
 50 | 
 51 | ```bash
 52 | syncoxiders --repo <REPO> <PATH1> <PATH2>
 53 | ```
 54 | 
 55 | - `inputs` (`<PATH1> <PATH2>`): a lists of paths that will be synced
 56 | - `<REPO>`: a directory that should persist between runs, we create a `git` repo with metadata of files from all paths.
 57 |   **MUST NOT BE IN ANY OF THE PATHS**. If it doesn't persist, next time it runs it will see all files as `Add`ed, but
 58 |   will skip copying them if already the same as on the other side
 59 | 
 60 | For now, it does `One-way` sync propagating the changes from `path1` to other paths:
 61 | 
 62 | - `Add`, `Modify`, `Delete`, `Rename`
 63 | - on `Add` and `Modify` we check if the file is already present in `path2` and if it's the same as in `path1` we skip it
 64 | - comparison between the files is made using `size` and `mtime` or `MD5 hash` (if enabled, see `--checksum` param below)
 65 | - on `Rename` if the `old` file is not present in the `path2` to move it, we copy it from `path1`
 66 | 
 67 | By default it detects changes in files based on `size` and `mtime`. After copying to `path2` it will set `atime`
 68 | and `mtime` for the files.
 69 | 
 70 | Other args:
 71 | 
 72 | - `--checksum`: (disabled by default): if specified it will calculate `MD5 hash` for files when detecting changes and
 73 |   when comparing file in `path1` with the file in `path2` when applying `Add` and `Modify` operations.  
 74 |   **This is especially useful if any pf the paths is accessed over the network and doesn't support `mtime` or
 75 |   even `size` or if the clocks are out of sync**
 76 |   **It will be considerably slower when enabled**
 77 | - `--no-crc`: (disabled by default): if specified it will skip `CRC` check after file was transferred. Without this it
 78 |   compares the `CRC` of the file in `path1` with the `CRC` of the file in `path2` after transferred. This ensures the
 79 |   file integrity after transfered.
 80 |   **Checking `CRC` is highly recommend if any of the paths is accessed over the network.**
 81 | - `--dry-run`: this simulates the sync. Will not apply any changes to the paths, will just print the operations that
 82 |   would have normally be applied to paths
 83 | - `--log-all-changes`: by default it doesn't log each change that is applied, but every 100th change so it won't clutter
 84 |   the logs. Setting this will log all changes
 85 | 
 86 | ## Limitations
 87 | 
 88 | - conflicts are not handled yet. If the file is changed in both `path1` and `path2` the winner is the one from `path1`.
 89 |   It's like `master-slave` sync where `path1` is the master
 90 | - it doesn't sync any of `Add`, `Delete`, or `Rename` operations on empty folders. This is actually a limitation
 91 |   of `git` as it works only on files. The directory tree will be recreated based on the file parent, but folders with no
 92 |   files in them will not be synced
 93 | 
 94 | ## Troubleshooting
 95 | 
 96 | In case you experience any inconsistencies in the way the files are synced, or not synced, you can delete the `repo`
 97 | directory and run it again. It will see all files as new but will not copy them to the oher sides if already present and
 98 | the same, it will just copy the new or changed ones.
 99 | 
100 | ## Compile it from source code
101 | 
102 | ### Clone the repo
103 | 
104 | ```bash
105 | git clone git@github.com:radumarias/syncoxiders.git
106 | cd syncoxiders
107 | ```
108 | 
109 | ### gcc
110 | 
111 | You need to have `gcc` installed locally.
112 | 
113 | ### Install rust
114 | 
115 | [Install Rust](https://www.rust-lang.org/tools/install)
116 | 
117 | ```bash
118 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
119 | ```
120 | 
121 | #### Configuring the PATH environment variable
122 | 
123 | In the Rust development environment, all tools are installed to the `~/.cargo/bin` directory, and this is where you will
124 | find the Rust toolchain, including rustc, cargo, and rustup.
125 | 
126 | Accordingly, it is customary for Rust developers to include this directory in their `PATH` environment variable. During
127 | installation rustup will attempt to configure the `PATH`. Because of differences between platforms, command shells, and
128 | bugs in rustup, the modifications to PATH may not take effect until the console is restarted, or the user is logged out,
129 | or it may not succeed at all.
130 | 
131 | If, after installation, running `rustc --version` in the console fails, this is the most likely reason.
132 | 
133 | You can try this also:
134 | 
135 | ```bash
136 | $HOME/.cargo/env
137 | ```
138 | 
139 | ### Compile the code
140 | 
141 | ```bash
142 | cargo build --release
143 | ```
144 | 
145 | ### Run it
146 | 
147 | ```bash
148 | target/release/syncoxiders --repo <REPO> <PATH1> <PATH2>
149 | ```
150 | 
151 | # Work in progress
152 | 
153 | - resolve conflicts
154 | - `One-to-Many` sync optimization (read one time from `path1` when applying changes to multiple paths)
155 | - apply changes by chunks in parallel
156 | - apply changes to both `path1` and `path2`, `Two-Way` sync
157 | - apply changes between multiple paths, `N-Way` sync
158 | 


--------------------------------------------------------------------------------
/website/pages/stack.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../../README.md#stack)
 2 | 
 3 | We’ll build it mostly in Rust with a bit of Java (Spark and Flink) and Python (Airflow) and deployed on AWS and Kubernetes.
 4 | 
 5 | | Scope | Solution |
 6 | | ----- | -------- |
 7 | | REST API, gRPC | axum, tonic |
 8 | | Websocket | tokio-tungstenite |
 9 | | DB | RDS, ScyllaDB, Google Cloud Spanner, Neo4j |
10 | | Local app DB | SurrealDB |
11 | | Browser app | egui, wasi, wasm-bindgen, rencfs, pgp |
12 | | Local app desktop | egui, mainline, transmission_rs, cratetorrent/rqbit, <br/> quinn, rencfs, pgp, fuse3 |
13 | | Local app mobile | Kotlin Multiplatform |
14 | | Sync daemon | tokio, rencfs, gdrive-rs, git2, rclone, Flink |
15 | | Use Kafka | rdkafka |
16 | | Keycloak | axum-keycloak-auth (in app token verificaton) or <br/> Keycloak Gatekeeper (reverse proxy in front of the services) |
17 | | Event Bus | Kafka / Amazon SQS / RabbitMQ |
18 | | Streaming processor | Flink |
19 | | File storage | S3 |
20 | | Search and Analytics | ELK, Apache Spark, Apache Flink, Apache Airflow |
21 | | Identity Provider | Keycloak |
22 | | Cache | Redis |
23 | | Deploy | AWS Lambda and Amazon EKS |
24 | | Metrics | Prometheus and Grafana Mimir |
25 | | Tracing | Prometheus and Grafana Tempo |
26 | | Logs | Grafana Loki |
27 | 


--------------------------------------------------------------------------------
/website/pages/use-cases.md:
--------------------------------------------------------------------------------
 1 | [⟵ Back](../../README.md#use-cases)
 2 | 
 3 | # Use cases
 4 | 
 5 | - You have various storage cloud providers, and you need to keep files in sync between them
 6 | - You're using a primary storage cloud provider, and you want to have backups to other providers
 7 | - You want to quickly share a local file with someone with minimal tools involved
 8 | - You want to share a Dropbox file with someone who doesn’t have a Dropbox account but wants to keep that file in sync with them
 9 | - Someone wants to send you a file but doesn’t have any storage cloud provider accounts
10 | - Keep local files in sync between multiple devices, without any other storage providers, directly with P2P
11 | - You want a global view of files among all providers, My Drive, shared, starred, recent, and trash folders
12 | - You want to access your remote storage provider files locally in the filesystem
13 | - Encrypt local files or files stored on remote providers
14 | - Inter-storage cloud provider encryption
15 | - Search and analytics, what files you most often access, what types of files you have, etc.
16 | - Quick and secure backups
17 | - Keep emails in sync between multiple providers
18 | - Backup your emails
19 | - Photo manager
20 | 


--------------------------------------------------------------------------------
/website/resources/app.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/app.png


--------------------------------------------------------------------------------
/website/resources/automation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/automation.png


--------------------------------------------------------------------------------
/website/resources/backup-emails-locally.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/backup-emails-locally.png


--------------------------------------------------------------------------------
/website/resources/backup-emails-with-service.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/backup-emails-with-service.png


--------------------------------------------------------------------------------
/website/resources/backup-local-files.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/backup-local-files.png


--------------------------------------------------------------------------------
/website/resources/backup-provider.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/backup-provider.png


--------------------------------------------------------------------------------
/website/resources/clients.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/clients.png


--------------------------------------------------------------------------------
/website/resources/diagram-encrypt-share.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-encrypt-share.png


--------------------------------------------------------------------------------
/website/resources/diagram-receive.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-receive.png


--------------------------------------------------------------------------------
/website/resources/diagram-share-external.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-share-external.png


--------------------------------------------------------------------------------
/website/resources/diagram-share-local.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-share-local.png


--------------------------------------------------------------------------------
/website/resources/diagram-share-provider.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-share-provider.png


--------------------------------------------------------------------------------
/website/resources/diagram-sync-local.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-sync-local.png


--------------------------------------------------------------------------------
/website/resources/diagram-sync-providers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/diagram-sync-providers.png


--------------------------------------------------------------------------------
/website/resources/frame-analytics.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/frame-analytics.png


--------------------------------------------------------------------------------
/website/resources/frame-local-view.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/frame-local-view.png


--------------------------------------------------------------------------------
/website/resources/frame-receive.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/frame-receive.png


--------------------------------------------------------------------------------
/website/resources/mvp.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/mvp.png


--------------------------------------------------------------------------------
/website/resources/poc.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/poc.png


--------------------------------------------------------------------------------
/website/resources/receive-file-with-provider.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/receive-file-with-provider.png


--------------------------------------------------------------------------------
/website/resources/receive-local-file.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/receive-local-file.png


--------------------------------------------------------------------------------
/website/resources/services.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/services.png


--------------------------------------------------------------------------------
/website/resources/share-encrypte-with-external-user.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/share-encrypte-with-external-user.png


--------------------------------------------------------------------------------
/website/resources/share-encryptyed-with-service-user.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/share-encryptyed-with-service-user.png


--------------------------------------------------------------------------------
/website/resources/share-local.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/share-local.png


--------------------------------------------------------------------------------
/website/resources/share-providers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/share-providers.png


--------------------------------------------------------------------------------
/website/resources/sync-emails.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/sync-emails.png


--------------------------------------------------------------------------------
/website/resources/sync-encrypted.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/sync-encrypted.png


--------------------------------------------------------------------------------
/website/resources/sync-local.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/sync-local.png


--------------------------------------------------------------------------------
/website/resources/sync-providers.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/sync-providers.png


--------------------------------------------------------------------------------
/website/resources/syncoxiders-icon-20p.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/radumarias/syncoxiders/15eefe592bcd40cd8ee2338ab0a464c7516eb41b/website/resources/syncoxiders-icon-20p.png


--------------------------------------------------------------------------------