(&self.snake).is_err() {
122 | format_ident!("r#{}", self.snake)
123 | } else {
124 | format_ident!("{}", self.snake)
125 | }
126 | }
127 | pub fn set_ident(&self) -> Ident {
128 | format_ident!("set_{}", self.snake)
129 | }
130 | pub fn _get_ident(&self) -> Ident {
131 | format_ident!("get_{}", self.snake)
132 | }
133 | }
134 |
135 | pub trait FromJson: Sized {
136 | fn from_json(json: &Value) -> Self;
137 | fn from_json_file(path: &Path) -> Self {
138 | let v = serde_json::from_reader(File::open(path).unwrap()).unwrap();
139 | Self::from_json(&v)
140 | }
141 | }
142 |
143 | impl FromJson for SmithyModel {
144 | fn from_json(json: &Value) -> Self {
145 | let shapes = SmithyShapeMap::from_json(&json["shapes"]);
146 | SmithyModel { shapes }
147 | }
148 | }
149 |
150 | impl FromJson for SmithyShapeMap {
151 | fn from_json(v: &Value) -> Self {
152 | v.as_object().map_or_else(
153 | || SmithyShapeMap::new(),
154 | |m| {
155 | m.iter()
156 | .map(|(k, v)| (k.to_owned(), SmithyShape::new(v, k)))
157 | .collect()
158 | },
159 | )
160 | }
161 | }
162 |
163 | impl FromJson for SmithyMemberMap {
164 | fn from_json(v: &Value) -> Self {
165 | v.as_object().map_or_else(
166 | || SmithyMemberMap::new(),
167 | |m| {
168 | m.iter()
169 | .map(|(k, v)| (k.to_owned(), SmithyMember::new(k, v)))
170 | .collect()
171 | },
172 | )
173 | }
174 | }
175 |
176 | /// SmithyTraits provide an interface to read the traits of a shape or a member
177 | pub trait SmithyTraits {
178 | fn set_trait(&mut self, t: &str);
179 | fn has_trait(&self, t: &str) -> bool;
180 | fn get_trait(&self, t: &str) -> String;
181 | fn get_trait_value(&self, t: &str) -> Value;
182 | fn has_http_trait(&self) -> bool {
183 | self.has_trait(SM_HTTP_LABEL)
184 | || self.has_trait(SM_HTTP_QUERY)
185 | || self.has_trait(SM_HTTP_HEADER)
186 | || self.has_trait(SM_HTTP_PREFIX_HEADERS)
187 | }
188 | fn get_doc_summary(&self) -> String {
189 | let doc = self.get_trait(SM_DOC);
190 | if doc.is_empty() {
191 | return String::new();
192 | }
193 | let startp = doc.find("").map_or(0, |p| p + 3);
194 | let endp = doc.find("
").unwrap_or(doc.len());
195 | let summary = doc[startp..endp].to_string();
196 | summary.split_whitespace().collect::>().join(" ")
197 | }
198 | }
199 |
200 | /// SmithyTraitor provide an interface to get direct access to the complete traits values of a shape or a member
201 | /// It is used to implement SmithyTraits easily for shapes and members loaded from json.
202 | pub trait SmithyTraitor {
203 | fn traits(&self) -> &Value;
204 | fn traits_mut(&mut self) -> &mut Value;
205 | }
206 |
207 | /// Implement SmithyTraits for any SmithyTraitor
208 | impl SmithyTraits for T {
209 | fn set_trait(&mut self, t: &str) {
210 | self.traits_mut()
211 | .as_object_mut()
212 | .map(|o| o.insert(t.to_string(), Value::Object(JsonObject::new())));
213 | }
214 | fn has_trait(&self, t: &str) -> bool {
215 | self.traits()
216 | .as_object()
217 | .map_or(false, |o| o.contains_key(t))
218 | }
219 | fn get_trait(&self, t: &str) -> String {
220 | self.traits()
221 | .as_object()
222 | .and_then(|o| o.get(t))
223 | .and_then(|v| v.as_str())
224 | .map(|s| s.to_string())
225 | .unwrap_or_default()
226 | }
227 | fn get_trait_value(&self, t: &str) -> Value {
228 | self.traits()
229 | .as_object()
230 | .and_then(|o| o.get(t))
231 | .map_or(Value::Null, |v| v.to_owned())
232 | }
233 | }
234 |
235 | impl SmithyTraitor for SmithyShape {
236 | fn traits(&self) -> &Value {
237 | &self.traits
238 | }
239 | fn traits_mut(&mut self) -> &mut Value {
240 | &mut self.traits
241 | }
242 | }
243 |
244 | impl SmithyTraitor for SmithyMember {
245 | fn traits(&self) -> &Value {
246 | &self.traits
247 | }
248 | fn traits_mut(&mut self) -> &mut Value {
249 | &mut self.traits
250 | }
251 | }
252 |
253 | /// unprefix returns just the suffix for `prefix#suffix` strings
254 | pub fn unprefix(s: &str) -> String {
255 | s.split_once('#')
256 | .map_or_else(|| s.to_string(), |(_prefix, suffix)| suffix.to_string())
257 | }
258 |
259 | /// camel changes from MIXOfUPPERCaseAndCamelCase to MixOfUpperCaseAndCamelCase
260 | pub fn camel(s: &str) -> String {
261 | let mut out = String::new();
262 | let mut upper_streak = 0;
263 | for c in s.chars() {
264 | if c.is_uppercase() || c.is_numeric() {
265 | if upper_streak == 0 {
266 | out.push(c);
267 | } else {
268 | out.push(c.to_lowercase().next().unwrap());
269 | }
270 | upper_streak += 1;
271 | } else {
272 | if upper_streak > 1 && out.len() > 1 {
273 | let c = out.pop().unwrap();
274 | out.push(c.to_uppercase().next().unwrap());
275 | }
276 | out.push(c);
277 | upper_streak = 0;
278 | }
279 | }
280 | out
281 | }
282 |
283 | /// snake changes from CamelCase to snake_case
284 | pub fn snake(s: &str) -> String {
285 | let mut out = String::new();
286 | let mut upper_streak = 0;
287 | for mut c in s.chars() {
288 | if c.is_uppercase() || c.is_numeric() {
289 | if upper_streak == 0 && out.len() > 0 && out.chars().last().unwrap() != '_' {
290 | out.push('_');
291 | }
292 | out.push(c.to_lowercase().next().unwrap());
293 | upper_streak += 1;
294 | } else {
295 | if !c.is_alphanumeric() {
296 | c = '_';
297 | }
298 | if upper_streak > 1 && out.len() > 1 && c != '_' {
299 | let c = out.pop().unwrap();
300 | out.push('_');
301 | out.push(c);
302 | }
303 | out.push(c);
304 | upper_streak = 0;
305 | }
306 | }
307 | out
308 | }
309 |
310 | /// smithy shape types
311 | /// https://awslabs.github.io/smithy/1.0/spec/core/model.html#
312 | #[derive(Debug, Clone, Copy, PartialEq, Eq)]
313 | pub enum SmithyType {
314 | // primitive-shapes
315 | Boolean,
316 | Byte,
317 | Short,
318 | Integer,
319 | Long,
320 | Float,
321 | Double,
322 | BigInteger,
323 | BigDecimal,
324 | // basic-shapes
325 | Blob,
326 | String,
327 | Timestamp,
328 | Document,
329 | // aggregate-shapes
330 | Member,
331 | List,
332 | Set,
333 | Map,
334 | Structure,
335 | Union,
336 | // service-shapes
337 | Service,
338 | Operation,
339 | Resource,
340 | }
341 |
342 | impl SmithyType {
343 | pub fn is_always_required(&self) -> bool {
344 | match self {
345 | SmithyType::Blob
346 | | SmithyType::Boolean
347 | | SmithyType::Byte
348 | | SmithyType::Short
349 | | SmithyType::Integer
350 | | SmithyType::Long
351 | | SmithyType::Float
352 | | SmithyType::Double
353 | | SmithyType::BigInteger
354 | | SmithyType::BigDecimal => true,
355 | _ => false,
356 | }
357 | }
358 | }
359 |
360 | impl ToString for SmithyType {
361 | fn to_string(&self) -> String {
362 | self.as_ref().to_string()
363 | }
364 | }
365 |
366 | impl AsRef for SmithyType {
367 | fn as_ref(&self) -> &str {
368 | match self {
369 | // primitive-shapes
370 | SmithyType::Boolean => SM_TYPE_BOOLEAN,
371 | SmithyType::Byte => SM_TYPE_BYTE,
372 | SmithyType::Short => SM_TYPE_SHORT,
373 | SmithyType::Integer => SM_TYPE_INTEGER,
374 | SmithyType::Long => SM_TYPE_LONG,
375 | SmithyType::Float => SM_TYPE_FLOAT,
376 | SmithyType::Double => SM_TYPE_DOUBLE,
377 | SmithyType::BigInteger => SM_TYPE_BIGINTEGER,
378 | SmithyType::BigDecimal => SM_TYPE_BIGDECIMAL,
379 | // basic-shapes
380 | SmithyType::Blob => SM_TYPE_BLOB,
381 | SmithyType::String => SM_TYPE_STRING,
382 | SmithyType::Timestamp => SM_TYPE_TIMESTAMP,
383 | SmithyType::Document => SM_TYPE_DOCUMENT,
384 | // aggregate-shapes
385 | SmithyType::Member => SM_TYPE_MEMBER,
386 | SmithyType::List => SM_TYPE_LIST,
387 | SmithyType::Set => SM_TYPE_SET,
388 | SmithyType::Map => SM_TYPE_MAP,
389 | SmithyType::Structure => SM_TYPE_STRUCTURE,
390 | SmithyType::Union => SM_TYPE_UNION,
391 | // service-shapes
392 | SmithyType::Service => SM_TYPE_SERVICE,
393 | SmithyType::Operation => SM_TYPE_OPERATION,
394 | SmithyType::Resource => SM_TYPE_RESOURCE,
395 | }
396 | }
397 | }
398 |
399 | impl From<&str> for SmithyType {
400 | fn from(s: &str) -> Self {
401 | match s {
402 | // primitive-shapes
403 | SM_TYPE_BOOLEAN => SmithyType::Boolean,
404 | SM_TYPE_BYTE => SmithyType::Byte,
405 | SM_TYPE_SHORT => SmithyType::Short,
406 | SM_TYPE_INTEGER => SmithyType::Integer,
407 | SM_TYPE_LONG => SmithyType::Long,
408 | SM_TYPE_FLOAT => SmithyType::Float,
409 | SM_TYPE_DOUBLE => SmithyType::Double,
410 | SM_TYPE_BIGINTEGER => SmithyType::BigInteger,
411 | SM_TYPE_BIGDECIMAL => SmithyType::BigDecimal,
412 | // basic-shapes
413 | SM_TYPE_BLOB => SmithyType::Blob,
414 | SM_TYPE_STRING => SmithyType::String,
415 | SM_TYPE_TIMESTAMP => SmithyType::Timestamp,
416 | SM_TYPE_DOCUMENT => SmithyType::Document,
417 | // aggregate-shapes
418 | SM_TYPE_MEMBER => SmithyType::Member,
419 | SM_TYPE_LIST => SmithyType::List,
420 | SM_TYPE_SET => SmithyType::Set,
421 | SM_TYPE_MAP => SmithyType::Map,
422 | SM_TYPE_STRUCTURE => SmithyType::Structure,
423 | SM_TYPE_UNION => SmithyType::Union,
424 | // service-shapes
425 | SM_TYPE_SERVICE => SmithyType::Service,
426 | SM_TYPE_OPERATION => SmithyType::Operation,
427 | SM_TYPE_RESOURCE => SmithyType::Resource,
428 | _ => panic!("unknown SmithyType: {}", s),
429 | }
430 | }
431 | }
432 |
433 | // primitive-shapes
434 | const SM_TYPE_BOOLEAN: &str = "boolean";
435 | const SM_TYPE_BYTE: &str = "byte";
436 | const SM_TYPE_SHORT: &str = "short";
437 | const SM_TYPE_INTEGER: &str = "integer";
438 | const SM_TYPE_LONG: &str = "long";
439 | const SM_TYPE_FLOAT: &str = "float";
440 | const SM_TYPE_DOUBLE: &str = "double";
441 | const SM_TYPE_BIGINTEGER: &str = "bigInteger";
442 | const SM_TYPE_BIGDECIMAL: &str = "bigDecimal";
443 | // basic-shapes
444 | const SM_TYPE_BLOB: &str = "blob";
445 | const SM_TYPE_STRING: &str = "string";
446 | const SM_TYPE_TIMESTAMP: &str = "timestamp";
447 | const SM_TYPE_DOCUMENT: &str = "document";
448 | // aggregate-shapes
449 | const SM_TYPE_MEMBER: &str = "member";
450 | const SM_TYPE_LIST: &str = "list";
451 | const SM_TYPE_SET: &str = "set";
452 | const SM_TYPE_MAP: &str = "map";
453 | const SM_TYPE_STRUCTURE: &str = "structure";
454 | const SM_TYPE_UNION: &str = "union";
455 | // service-shapes
456 | const SM_TYPE_SERVICE: &str = "service";
457 | const SM_TYPE_OPERATION: &str = "operation";
458 | const SM_TYPE_RESOURCE: &str = "resource";
459 |
460 | // smithy traits used in s3.json:
461 | const _SM_PREFIX: &str = "smithy.api#";
462 | pub const SM_ENUM: &str = "smithy.api#enum";
463 | pub const SM_REQUIRED: &str = "smithy.api#required";
464 | const SM_DOC: &str = "smithy.api#documentation";
465 | const _SM_ERROR: &str = "smithy.api#error";
466 | const _SM_HTTP: &str = "smithy.api#http";
467 | #[allow(unused)]
468 | const SM_HTTP_LABEL: &str = "smithy.api#httpLabel";
469 | #[allow(unused)]
470 | const SM_HTTP_QUERY: &str = "smithy.api#httpQuery";
471 | #[allow(unused)]
472 | const SM_HTTP_HEADER: &str = "smithy.api#httpHeader";
473 | const _SM_HTTP_PAYLOAD: &str = "smithy.api#httpPayload";
474 | #[allow(unused)]
475 | const SM_HTTP_PREFIX_HEADERS: &str = "smithy.api#httpPrefixHeaders";
476 | const _SM_HTTP_CHECKSUM_REQUIRED: &str = "smithy.api#httpChecksumRequired";
477 | const _SM_XML_NS: &str = "smithy.api#xmlNamespace";
478 | const _SM_XML_NAME: &str = "smithy.api#xmlName";
479 | const _SM_XML_ATTR: &str = "smithy.api#xmlAttribute";
480 | const _SM_XML_FLATTENED: &str = "smithy.api#xmlFlattened";
481 | const _SM_SENSITIVE: &str = "smithy.api#sensitive";
482 | const _SM_TIMESTAMP_FORMAT: &str = "smithy.api#timestampFormat";
483 | const _SM_EVENT_PAYLOAD: &str = "smithy.api#eventPayload";
484 | const _SM_STREAMING: &str = "smithy.api#streaming";
485 | const _SM_PAGINATED: &str = "smithy.api#paginated";
486 | const _SM_DEPRECATED: &str = "smithy.api#deprecated";
487 | const _SM_TITLE: &str = "smithy.api#title";
488 | const _SM_PATTERN: &str = "smithy.api#pattern";
489 | const _SM_LENGTH: &str = "smithy.api#length";
490 | const _SM_HOST_LABEL: &str = "smithy.api#hostLabel";
491 | const _SM_ENDPOINT: &str = "smithy.api#endpoint";
492 | const _SM_AUTH: &str = "smithy.api#auth";
493 |
--------------------------------------------------------------------------------
/codegen/utils.rs:
--------------------------------------------------------------------------------
1 | use std::{
2 | fs::File,
3 | io::{BufWriter, Write},
4 | path::{Path, PathBuf},
5 | // process::{Child, ChildStdin, Command, Stdio},
6 | };
7 |
8 | /// CodeWriter pipes generated code through rustfmt and then into an output file.
9 | /// However rustfmt seems to get stuck so we had to disable it for now.
10 | pub struct CodeWriter {
11 | path: PathBuf,
12 | w: Option>,
13 | // w: Box,
14 | // rustfmt: Option,
15 | }
16 |
17 | impl CodeWriter {
18 | pub fn new(file_path: &Path) -> Self {
19 | println!("CodeWriter file {:?}", file_path);
20 | let file = File::create(file_path).unwrap();
21 | // let mut rustfmt = Command::new("rustfmt")
22 | // .arg("--edition=2021")
23 | // .stdin(Stdio::piped())
24 | // .stdout(file)
25 | // .spawn()
26 | // .unwrap();
27 | // println!("CodeWriter rustfmt {:?}", rustfmt);
28 | // let w = BufWriter::new(rustfmt.stdin.take().unwrap());
29 | let w = BufWriter::new(file);
30 | CodeWriter {
31 | path: file_path.to_path_buf(),
32 | w: Some(w),
33 | // w: Box::new(w),
34 | // rustfmt: Some(rustfmt),
35 | // rustfmt: None,
36 | }
37 | }
38 |
39 | pub fn write_code(&mut self, code: T) {
40 | self.write_all(code.to_string().as_bytes()).unwrap();
41 | self.write_all(b"\n\n").unwrap();
42 | }
43 |
44 | pub fn done(mut self) {
45 | self.flush().unwrap();
46 | }
47 | }
48 |
49 | impl Write for CodeWriter {
50 | fn write(&mut self, buf: &[u8]) -> std::io::Result {
51 | self.w.as_mut().unwrap().write(buf)
52 | // self.w.as_mut().write(buf)
53 | }
54 | fn flush(&mut self) -> std::io::Result<()> {
55 | println!("CodeWriter flush buffers {}", self.path.display());
56 | self.w.take().unwrap().flush()?;
57 | // self.w.flush()?;
58 | // println!("CodeWriter wait rustfmt {}", self.path.display());
59 | // self.rustfmt.take().unwrap().wait()?;
60 | println!("CodeWriter done {}", self.path.display());
61 | Ok(())
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/docs/architecture.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: Architecture
3 | ---
4 |
5 | > 🚧 **Warning - work in progress** 🚧
6 | >
7 | > This page is under development and not yet complete.\
8 | > Apologies for the inconvenience.
9 |
10 | # System Design
11 |
12 | 
13 |
14 | ## Service
15 |
16 | `s3d` is meant to run alongside the application(s) and provide transparent access to remote S3 storage:
17 |
18 | ```
19 | /-------------\
20 | | apps -> s3d | ----> remote S3
21 | \-------------/
22 | ```
23 |
24 | In containerized environments, such as Kubernetes, `s3d` can run in several different ways:
25 | - Per app - as a Pod or Sidecar Container.
26 | - Per node - as a DaemonSet.
27 | - Per cluster - as a scalable fleet of daemons with a Deployment + Service.
28 |
29 | ## Storage
30 |
31 | Every `s3d` instance requires its own local storage volume (FS) to store its data.
32 |
33 | This volume is recommended to be persistent to avoid data loss of pending data in the write queue,
34 | but it can be ephemeral and `s3d` will try to flush the data to S3 on shutdown.
35 |
36 | The capacity of the volume is not required to have the same size of the remote bucket,
37 | as `s3d` will use it to store pending writes, and cached reads, which allow it to operate
38 | with limited capacity by recycling the available capacity on demand.
39 |
40 | ## Security
41 |
42 | The connection from `s3d` to the remote S3 storage is encrypted and authenticated.
43 | However, the connectivity from clients to `s3d` is currently not encrypted and not authenticated.
44 | For testing and development this works fine, but for real production environments
45 | it will be required to support HTTPS and verify client requests authentication.
46 |
47 | # Software Design
48 |
49 | In order to keep `s3d` as simple as possible, and yet use bleeding-edge technology and provide a fully capable service for edge computing stack, `s3d` builds on the shoulders of great open source projects.
50 |
51 | The following sections describe the software used in the making of `s3d`:
52 |
53 | ## Rust-lang
54 |
55 | - The choice of the Rust language was a natural fit for edge systems,
56 | as it is a modern language with a focus on functionality, safety and performance.
57 | - Building with the rust toolchain into a single, standalone, lightweight binary,
58 | makes it easy to set up and configure for linux and containers,
59 | in order to run alongside any application.
60 | - Libraries from crates.io provide a great set of features for building daemons,
61 | such as the `tokio` library for async I/O, `hyper` for HTTP, etc.
62 |
63 | ## Smithy-rs
64 |
65 | - [awslabs/smithy-rs](https://github.com/awslabs/smithy-rs) builds the official AWS SDK for Rust.
66 | - It aims for high API compatibility and provides the solid S3 protocol foundation.
67 | - Using it to generate server and client S3 protocol code, and hook in the added functionality.
68 |
69 | ## Filters
70 |
71 | - A simple textual syntax is defined for fine grain object filters
72 | to include/exclude by bucket-name, key/prefix, tags, headers, meta-data.
73 |
74 | ## OpenPolicyAgent (OPA)
75 |
76 | - [OPA](https://www.openpolicyagent.org/) provides tools for declaring and evaluating policies
77 | which would extend the capabilities of filters.
78 |
79 | ## OpenTelemetry (OTEL)
80 |
81 | - [OTEL](https://opentelemetry.io/) provides a set of tools for logging, tracing and metrics.
82 | - The [opentelemetry crate](https://crates.io/crates/opentelemetry) provides the library.
83 |
84 | ## FUSE ("Filesystem in Userspace")
85 |
86 | - FUSE provides POSIX-like data access for applications that do not use the S3 API (see [kernel fuse docs](https://www.kernel.org/doc/html/latest/filesystems/fuse.html))
87 | - The daemon binds a FUSE filesystem and creates the mountpoint that maps the filesystem to the S3 API.
88 | - FUSE is a good fit for immutable files, and reading small portions of large datasets.
89 | - FUSE is a **not** a good fit for mutable files (overwrites/appends), or file locks (not supported).
90 | - The [fuser crate](https://crates.io/crates/fuser) is used to set up the FUSE binding.
91 |
92 | # Roadmap
93 |
94 | - Wasm support for filters and S3-select.
95 | - Multi-tenancy and authentication:
96 | - IAM - Identity and Access Management (long-term credentials)
97 | - STS - Secure Token Service (short-term credentials)
98 | - IMDSv2 - Instance Meta-Data Service (integrated credential provider)
99 |
--------------------------------------------------------------------------------
/docs/developer-guide.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: Developer Guide
3 | ---
4 |
5 | # Prerequisites:
6 |
7 | - [Rust](https://www.rust-lang.org/tools/install) - stable channel, currently rustc 1.58.1 (db9d1b20b 2022-01-20).
8 | - [Java 14](https://jdk.java.net/archive/) - currently openjdk 14.0.2 (2020-07-14).
9 |
10 | Notice that JDK <= 14 is still the required for smithy-rs, but this restriction would be removed with the move to gradle 7 - see [tracking issue](https://github.com/awslabs/smithy-rs/issues/1167).
11 |
12 | # Build from source
13 |
14 | Clone the repo (use a fork if you want to contribute back upstream):
15 | ```bash
16 | git clone https://github.com/s3d-rs/s3d.git
17 | cd s3d
18 | ```
19 |
20 | Build debug mode:
21 | ```bash
22 | make
23 | ```
24 |
25 | Build release mode:
26 | ```bash
27 | make RELEASE=1
28 | ```
29 |
30 | # Run locally
31 |
32 | Run from target dir:
33 | ```bash
34 | ./target/debug/s3d
35 | ```
36 |
37 | Load shell env to simplify running (run `make env` to show the commands):
38 | ```bash
39 | eval $(make env)
40 | s3d # aliased to ./target/debug/s3d
41 | eval $(make env RELEASE=1)
42 | s3d # aliased to ./target/release/s3d
43 | ```
44 |
--------------------------------------------------------------------------------
/docs/s3d-diagram.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/s3d-rs/s3d/6c23cc627ee4cc5f718efaeeb07fdec77cdeef46/docs/s3d-diagram.png
--------------------------------------------------------------------------------
/docs/user-guide.md:
--------------------------------------------------------------------------------
1 | ---
2 | title: User Guide
3 | ---
4 |
5 | > 🚧 **Warning - work in progress** 🚧
6 | >
7 | > This page is under development and most of the mentioned options are not yet available.\
8 | > Apologies for the inconvenience.
9 |
10 | # User Guide
11 |
12 | To run the daemon in foreground:
13 |
14 | ```bash
15 | s3d run
16 | ```
17 |
18 | Configuration using environment variables:
19 |
20 | - `S3D_LOCAL_DIR` - path to the local storage dir, default `$HOME/.s3d`.
21 | - `S3D_ENDPOINT` - S3 listen address, default `http://localhost:33333`.
22 | - `S3_ENDPOINT` - remote S3 address, default empty (SDK will choose default -> AWS).
23 | - `AWS_ACCESS_KEY_ID` - AWS access key ID, default empty (SDK will choose default).
24 | - `AWS_SECRET_ACCESS_KEY` - AWS secret access key, default empty (SDK will choose default).
25 | - `AWS_SESSION_TOKEN` - AWS session token, default empty (SDK will choose default).
26 |
27 | `s3d` uses the filesystem dir as a local storage, which is used for queueing, caching, and synching data from and to the remote storage.
28 |
29 | `s3d` reads the remote S3 config and credential files and environment variables
30 | just like any other S3 SDK client in order to connect to its remote storage.
31 | In addition, to support S3 compatible endpoints, it reads the `S3_ENDPOINT` environment variable.
32 |
33 | The credentials provided for `s3d` in the aws config files should be valid for the main storage,
34 | and the identity provided to `s3d` is the one it will use in all the requests to the main storage.
35 |
36 | To check and report the status of the daemon and the remote S3 storage, use:
37 |
38 | ```bash
39 | s3d status
40 | ```
41 |
42 | # Write Queue
43 |
44 | Environment variables:
45 |
46 | - `S3D_WRITE_QUEUE` - true/false, default false.
47 | - `S3D_WRITE_QUEUE_DIR` - directory to store the queue, default `$S3D_LOCAL_DIR/write_queue`.
48 | - `S3D_WRITE_QUEUE_FILTER` - object filter to push, default all.
49 | - `S3D_WRITE_QUEUE_MAX_SIZE` - maximum size of the queue in bytes, default 1GB.
50 | - `S3D_WRITE_QUEUE_MAX_FILES` - maximum number of files in the queue, default 100.
51 | - `S3D_WRITE_QUEUE_MAX_AGE` - maximum age of writes in the queue in seconds, default 3600.
52 |
53 | When enabled, `s3d` first writes new objects to files in the local store, and will push them to the main storage in the background. This is to mitigate connection issues and improve performance.
54 |
55 | When the limits are exceeded, new write requests will not be added to the queue, instead it will wait for pending writes to push and make room for it.
56 |
57 | See filters syntax for fine grain control of which data to push. In order to dynamically change the filtering of an object that was not pushed, use put-object-tagging which can be used on an existing in the write queue.
58 |
59 | # Read Cache
60 |
61 | Environment variables:
62 |
63 | - `S3D_READ_CACHE` - true/false, default false.
64 | - `S3D_READ_CACHE_DIR` - directory to store the cache, default `$S3D_LOCAL_DIR/read_cache`.
65 | - `S3D_READ_CACHE_FILTER` - object filter to cache, default all.
66 | - `S3D_READ_CACHE_MAX_SIZE` - maximum size of the cache in bytes, default 1GB.
67 | - `S3D_READ_CACHE_MAX_FILES` - maximum number of files in the cache, default 100.
68 | - `S3D_READ_CACHE_MAX_AGE` - maximum age of files in the cache in seconds, default 3600.
69 |
70 | When enabled, `s3d` will store objects in the local store on read, in order to reduce egress costs and latency on repeated reads from the main storage.
71 |
72 | When the limits are exceeded, old items from the cache will be pruned before adding new items.
73 |
74 | See filters syntax for fine grain control of which data to cache.
75 |
76 | # Filters
77 |
78 | By default, `s3d` will include all objects eligible for write queueing, read caching, and folder syncing. However, for fine control over which objects to include, filters can be configured.
79 |
80 | Here are a few examples of a filters syntax:
81 |
82 | ```re
83 | bucket[tag:key]
84 | bucket[tag:key=value]
85 | bucket[tag:key!=value]
86 | bucket[tag:key=value][tag:key=value]
87 | bucket/prefix*
88 | bucket/prefix*[tag:key]
89 | bucket/prefix*[tag:key=value]
90 | bucket/prefix*[tag:key!=value]
91 | bucket/prefix*[tag:key1=value][tag:key2=value]
92 | bucket/prefix*[hdr:content-type=value]
93 | bucket/prefix*[hdr:content-length<100]
94 | bucket/prefix*[md:custom-meta-data=value]
95 | bucket[tag:key1=val1]/prefix*[tag:key2=val2][hdr:content-type='video/*']
96 | ```
97 |
98 | Tags provide a way to update the filtering of existing objects,
99 | for example using the S3 put-object-tagging API:
100 |
101 | ```bash
102 | alias s3api='aws --endpoint localhost:33333 s3api'
103 | s3api put-object-tagging --bucket bucket --key key --tagging '{"TagSet":[
104 | { "Key": "s3d.upload", "Value": "false" }
105 | ]}'
106 | ```
107 |
108 | Notice that put-object-tagging is overriding the entire tag set, so in order to add a tag to existing set, you will need to use get-object-tagging, append to the TagSet array and then put-object-tagging.
109 |
110 | # Sync Folder
111 |
112 | When enabled, `s3d` will perform a continuous bidirectional background sync of the remote buckets with a local dir (aka "dropbox folder").
113 |
114 | The following environment variables can be used to configure the sync-folder:
115 |
116 | - `S3D_SYNC_FOLDER` - true/false, default false.
117 | - `S3D_SYNC_FOLDER_DIR` - directory to store the folder, default `$S3D_LOCAL_DIR/sync_folder`.
118 | - `S3D_SYNC_FOLDER_FILTER` - object filter to sync, default all.
119 | - `S3D_SYNC_FOLDER_MAX_SIZE` - maximum size of the folder in bytes, default 1GB.
120 | - `S3D_SYNC_FOLDER_MAX_FILES` - maximum number of files in the folder, default 100.
121 | - `S3D_SYNC_FOLDER_MAX_AGE` - maximum age of (unsync-ed) files in the folder in seconds, default 3600.
122 |
123 | When the limits are exceeded, sync will skip adding new data to the local folder.
124 | See filters syntax for fine grain control of which data to sync.
125 |
126 | # Fuse Mount
127 |
128 | When enabled, `s3d` will set up a FUSE mount point, which exposes the same buckets and objects through a POSIX-like file interface.
129 |
130 | The following environment variables can be used to configure the fuse-mount:
131 |
132 | - `S3D_FUSE_MOUNT` - true/false, default false.
133 | - `S3D_FUSE_MOUNT_DIR` - directory to bind the mount point, default `$S3D_LOCAL_DIR/fuse_mount`.
134 |
135 | # Kubernetes Deployment
136 |
137 | Container image for s3d can be built using the `Dockerfile` in the project root (also used in `make image`):
138 | ```bash
139 | IMG="//s3d:"
140 | docker build . -t $IMG
141 | docker push $IMG
142 | ```
143 |
144 | For Kubernetes see `examples/s3d-kube-deploy.yaml` which you can use as a base, just REMEMBER to set the image to match the one you built and pushed above.
145 |
146 | ```bash
147 | # set image in yaml ... TODO kustomize
148 | kubectl apply -f examples/s3d-kube-deploy.yaml
149 | ```
150 |
--------------------------------------------------------------------------------
/examples/k8s-s3d-deployment.yaml:
--------------------------------------------------------------------------------
1 | # TODO: Work in progress
2 | ---
3 | apiVersion: apps/v1
4 | kind: Deployment
5 | metadata:
6 | name: s3d
7 | labels:
8 | app: s3d
9 | spec:
10 | replicas: 1
11 | selector:
12 | matchLabels:
13 | app: s3d
14 | template:
15 | metadata:
16 | labels:
17 | app: s3d
18 | spec:
19 | containers:
20 | - name: s3d
21 | image: s3d:dev
22 | ports:
23 | - containerPort: 33333
24 | env:
25 | - name: TODO
26 | value: "Work in progress"
27 | ---
28 | apiVersion: v1
29 | kind: Service
30 | metadata:
31 | name: s3d
32 | labels:
33 | app: s3d
34 | spec:
35 | type: NodePort
36 | selector:
37 | app: s3d
38 | ports:
39 | - port: 33333
40 | targetPort: 33333
41 |
--------------------------------------------------------------------------------
/s3d.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/s3d-rs/s3d/6c23cc627ee4cc5f718efaeeb07fdec77cdeef46/s3d.png
--------------------------------------------------------------------------------
/src/bin/s3.rs:
--------------------------------------------------------------------------------
1 | extern crate s3d;
2 | use clap::{IntoApp, Parser};
3 | use std::fmt::Debug;
4 |
5 | #[tokio::main]
6 | pub async fn main() -> anyhow::Result<()> {
7 | // env_logger::init_from_env(env_logger::Env::default().default_filter_or("warn,s3d=info"));
8 | env_logger::init();
9 | CLI::parse().run().await
10 | }
11 |
12 | #[derive(clap::Parser, Debug, Clone)]
13 | #[clap(name = "s3")]
14 | #[clap(about = "S3 CLI tool for applications or services that need to access S3 buckets (with/out the s3d daemon)")]
15 | #[clap(version = clap::crate_version!())]
16 | #[clap(setting = clap::AppSettings::DeriveDisplayOrder)]
17 | pub struct CLI {
18 | /// subcommand
19 | #[clap(subcommand)]
20 | cmd: Cmd,
21 | }
22 |
23 | impl CLI {
24 | pub async fn run(self) -> anyhow::Result<()> {
25 | log::debug!("{:?}", self);
26 | match self.cmd {
27 | Cmd::Api(cmd) => cmd.run().await,
28 | Cmd::List(cmd) => cmd.run().await,
29 | Cmd::Get(cmd) => cmd.run().await,
30 | Cmd::Put(cmd) => cmd.run().await,
31 | // Cmd::Tag(cmd) => cmd.run().await,
32 | // Cmd::Remote(cmd) => cmd.run().await,
33 | // Cmd::Status(cmd) => cmd.run().await,
34 | Cmd::Completion(cmd) => cmd.run(CLI::command()).await,
35 | }
36 | }
37 | }
38 |
39 | #[derive(clap::Subcommand, Debug, Clone)]
40 | enum Cmd {
41 | Api(s3d::cli::api_cmd::ApiCmd),
42 | List(s3d::cli::list_cmd::ListCmd),
43 | Get(s3d::cli::get_cmd::GetCmd),
44 | Put(s3d::cli::put_cmd::PutCmd),
45 | // Tag(s3d::cli::tag_cmd::TagCmd),
46 | // Remote(s3d::cli::remote_cmd::RemoteCmd),
47 | // Status(s3d::cli::status_cmd::StatusCmd),
48 | Completion(s3d::cli::completion_cmd::CompletionCmd),
49 | }
50 |
51 | // #[clap(about = "Init sets up config and local store for the daemon")]
52 | // #[clap(about = "Fetch metadata only from remote")]
53 | // #[clap(about = "Pull changes from remote")]
54 | // #[clap(about = "Push changes to remote")]
55 | // #[clap(about = "Prune objects from local store")]
56 | // #[clap(about = "Diff shows objects pending for pull/push")]
57 |
--------------------------------------------------------------------------------
/src/bin/s3d.rs:
--------------------------------------------------------------------------------
1 | extern crate s3d;
2 | use clap::Parser;
3 | use std::fmt::Debug;
4 |
5 | #[tokio::main]
6 | pub async fn main() -> anyhow::Result<()> {
7 | // env_logger::init();
8 | env_logger::init_from_env(env_logger::Env::default().default_filter_or("warn,s3d=info"));
9 | Daemon::parse().run().await
10 | }
11 |
12 | #[derive(clap::Parser, Debug, Clone)]
13 | #[clap(name = "s3d")]
14 | #[clap(about = clap::crate_description!())]
15 | #[clap(version = clap::crate_version!())]
16 | #[clap(setting = clap::AppSettings::DeriveDisplayOrder)]
17 | pub struct Daemon {}
18 |
19 | impl Daemon {
20 | pub async fn run(self) -> anyhow::Result<()> {
21 | log::debug!("{:?}", self);
22 | #[cfg(feature = "fuse")]
23 | {
24 | s3d::fuse::Fuse::start_fuse_mount().await?;
25 | }
26 | s3d::s3::server::serve().await?;
27 | Ok(())
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/cli/api_cmd.rs:
--------------------------------------------------------------------------------
1 | // S3OpsCommands is generated into build_gen and contain a subcommand per operation in the model.
2 | use crate::codegen_include::S3OpsCommands;
3 | use crate::utils::{new_s3_client, staticify};
4 |
5 | /// Call S3 API operations
6 | #[derive(clap::Parser, Debug, Clone)]
7 | pub struct ApiCmd {
8 | #[clap(subcommand)]
9 | op: S3OpsCommands,
10 | }
11 |
12 | impl ApiCmd {
13 | pub async fn run(&self) -> anyhow::Result<()> {
14 | let s3 = staticify(new_s3_client().await);
15 | self.op.run(s3).await;
16 | Ok(())
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/cli/completion_cmd.rs:
--------------------------------------------------------------------------------
1 | /// Generates shell commpletion script
2 | #[derive(clap::Parser, Debug, Clone)]
3 | pub struct CompletionCmd {
4 | #[clap(arg_enum)]
5 | shell: clap_complete::Shell,
6 | }
7 |
8 | impl CompletionCmd {
9 | pub async fn run(&self, mut cmd: clap::Command<'_>) -> anyhow::Result<()> {
10 | let name = cmd.get_name().to_owned();
11 | Ok(clap_complete::generate(
12 | self.shell,
13 | &mut cmd,
14 | name,
15 | &mut std::io::stdout(),
16 | ))
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/cli/get_cmd.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::{new_s3_client, parse_bucket_and_key, pipe_stream_to_outfile_or_stdout};
2 |
3 | /// Get object data to stdout, and meta-data and tags to stderr
4 | #[derive(clap::Parser, Debug, Clone)]
5 | pub struct GetCmd {
6 | /// Get object from `bucket/key`
7 | #[clap(name = "bucket/key")]
8 | bucket_and_key: String,
9 |
10 | /// Output file name, if not specified, stdout is used
11 | #[clap(name = "outfile")]
12 | outfile: Option,
13 | }
14 |
15 | impl GetCmd {
16 | pub async fn run(&self) -> anyhow::Result<()> {
17 | let s3 = new_s3_client().await;
18 | let (bucket, key) = parse_bucket_and_key(&self.bucket_and_key)?;
19 | let mut res = s3.get_object().bucket(bucket).key(key).send().await?;
20 |
21 | info!("{:#?}", res);
22 |
23 | let num_bytes = pipe_stream_to_outfile_or_stdout(&mut res.body, self.outfile.as_deref()).await?;
24 |
25 | info!("Received {} bytes", num_bytes);
26 |
27 | Ok(())
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/cli/list_cmd.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::{new_s3_client, parse_bucket_and_prefix};
2 | use aws_smithy_types::date_time::Format;
3 |
4 | /// List buckets or objects
5 | #[derive(clap::Parser, Debug, Clone)]
6 | #[clap(aliases = &["ls"])]
7 | pub struct ListCmd {
8 | /// When empty list all buckets.
9 | /// Otherwise list objects in bucket with optional key prefix (`bucket` or `bucket/prefix`)
10 | #[clap(name = "bucket[/prefix]", default_value = "")]
11 | bucket_and_prefix: String,
12 | }
13 |
14 | impl ListCmd {
15 | pub async fn run(&self) -> anyhow::Result<()> {
16 | let s3 = new_s3_client().await;
17 | let (bucket, prefix) = parse_bucket_and_prefix(&self.bucket_and_prefix)?;
18 |
19 | if bucket.is_empty() {
20 | let res = s3.list_buckets().send().await?;
21 | for it in res.buckets.unwrap_or_default() {
22 | println!(
23 | "{} {}",
24 | it.creation_date().unwrap().fmt(Format::DateTime).unwrap(),
25 | it.name().unwrap()
26 | );
27 | }
28 | } else {
29 | let res = s3
30 | .list_objects()
31 | .bucket(bucket)
32 | .prefix(prefix)
33 | .delimiter("/")
34 | .send()
35 | .await?;
36 | for it in res.common_prefixes.unwrap_or_default() {
37 | println!(
38 | "{:.^20} {:.>12} {}",
39 | "PREFIX",
40 | "",
41 | it.prefix().unwrap()
42 | );
43 | }
44 | for it in res.contents.unwrap_or_default() {
45 | println!(
46 | "{:>20} {:.>12} {}",
47 | it.last_modified().unwrap().fmt(Format::DateTime).unwrap(),
48 | it.size(),
49 | it.key().unwrap(),
50 | );
51 | }
52 | }
53 |
54 | Ok(())
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/cli/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod api_cmd;
2 | pub mod completion_cmd;
3 | pub mod get_cmd;
4 | pub mod list_cmd;
5 | pub mod put_cmd;
6 | pub mod tag_cmd;
7 |
--------------------------------------------------------------------------------
/src/cli/put_cmd.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::{byte_stream_from_infile_or_stdin, new_s3_client, parse_bucket_and_key};
2 |
3 | /// Put object from stdin
4 | #[derive(clap::Parser, Debug, Clone)]
5 | pub struct PutCmd {
6 | /// Put object in `bucket/key`
7 | #[clap(name = "bucket/key")]
8 | bucket_and_key: String,
9 |
10 | /// Input file name, if not specified, stdin is used
11 | #[clap(name = "infile")]
12 | infile: Option,
13 | }
14 |
15 | impl PutCmd {
16 | pub async fn run(&self) -> anyhow::Result<()> {
17 | let s3 = new_s3_client().await;
18 | let (bucket, key) = parse_bucket_and_key(&self.bucket_and_key)?;
19 | let body = byte_stream_from_infile_or_stdin(self.infile.as_deref()).await?;
20 | let res = s3
21 | .put_object()
22 | .bucket(bucket)
23 | .key(key)
24 | .body(body)
25 | .send()
26 | .await?;
27 | info!("{:#?}", res);
28 |
29 | Ok(())
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/cli/tag_cmd.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::{new_s3_client, parse_bucket_and_key};
2 |
3 | /// Get or set tags for bucket or object
4 | #[derive(clap::Parser, Debug, Clone)]
5 | pub struct TagCmd {
6 | /// Set tags for `bucket` or `bucket/key`
7 | #[clap(name = "bucket[/key]")]
8 | bucket_and_key: String,
9 |
10 | /// Tag `name=value`. Can be used multiple times.
11 | #[clap(long, short, multiple_occurrences(true))]
12 | tag: Option>,
13 |
14 | /// Reset previous tags instead of appending
15 | #[clap(long, short)]
16 | reset: bool,
17 | }
18 |
19 | impl TagCmd {
20 | pub async fn run(&self) -> anyhow::Result<()> {
21 | let s3 = new_s3_client().await;
22 | let (bucket, key) = parse_bucket_and_key(&self.bucket_and_key)?;
23 | let tagging = aws_sdk_s3::model::Tagging::builder()
24 | .set_tag_set(self.tag.clone().map(|v| {
25 | v.iter()
26 | .map(|t| {
27 | let mut parts = t.splitn(2, '=');
28 | let k = parts.next().map(String::from);
29 | let v = parts.next().map(String::from);
30 | aws_sdk_s3::model::Tag::builder()
31 | .set_key(k)
32 | .set_value(v)
33 | .build()
34 | })
35 | .collect::>()
36 | }))
37 | .build();
38 |
39 | if tagging.tag_set().is_none() {
40 | if key.is_empty() {
41 | let res = s3.get_bucket_tagging().bucket(bucket).send().await?;
42 | info!("{:#?}", res);
43 | } else {
44 | let res = s3
45 | .get_object_tagging()
46 | .bucket(bucket)
47 | .key(key)
48 | .send()
49 | .await?;
50 | info!("{:#?}", res);
51 | }
52 | } else {
53 | if key.is_empty() {
54 | let res = s3
55 | .put_bucket_tagging()
56 | .bucket(bucket)
57 | .tagging(tagging)
58 | .send()
59 | .await?;
60 | info!("{:#?}", res);
61 | } else {
62 | let res = s3
63 | .put_object_tagging()
64 | .bucket(bucket)
65 | .key(key)
66 | .tagging(tagging)
67 | .send()
68 | .await?;
69 | info!("{:#?}", res);
70 | }
71 | }
72 |
73 | Ok(())
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/codegen_include.rs:
--------------------------------------------------------------------------------
1 | //! This includes files generated by build.rs.
2 | #![allow(unused)]
3 | include!(concat!(env!("OUT_DIR"), "/s3_cli.rs"));
4 | include!(concat!(env!("OUT_DIR"), "/s3_conv.rs"));
5 |
--------------------------------------------------------------------------------
/src/config.rs:
--------------------------------------------------------------------------------
1 | macro_rules! env_config {
2 | ($env:ident optional) => {
3 | lazy_static::lazy_static! {
4 | pub static ref $env : Option = std::env::var(stringify!($env)).ok();
5 | }
6 | };
7 | ($env:ident required) => {
8 | lazy_static::lazy_static! {
9 | pub static ref $env : String = std::env::var(stringify!($env)).unwrap();
10 | }
11 | };
12 | ($env:ident default $default_val:expr) => {
13 | lazy_static::lazy_static! {
14 | pub static ref $env : String = std::env::var(stringify!($env)).unwrap_or(($default_val).to_string());
15 | }
16 | };
17 | }
18 |
19 | env_config!(HOME required);
20 | env_config!(S3D_LOCAL_DIR default ".s3d");
21 | env_config!(S3D_ENDPOINT default "http://localhost:33333");
22 |
23 | env_config!(S3_ENDPOINT optional);
24 | env_config!(S3_ACCESS_KEY optional);
25 | env_config!(S3_SECRET_KEY optional);
26 |
27 | env_config!(S3D_WRITE_QUEUE default "false");
28 | env_config!(S3D_WRITE_QUEUE_DIR default format!("{}/write_queue", *S3D_LOCAL_DIR));
29 | env_config!(S3D_WRITE_QUEUE_FILTER optional);
30 | env_config!(S3D_WRITE_QUEUE_MAX_SIZE optional);
31 | env_config!(S3D_WRITE_QUEUE_MAX_FILES optional);
32 | env_config!(S3D_WRITE_QUEUE_MAX_AGE optional);
33 |
34 | env_config!(S3D_READ_CACHE default "false");
35 | env_config!(S3D_READ_CACHE_DIR default format!("{}/read_cache", *S3D_LOCAL_DIR));
36 | env_config!(S3D_READ_CACHE_FILTER optional);
37 | env_config!(S3D_READ_CACHE_MAX_SIZE optional);
38 | env_config!(S3D_READ_CACHE_MAX_FILES optional);
39 | env_config!(S3D_READ_CACHE_MAX_AGE optional);
40 |
41 | env_config!(S3D_SYNC_FOLDER default "false");
42 | env_config!(S3D_SYNC_FOLDER_DIR default format!("{}/sync_folder", *S3D_LOCAL_DIR));
43 | env_config!(S3D_SYNC_FOLDER_FILTER optional);
44 | env_config!(S3D_SYNC_FOLDER_MAX_SIZE optional);
45 | env_config!(S3D_SYNC_FOLDER_MAX_FILES optional);
46 | env_config!(S3D_SYNC_FOLDER_MAX_AGE optional);
47 |
48 | env_config!(S3D_FUSE_MOUNT default "false");
49 | env_config!(S3D_FUSE_MOUNT_DIR default format!("{}/fuse_mount", *S3D_LOCAL_DIR));
50 |
--------------------------------------------------------------------------------
/src/fuse.rs:
--------------------------------------------------------------------------------
1 | //! FUSE
2 | //!
3 | //! - Filesystems in the Linux kernel - FUSE
4 | //! https://www.kernel.org/doc/html/latest/filesystems/fuse.html
5 | //!
6 | //! - To FUSE or Not to FUSE: Performance of User-Space File Systems
7 | //! https://www.usenix.org/system/files/conference/fast17/fast17-vangoor.pdf
8 |
9 | use crate::config;
10 | use crate::utils::staticify;
11 | use fuser::{FileAttr, FileType, Filesystem, Request};
12 | use std::time::Duration;
13 |
14 | pub const BLOCK_SIZE: u32 = 4096;
15 | pub const NAMELEN: u32 = 1024;
16 | pub const KB: u64 = 1u64 << 10;
17 | pub const MB: u64 = 1u64 << 20;
18 | pub const GB: u64 = 1u64 << 30;
19 | pub const TB: u64 = 1u64 << 40;
20 | pub const PB: u64 = 1u64 << 50;
21 |
22 | pub struct Fuse {}
23 |
24 | impl Fuse {
25 | pub async fn start_fuse_mount() -> anyhow::Result<()> {
26 | if *config::S3D_FUSE_MOUNT != "true" {
27 | debug!("Fuse mount disabled");
28 | return Ok(());
29 | }
30 | info!("Fuse mount enabled");
31 |
32 | let fuse = staticify(Fuse {});
33 |
34 | let mountpoint = config::S3D_FUSE_MOUNT_DIR.as_str();
35 | if mountpoint.is_empty() {
36 | return Ok(());
37 | }
38 | let mut session = fuser::Session::new(
39 | fuse,
40 | mountpoint.as_ref(),
41 | &[
42 | fuser::MountOption::RW,
43 | // fuser::MountOption::RO,
44 | // fuser::MountOption::Sync,
45 | // fuser::MountOption::DirSync,
46 | // fuser::MountOption::Async,
47 | fuser::MountOption::AllowRoot,
48 | fuser::MountOption::AllowOther,
49 | fuser::MountOption::AutoUnmount,
50 | fuser::MountOption::DefaultPermissions,
51 | fuser::MountOption::NoDev,
52 | fuser::MountOption::NoSuid,
53 | fuser::MountOption::NoAtime,
54 | fuser::MountOption::CUSTOM("nobrowse".to_string()),
55 | fuser::MountOption::FSName("s3d".to_string()),
56 | fuser::MountOption::Subtype("s3d".to_string()),
57 | ],
58 | )?;
59 | // run the fuse event loop in a separate thread
60 | let res = tokio::task::spawn_blocking(move || session.run()).await;
61 | Ok(res??)
62 | }
63 |
64 | fn make_fuse_attr(&self, ino: u64, kind: FileType, size: u64) -> FileAttr {
65 | let now = std::time::SystemTime::now();
66 | FileAttr {
67 | ino, // inode's number
68 | size,
69 | blocks: (size + (BLOCK_SIZE as u64) - 1) / BLOCK_SIZE as u64,
70 | blksize: BLOCK_SIZE,
71 | kind,
72 | rdev: 0, // device type, for special file inode
73 | uid: unsafe { libc::geteuid() }, // user-id of owner
74 | gid: unsafe { libc::getegid() }, // group-id of owner
75 | perm: if kind == FileType::Directory {
76 | 0o755
77 | } else {
78 | 0o644
79 | }, // inode protection mode
80 | nlink: if kind == FileType::Directory {
81 | 2 // parent + '.' + (subdirs * '..')
82 | } else {
83 | 1
84 | }, // number of hard links to the file
85 | flags: 0,
86 | atime: now,
87 | mtime: now,
88 | ctime: now,
89 | crtime: now,
90 | }
91 | }
92 | }
93 |
94 | impl Filesystem for &Fuse {
95 | fn statfs(&mut self, _req: &Request<'_>, ino: u64, reply: fuser::ReplyStatfs) {
96 | trace!("FUSE::statfs() ino={}", ino);
97 | reply.statfs(
98 | 42, // total data blocks in file system
99 | 1u64 << 38, // free blocks in fs
100 | 1u64 << 38, // free blocks avail to non-superuser
101 | 42, // total file nodes in file system
102 | 1_000_000_000, // free file nodes in fs
103 | BLOCK_SIZE, // fundamental file system block size
104 | 1024, // namelen
105 | 1024 * 1024, // optimal transfer block size
106 | );
107 | }
108 |
109 | fn open(&mut self, _req: &Request<'_>, ino: u64, flags: i32, reply: fuser::ReplyOpen) {
110 | trace!("FUSE::open() ino={} flags={}", ino, flags);
111 | reply.opened(ino, flags as u32);
112 | }
113 |
114 | fn release(
115 | &mut self,
116 | _req: &Request<'_>,
117 | ino: u64,
118 | fh: u64,
119 | flags: i32,
120 | lock_owner: Option,
121 | flush: bool,
122 | reply: fuser::ReplyEmpty,
123 | ) {
124 | trace!(
125 | "FUSE::release() ino={} fh={} flags={} lock_owner={:?} flush={}",
126 | ino,
127 | fh,
128 | flags,
129 | lock_owner,
130 | flush
131 | );
132 | reply.ok();
133 | }
134 |
135 | fn opendir(&mut self, _req: &Request<'_>, ino: u64, flags: i32, reply: fuser::ReplyOpen) {
136 | trace!("FUSE::opendir() ino={} flags={}", ino, flags);
137 | if ino < 1000 {
138 | reply.opened(ino, flags as u32);
139 | } else {
140 | reply.error(libc::ENOTDIR);
141 | }
142 | }
143 |
144 | fn releasedir(
145 | &mut self,
146 | _req: &Request<'_>,
147 | ino: u64,
148 | fh: u64,
149 | flags: i32,
150 | reply: fuser::ReplyEmpty,
151 | ) {
152 | trace!("FUSE::releasedir() ino={} fh={} flags={}", ino, fh, flags);
153 | if ino < 1000 {
154 | reply.ok();
155 | } else {
156 | reply.error(libc::ENOTDIR);
157 | }
158 | }
159 |
160 | fn lookup(
161 | &mut self,
162 | _req: &Request<'_>,
163 | ino: u64,
164 | name: &std::ffi::OsStr,
165 | reply: fuser::ReplyEntry,
166 | ) {
167 | let name = name.to_str().unwrap();
168 | trace!("FUSE::lookup() ino={} name={}", ino, name);
169 | if ino >= 1000 {
170 | reply.error(libc::ENOTDIR);
171 | return;
172 | }
173 | if !name.starts_with("file") {
174 | reply.error(libc::ENOENT);
175 | return;
176 | }
177 | let i = name[4..].parse::().unwrap();
178 | let kind = if i < 1000 {
179 | FileType::Directory
180 | } else {
181 | FileType::RegularFile
182 | };
183 | let attr = self.make_fuse_attr(i, kind, 10);
184 | let ttl = Duration::from_secs(60);
185 | reply.entry(&ttl, &attr, 0);
186 | }
187 |
188 | fn readdir(
189 | &mut self,
190 | _req: &Request<'_>,
191 | ino: u64,
192 | fh: u64,
193 | offset: i64,
194 | mut reply: fuser::ReplyDirectory,
195 | ) {
196 | trace!("FUSE::readdir() ino={} fh={} offset={}", ino, fh, offset);
197 | if ino >= 1000 {
198 | reply.error(libc::ENOTDIR);
199 | return;
200 | }
201 | for i in 1000..1003 as u64 {
202 | if i > offset as u64 {
203 | if reply.add(i, i as i64, FileType::RegularFile, &format!("file{}", i)) {
204 | break;
205 | }
206 | }
207 | }
208 | reply.ok();
209 | }
210 |
211 | fn readdirplus(
212 | &mut self,
213 | _req: &Request<'_>,
214 | ino: u64,
215 | fh: u64,
216 | offset: i64,
217 | mut reply: fuser::ReplyDirectoryPlus,
218 | ) {
219 | trace!(
220 | "FUSE::readdirplus() ino={} fh={} offset={}",
221 | ino,
222 | fh,
223 | offset
224 | );
225 | if ino >= 1000 {
226 | reply.error(libc::ENOTDIR);
227 | return;
228 | }
229 | let ttl = Duration::from_secs(60);
230 | for i in 1000..1003 as u64 {
231 | if i >= offset as u64 {
232 | let attr = self.make_fuse_attr(i as u64, FileType::RegularFile, 10);
233 | reply.add(i, i as i64, &format!("file{}", i), &ttl, &attr, 0);
234 | }
235 | }
236 | reply.ok();
237 | }
238 |
239 | fn getattr(&mut self, _req: &Request<'_>, ino: u64, reply: fuser::ReplyAttr) {
240 | trace!("FUSE::getattr() ino={}", ino);
241 | let ttl = Duration::from_secs(60);
242 | if ino < 1000 {
243 | let attr = self.make_fuse_attr(ino, FileType::Directory, 10);
244 | reply.attr(&ttl, &attr);
245 | } else {
246 | let attr = self.make_fuse_attr(ino, FileType::RegularFile, 10);
247 | reply.attr(&ttl, &attr);
248 | }
249 | }
250 |
251 | fn read(
252 | &mut self,
253 | _req: &Request<'_>,
254 | ino: u64,
255 | fh: u64,
256 | offset: i64,
257 | size: u32,
258 | flags: i32,
259 | lock_owner: Option,
260 | reply: fuser::ReplyData,
261 | ) {
262 | trace!(
263 | "FUSE::read() ino={} fh={} offset={} size={} flags={} lock_owner={:?}",
264 | ino,
265 | fh,
266 | offset,
267 | size,
268 | flags,
269 | lock_owner
270 | );
271 | if ino < 1000 {
272 | reply.error(libc::EISDIR);
273 | } else {
274 | reply.data("0123456789\n".to_string().as_bytes());
275 | }
276 | }
277 | }
278 |
--------------------------------------------------------------------------------
/src/lib.rs:
--------------------------------------------------------------------------------
1 | //! `s3d` is an S3 daemon for the Edge written in Rust
2 | //! - https://s3d.rs
3 | //! - https://github.com/s3d-rs/s3d
4 |
5 | // #![doc = include_str!("../README.md")]
6 | // #![allow(unused)]
7 |
8 | pub mod cli;
9 | pub mod codegen_include;
10 | pub mod config;
11 | pub mod s3;
12 | pub mod utils;
13 | pub mod write_queue;
14 |
15 | #[cfg(feature = "fuse")]
16 | pub mod fuse;
17 |
18 | #[macro_use]
19 | extern crate log;
20 |
21 | // #[macro_use]
22 | // extern crate clap;
23 |
24 | // #[macro_use]
25 | // extern crate anyhow;
26 |
--------------------------------------------------------------------------------
/src/s3/api.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::to_internal_err;
2 | use std::future::Future;
3 | use std::pin::Pin;
4 |
5 | /// Why we need this TraitFuture:
6 | /// We can't use async_trait macro inside our macro so we use the same thing it does
7 | /// which is this pin-box-dyn-future - see long explanation here:
8 | /// https://smallcultfollowing.com/babysteps/blog/2019/10/26/async-fn-in-traits-are-hard/
9 | pub type TraitFuture<'a, O, E> = Pin> + Send + 'a>>;
10 |
11 | pub type SMClient = aws_smithy_client::Client<
12 | aws_smithy_client::erase::DynConnector,
13 | aws_sdk_s3::middleware::DefaultMiddleware,
14 | >;
15 |
16 | macro_rules! s3_op_trait {
17 | ($op:ident) => {
18 | paste::paste! {
19 | fn [<$op:snake>](&self, i: s3d_smithy_codegen_server_s3::input::[<$op Input>])
20 | -> TraitFuture<
21 | s3d_smithy_codegen_server_s3::output::[<$op Output>],
22 | s3d_smithy_codegen_server_s3::error::[<$op Error>],
23 | >;
24 | }
25 | };
26 | }
27 |
28 | macro_rules! s3_op_impl {
29 | ($op:ident) => {
30 | paste::paste! {
31 | fn [<$op:snake>](&self, i: s3d_smithy_codegen_server_s3::input::[<$op Input>]) ->
32 | TraitFuture<
33 | s3d_smithy_codegen_server_s3::output::[<$op Output>],
34 | s3d_smithy_codegen_server_s3::error::[<$op Error>],
35 | >
36 | {
37 | Box::pin(async move {
38 | info!("{}: {:?}", stringify!([<$op:snake>]), i);
39 | let to_client = crate::codegen_include::[];
40 | let from_client = crate::codegen_include::[];
41 | let r = self.sm_client
42 | .call(to_client(i).make_operation(self.s3_client.conf()).await.unwrap())
43 | .await
44 | .map(from_client)
45 | .map_err(to_internal_err);
46 | info!("{}: {:?}", stringify!([<$op:snake>]), r);
47 | r
48 | })
49 | }
50 | }
51 | };
52 | }
53 |
54 | pub trait S3Api {
55 | // LIST OPS
56 | s3_op_trait!(ListBuckets);
57 | s3_op_trait!(ListObjects);
58 | s3_op_trait!(ListObjectsV2);
59 | s3_op_trait!(ListObjectVersions);
60 | // SIMPLE OBJECT OPS
61 | s3_op_trait!(HeadObject);
62 | s3_op_trait!(GetObject);
63 | s3_op_trait!(PutObject);
64 | s3_op_trait!(CopyObject);
65 | s3_op_trait!(DeleteObject);
66 | s3_op_trait!(DeleteObjects);
67 | s3_op_trait!(GetObjectTagging);
68 | s3_op_trait!(PutObjectTagging);
69 | s3_op_trait!(DeleteObjectTagging);
70 | // SIMPLE BUCKET OPS
71 | s3_op_trait!(HeadBucket);
72 | s3_op_trait!(CreateBucket);
73 | s3_op_trait!(DeleteBucket);
74 | s3_op_trait!(GetBucketTagging);
75 | s3_op_trait!(PutBucketTagging);
76 | s3_op_trait!(DeleteBucketTagging);
77 | // MULTIPART UPLOAD OPS
78 | s3_op_trait!(CreateMultipartUpload);
79 | s3_op_trait!(CompleteMultipartUpload);
80 | s3_op_trait!(AbortMultipartUpload);
81 | s3_op_trait!(ListMultipartUploads);
82 | s3_op_trait!(ListParts);
83 | s3_op_trait!(UploadPart);
84 | s3_op_trait!(UploadPartCopy);
85 | // ADVANCED OBJECT OPS
86 | s3_op_trait!(GetObjectAcl);
87 | s3_op_trait!(PutObjectAcl);
88 | s3_op_trait!(GetObjectLegalHold);
89 | s3_op_trait!(PutObjectLegalHold);
90 | s3_op_trait!(GetObjectLockConfiguration);
91 | s3_op_trait!(PutObjectLockConfiguration);
92 | s3_op_trait!(GetObjectRetention);
93 | s3_op_trait!(PutObjectRetention);
94 | s3_op_trait!(GetObjectTorrent);
95 | s3_op_trait!(RestoreObject);
96 | // ADVANCED BUCKET OPS
97 | s3_op_trait!(GetBucketAccelerateConfiguration);
98 | s3_op_trait!(GetBucketAcl);
99 | s3_op_trait!(GetBucketAnalyticsConfiguration);
100 | s3_op_trait!(GetBucketCors);
101 | s3_op_trait!(GetBucketEncryption);
102 | s3_op_trait!(GetBucketIntelligentTieringConfiguration);
103 | s3_op_trait!(GetBucketInventoryConfiguration);
104 | s3_op_trait!(GetBucketLifecycleConfiguration);
105 | s3_op_trait!(GetBucketLocation);
106 | s3_op_trait!(GetBucketLogging);
107 | s3_op_trait!(GetBucketMetricsConfiguration);
108 | s3_op_trait!(GetBucketNotificationConfiguration);
109 | s3_op_trait!(GetBucketOwnershipControls);
110 | s3_op_trait!(GetBucketPolicy);
111 | s3_op_trait!(GetBucketPolicyStatus);
112 | s3_op_trait!(GetBucketReplication);
113 | s3_op_trait!(GetBucketRequestPayment);
114 | s3_op_trait!(GetBucketVersioning);
115 | s3_op_trait!(GetBucketWebsite);
116 | s3_op_trait!(GetPublicAccessBlock);
117 | s3_op_trait!(PutBucketAccelerateConfiguration);
118 | s3_op_trait!(PutBucketAcl);
119 | s3_op_trait!(PutBucketAnalyticsConfiguration);
120 | s3_op_trait!(PutBucketCors);
121 | s3_op_trait!(PutBucketEncryption);
122 | s3_op_trait!(PutBucketIntelligentTieringConfiguration);
123 | s3_op_trait!(PutBucketInventoryConfiguration);
124 | s3_op_trait!(PutBucketLifecycleConfiguration);
125 | s3_op_trait!(PutBucketLogging);
126 | s3_op_trait!(PutBucketMetricsConfiguration);
127 | s3_op_trait!(PutBucketNotificationConfiguration);
128 | s3_op_trait!(PutBucketOwnershipControls);
129 | s3_op_trait!(PutBucketPolicy);
130 | s3_op_trait!(PutBucketReplication);
131 | s3_op_trait!(PutBucketRequestPayment);
132 | s3_op_trait!(PutBucketVersioning);
133 | s3_op_trait!(PutBucketWebsite);
134 | s3_op_trait!(PutPublicAccessBlock);
135 | s3_op_trait!(WriteGetObjectResponse);
136 | s3_op_trait!(DeleteBucketAnalyticsConfiguration);
137 | s3_op_trait!(DeleteBucketCors);
138 | s3_op_trait!(DeleteBucketEncryption);
139 | s3_op_trait!(DeleteBucketIntelligentTieringConfiguration);
140 | s3_op_trait!(DeleteBucketInventoryConfiguration);
141 | s3_op_trait!(DeleteBucketLifecycle);
142 | s3_op_trait!(DeleteBucketMetricsConfiguration);
143 | s3_op_trait!(DeleteBucketOwnershipControls);
144 | s3_op_trait!(DeleteBucketPolicy);
145 | s3_op_trait!(DeleteBucketReplication);
146 | s3_op_trait!(DeleteBucketWebsite);
147 | s3_op_trait!(DeletePublicAccessBlock);
148 | s3_op_trait!(ListBucketAnalyticsConfigurations);
149 | s3_op_trait!(ListBucketIntelligentTieringConfigurations);
150 | s3_op_trait!(ListBucketInventoryConfigurations);
151 | s3_op_trait!(ListBucketMetricsConfigurations);
152 | }
153 |
154 | pub struct S3ApiClient {
155 | sm_client: &'static SMClient,
156 | s3_client: &'static aws_sdk_s3::Client,
157 | }
158 |
159 | impl S3Api for S3ApiClient {
160 | // LIST OPS
161 | s3_op_impl!(ListBuckets);
162 | s3_op_impl!(ListObjects);
163 | s3_op_impl!(ListObjectsV2);
164 | s3_op_impl!(ListObjectVersions);
165 | // SIMPLE OBJECT OPS
166 | s3_op_impl!(HeadObject);
167 | s3_op_impl!(GetObject);
168 | s3_op_impl!(PutObject);
169 | s3_op_impl!(CopyObject);
170 | s3_op_impl!(DeleteObject);
171 | s3_op_impl!(DeleteObjects);
172 | s3_op_impl!(GetObjectTagging);
173 | s3_op_impl!(PutObjectTagging);
174 | s3_op_impl!(DeleteObjectTagging);
175 | // SIMPLE BUCKET OPS
176 | s3_op_impl!(HeadBucket);
177 | s3_op_impl!(CreateBucket);
178 | s3_op_impl!(DeleteBucket);
179 | s3_op_impl!(GetBucketTagging);
180 | s3_op_impl!(PutBucketTagging);
181 | s3_op_impl!(DeleteBucketTagging);
182 | // MULTIPART UPLOAD OPS
183 | s3_op_impl!(CreateMultipartUpload);
184 | s3_op_impl!(CompleteMultipartUpload);
185 | s3_op_impl!(AbortMultipartUpload);
186 | s3_op_impl!(ListMultipartUploads);
187 | s3_op_impl!(ListParts);
188 | s3_op_impl!(UploadPart);
189 | s3_op_impl!(UploadPartCopy);
190 | // ADVANCED OBJECT OPS
191 | s3_op_impl!(GetObjectAcl);
192 | s3_op_impl!(PutObjectAcl);
193 | s3_op_impl!(GetObjectLegalHold);
194 | s3_op_impl!(PutObjectLegalHold);
195 | s3_op_impl!(GetObjectLockConfiguration);
196 | s3_op_impl!(PutObjectLockConfiguration);
197 | s3_op_impl!(GetObjectRetention);
198 | s3_op_impl!(PutObjectRetention);
199 | s3_op_impl!(GetObjectTorrent);
200 | s3_op_impl!(RestoreObject);
201 | // ADVANCED BUCKET OPS
202 | s3_op_impl!(GetBucketAccelerateConfiguration);
203 | s3_op_impl!(GetBucketAcl);
204 | s3_op_impl!(GetBucketAnalyticsConfiguration);
205 | s3_op_impl!(GetBucketCors);
206 | s3_op_impl!(GetBucketEncryption);
207 | s3_op_impl!(GetBucketIntelligentTieringConfiguration);
208 | s3_op_impl!(GetBucketInventoryConfiguration);
209 | s3_op_impl!(GetBucketLifecycleConfiguration);
210 | s3_op_impl!(GetBucketLocation);
211 | s3_op_impl!(GetBucketLogging);
212 | s3_op_impl!(GetBucketMetricsConfiguration);
213 | s3_op_impl!(GetBucketNotificationConfiguration);
214 | s3_op_impl!(GetBucketOwnershipControls);
215 | s3_op_impl!(GetBucketPolicy);
216 | s3_op_impl!(GetBucketPolicyStatus);
217 | s3_op_impl!(GetBucketReplication);
218 | s3_op_impl!(GetBucketRequestPayment);
219 | s3_op_impl!(GetBucketVersioning);
220 | s3_op_impl!(GetBucketWebsite);
221 | s3_op_impl!(GetPublicAccessBlock);
222 | s3_op_impl!(PutBucketAccelerateConfiguration);
223 | s3_op_impl!(PutBucketAcl);
224 | s3_op_impl!(PutBucketAnalyticsConfiguration);
225 | s3_op_impl!(PutBucketCors);
226 | s3_op_impl!(PutBucketEncryption);
227 | s3_op_impl!(PutBucketIntelligentTieringConfiguration);
228 | s3_op_impl!(PutBucketInventoryConfiguration);
229 | s3_op_impl!(PutBucketLifecycleConfiguration);
230 | s3_op_impl!(PutBucketLogging);
231 | s3_op_impl!(PutBucketMetricsConfiguration);
232 | s3_op_impl!(PutBucketNotificationConfiguration);
233 | s3_op_impl!(PutBucketOwnershipControls);
234 | s3_op_impl!(PutBucketPolicy);
235 | s3_op_impl!(PutBucketReplication);
236 | s3_op_impl!(PutBucketRequestPayment);
237 | s3_op_impl!(PutBucketVersioning);
238 | s3_op_impl!(PutBucketWebsite);
239 | s3_op_impl!(PutPublicAccessBlock);
240 | s3_op_impl!(WriteGetObjectResponse);
241 | s3_op_impl!(DeleteBucketAnalyticsConfiguration);
242 | s3_op_impl!(DeleteBucketCors);
243 | s3_op_impl!(DeleteBucketEncryption);
244 | s3_op_impl!(DeleteBucketIntelligentTieringConfiguration);
245 | s3_op_impl!(DeleteBucketInventoryConfiguration);
246 | s3_op_impl!(DeleteBucketLifecycle);
247 | s3_op_impl!(DeleteBucketMetricsConfiguration);
248 | s3_op_impl!(DeleteBucketOwnershipControls);
249 | s3_op_impl!(DeleteBucketPolicy);
250 | s3_op_impl!(DeleteBucketReplication);
251 | s3_op_impl!(DeleteBucketWebsite);
252 | s3_op_impl!(DeletePublicAccessBlock);
253 | s3_op_impl!(ListBucketAnalyticsConfigurations);
254 | s3_op_impl!(ListBucketIntelligentTieringConfigurations);
255 | s3_op_impl!(ListBucketInventoryConfigurations);
256 | s3_op_impl!(ListBucketMetricsConfigurations);
257 | }
258 |
--------------------------------------------------------------------------------
/src/s3/mod.rs:
--------------------------------------------------------------------------------
1 | pub mod api;
2 | pub mod server;
3 |
--------------------------------------------------------------------------------
/src/s3/server.rs:
--------------------------------------------------------------------------------
1 | use crate::config;
2 | use crate::utils::{staticify, to_internal_err};
3 | use crate::write_queue::WriteQueue;
4 | use s3d_smithy_codegen_server_s3::{input::*, operation_registry::*};
5 |
6 | pub type Router = aws_smithy_http_server::Router;
7 |
8 | pub type SMClient = aws_smithy_client::Client<
9 | aws_smithy_client::erase::DynConnector,
10 | aws_sdk_s3::middleware::DefaultMiddleware,
11 | >;
12 |
13 | pub async fn serve() -> anyhow::Result<()> {
14 | let s3_config = aws_config::load_from_env().await;
15 | let s3_client = staticify(aws_sdk_s3::Client::new(&s3_config));
16 | let sleep_impl = aws_smithy_async::rt::sleep::default_async_sleep();
17 | let sm_builder = aws_sdk_s3::client::Builder::dyn_https()
18 | .sleep_impl(sleep_impl)
19 | .middleware(aws_sdk_s3::middleware::DefaultMiddleware::new());
20 | let sm_client = staticify(sm_builder.build());
21 | let write_queue = staticify(WriteQueue {
22 | s3_client,
23 | write_queue_dir: config::S3D_WRITE_QUEUE_DIR.to_string(),
24 | });
25 | write_queue.start();
26 | let addr = std::net::SocketAddr::from(([127, 0, 0, 1], 33333));
27 | let router = build_router(sm_client, s3_client, write_queue);
28 | let server = hyper::Server::bind(&addr).serve(router.into_make_service());
29 | info!("###################################");
30 | info!("Listening on http://{}", addr);
31 | info!("###################################");
32 | server.await?;
33 | Ok(())
34 | }
35 |
36 | pub fn build_router(
37 | sm_client: &'static SMClient,
38 | s3_client: &'static aws_sdk_s3::Client,
39 | write_queue: &'static WriteQueue,
40 | ) -> Router {
41 | let mut b = OperationRegistryBuilder::default();
42 |
43 | macro_rules! register_s3_gateway_op {
44 | ($op:ident) => {
45 | paste::paste! {
46 | b = b.[<$op:snake>](move |i: [<$op Input>]| async {
47 | info!("{}: {:?}", stringify!([<$op:snake>]), i);
48 | let to_client = crate::codegen_include::[];
49 | let from_client = crate::codegen_include::[];
50 | let r = sm_client
51 | .call(to_client(i).make_operation(s3_client.conf()).await.unwrap())
52 | .await
53 | .map(from_client)
54 | .map_err(to_internal_err);
55 | info!("{}: {:?}", stringify!([<$op:snake>]), r);
56 | r
57 | });
58 | }
59 | };
60 | }
61 |
62 | b = b.put_object(move |i: PutObjectInput| async {
63 | info!("put_object: {:?}", i);
64 | write_queue.put_object(i).await
65 | });
66 |
67 | b = b.get_object(move |i: GetObjectInput| async move {
68 | info!("get_object: {:?}", i);
69 | let i1 = i.to_owned();
70 | let i2 = i.to_owned();
71 | let qres = write_queue.get_object(i1).await;
72 | if qres.is_ok() {
73 | return qres;
74 | }
75 | info!("get_object: read from remote");
76 | let to_client = crate::codegen_include::conv_to_client_get_object_input;
77 | let from_client = crate::codegen_include::conv_from_client_get_object_output;
78 | let r = sm_client
79 | .call(
80 | to_client(i2)
81 | .make_operation(s3_client.conf())
82 | .await
83 | .unwrap(),
84 | )
85 | .await
86 | .map(from_client)
87 | .map_err(to_internal_err);
88 | info!("get_object: read from remote {:?}", r);
89 | r
90 | });
91 |
92 | // LIST OPS
93 | register_s3_gateway_op!(ListBuckets);
94 | register_s3_gateway_op!(ListObjects);
95 | register_s3_gateway_op!(ListObjectsV2);
96 | register_s3_gateway_op!(ListObjectVersions);
97 | // SIMPLE OBJECT OPS
98 | register_s3_gateway_op!(HeadObject);
99 | register_s3_gateway_op!(CopyObject);
100 | register_s3_gateway_op!(DeleteObject);
101 | register_s3_gateway_op!(DeleteObjects);
102 | register_s3_gateway_op!(GetObjectTagging);
103 | register_s3_gateway_op!(PutObjectTagging);
104 | register_s3_gateway_op!(DeleteObjectTagging);
105 | // SIMPLE BUCKET OPS
106 | register_s3_gateway_op!(HeadBucket);
107 | register_s3_gateway_op!(CreateBucket);
108 | register_s3_gateway_op!(DeleteBucket);
109 | register_s3_gateway_op!(GetBucketTagging);
110 | register_s3_gateway_op!(PutBucketTagging);
111 | register_s3_gateway_op!(DeleteBucketTagging);
112 | // MULTIPART UPLOAD OPS
113 | register_s3_gateway_op!(CreateMultipartUpload);
114 | register_s3_gateway_op!(CompleteMultipartUpload);
115 | register_s3_gateway_op!(AbortMultipartUpload);
116 | register_s3_gateway_op!(ListMultipartUploads);
117 | register_s3_gateway_op!(ListParts);
118 | register_s3_gateway_op!(UploadPart);
119 | register_s3_gateway_op!(UploadPartCopy);
120 | // ADVANCED OBJECT OPS
121 | register_s3_gateway_op!(GetObjectAcl);
122 | register_s3_gateway_op!(PutObjectAcl);
123 | register_s3_gateway_op!(GetObjectLegalHold);
124 | register_s3_gateway_op!(PutObjectLegalHold);
125 | register_s3_gateway_op!(GetObjectLockConfiguration);
126 | register_s3_gateway_op!(PutObjectLockConfiguration);
127 | register_s3_gateway_op!(GetObjectRetention);
128 | register_s3_gateway_op!(PutObjectRetention);
129 | register_s3_gateway_op!(GetObjectTorrent);
130 | register_s3_gateway_op!(RestoreObject);
131 | // ADVANCED BUCKET OPS
132 | register_s3_gateway_op!(GetBucketAccelerateConfiguration);
133 | register_s3_gateway_op!(GetBucketAcl);
134 | register_s3_gateway_op!(GetBucketAnalyticsConfiguration);
135 | register_s3_gateway_op!(GetBucketCors);
136 | register_s3_gateway_op!(GetBucketEncryption);
137 | register_s3_gateway_op!(GetBucketIntelligentTieringConfiguration);
138 | register_s3_gateway_op!(GetBucketInventoryConfiguration);
139 | register_s3_gateway_op!(GetBucketLifecycleConfiguration);
140 | register_s3_gateway_op!(GetBucketLocation);
141 | register_s3_gateway_op!(GetBucketLogging);
142 | register_s3_gateway_op!(GetBucketMetricsConfiguration);
143 | register_s3_gateway_op!(GetBucketNotificationConfiguration);
144 | register_s3_gateway_op!(GetBucketOwnershipControls);
145 | register_s3_gateway_op!(GetBucketPolicy);
146 | register_s3_gateway_op!(GetBucketPolicyStatus);
147 | register_s3_gateway_op!(GetBucketReplication);
148 | register_s3_gateway_op!(GetBucketRequestPayment);
149 | register_s3_gateway_op!(GetBucketVersioning);
150 | register_s3_gateway_op!(GetBucketWebsite);
151 | register_s3_gateway_op!(GetPublicAccessBlock);
152 | register_s3_gateway_op!(PutBucketAccelerateConfiguration);
153 | register_s3_gateway_op!(PutBucketAcl);
154 | register_s3_gateway_op!(PutBucketAnalyticsConfiguration);
155 | register_s3_gateway_op!(PutBucketCors);
156 | register_s3_gateway_op!(PutBucketEncryption);
157 | register_s3_gateway_op!(PutBucketIntelligentTieringConfiguration);
158 | register_s3_gateway_op!(PutBucketInventoryConfiguration);
159 | register_s3_gateway_op!(PutBucketLifecycleConfiguration);
160 | register_s3_gateway_op!(PutBucketLogging);
161 | register_s3_gateway_op!(PutBucketMetricsConfiguration);
162 | register_s3_gateway_op!(PutBucketNotificationConfiguration);
163 | register_s3_gateway_op!(PutBucketOwnershipControls);
164 | register_s3_gateway_op!(PutBucketPolicy);
165 | register_s3_gateway_op!(PutBucketReplication);
166 | register_s3_gateway_op!(PutBucketRequestPayment);
167 | register_s3_gateway_op!(PutBucketVersioning);
168 | register_s3_gateway_op!(PutBucketWebsite);
169 | register_s3_gateway_op!(PutPublicAccessBlock);
170 | register_s3_gateway_op!(WriteGetObjectResponse);
171 | register_s3_gateway_op!(DeleteBucketAnalyticsConfiguration);
172 | register_s3_gateway_op!(DeleteBucketCors);
173 | register_s3_gateway_op!(DeleteBucketEncryption);
174 | register_s3_gateway_op!(DeleteBucketIntelligentTieringConfiguration);
175 | register_s3_gateway_op!(DeleteBucketInventoryConfiguration);
176 | register_s3_gateway_op!(DeleteBucketLifecycle);
177 | register_s3_gateway_op!(DeleteBucketMetricsConfiguration);
178 | register_s3_gateway_op!(DeleteBucketOwnershipControls);
179 | register_s3_gateway_op!(DeleteBucketPolicy);
180 | register_s3_gateway_op!(DeleteBucketReplication);
181 | register_s3_gateway_op!(DeleteBucketWebsite);
182 | register_s3_gateway_op!(DeletePublicAccessBlock);
183 | register_s3_gateway_op!(ListBucketAnalyticsConfigurations);
184 | register_s3_gateway_op!(ListBucketIntelligentTieringConfigurations);
185 | register_s3_gateway_op!(ListBucketInventoryConfigurations);
186 | register_s3_gateway_op!(ListBucketMetricsConfigurations);
187 |
188 | let ops = b.build().unwrap();
189 |
190 | {
191 | #[rustfmt::skip]
192 | let _: &OperationRegistry<
193 | hyper::Body,
194 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
195 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
196 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
197 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
198 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
199 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
200 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
201 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
202 | _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (), _, (),
203 | _, ()> = &ops;
204 | }
205 |
206 | let router = Router::from(ops);
207 | router
208 | }
209 |
--------------------------------------------------------------------------------
/src/utils.rs:
--------------------------------------------------------------------------------
1 | use crate::config;
2 | use aws_smithy_http::byte_stream::ByteStream;
3 | use s3d_smithy_codegen_server_s3::error::InternalServerError;
4 | use serde::Deserialize;
5 | use std::os::unix::io::FromRawFd;
6 | use std::path::Path;
7 | use std::str::FromStr;
8 | use tokio::fs::{read_to_string, File};
9 | use tokio::io::AsyncWriteExt;
10 | use tokio_stream::StreamExt;
11 |
12 | /// staticify uses Box::leak to make a struct with static lifetime.
13 | /// This is useful for async flows that require structs to live throughout the flow,
14 | /// where not releasing their memory is fine.
15 | pub fn staticify(x: T) -> &'static T {
16 | Box::leak(Box::new(x))
17 | }
18 |
19 | /// new_s3_client creates a new s3 client which defaults to connect to the local daemon.
20 | pub async fn new_s3_client() -> aws_sdk_s3::Client {
21 | if config::S3_ENDPOINT.is_none() {
22 | let s3_config = aws_config::load_from_env().await;
23 | return aws_sdk_s3::Client::new(&s3_config);
24 | }
25 |
26 | aws_sdk_s3::Client::from_conf({
27 | let ep = aws_sdk_s3::Endpoint::immutable(
28 | hyper::Uri::from_str(config::S3_ENDPOINT.as_ref().unwrap()).unwrap(),
29 | );
30 | let creds = aws_sdk_s3::Credentials::new("s3d", "s3d", None, None, "s3d");
31 | let region = aws_sdk_s3::Region::new("s3d");
32 | let sleep_impl = aws_smithy_async::rt::sleep::default_async_sleep().unwrap();
33 | aws_sdk_s3::Config::builder()
34 | .sleep_impl(sleep_impl)
35 | .endpoint_resolver(ep)
36 | .credentials_provider(creds)
37 | .region(region)
38 | .build()
39 | })
40 | }
41 |
42 | pub fn parse_bucket_and_key(s: &str) -> anyhow::Result<(String, String)> {
43 | let mut parts = s.splitn(2, '/');
44 | let bucket = parts
45 | .next()
46 | .ok_or_else(|| anyhow::anyhow!("Missing bucket"))?;
47 | let key = parts.next().ok_or_else(|| anyhow::anyhow!("Missing key"))?;
48 | Ok((String::from(bucket), String::from(key)))
49 | }
50 |
51 | pub fn parse_bucket_and_prefix(s: &str) -> anyhow::Result<(String, String)> {
52 | let mut parts = s.splitn(2, '/');
53 | let bucket = parts.next().unwrap_or("");
54 | let key = parts.next().unwrap_or("");
55 | Ok((String::from(bucket), String::from(key)))
56 | }
57 |
58 | pub async fn read_file_as_stream(fname: &str) -> anyhow::Result {
59 | Ok(ByteStream::from_path(Path::new(&fname)).await?)
60 | }
61 |
62 | pub async fn write_stream_to_file(fname: &str, stream: &mut ByteStream) -> anyhow::Result {
63 | let mut file = File::create(fname).await?;
64 | let num_bytes = pipe_stream(stream, &mut file).await?;
65 | file.flush().await?;
66 | file.sync_all().await?;
67 | file.shutdown().await?;
68 | Ok(num_bytes)
69 | }
70 |
71 | pub async fn pipe_stream(input: &mut I, output: &mut O) -> anyhow::Result
72 | where
73 | I: tokio_stream::Stream- > + std::marker::Unpin,
74 | O: tokio::io::AsyncWrite + std::marker::Unpin,
75 | E: std::error::Error + Send + Sync + 'static,
76 | {
77 | let mut num_bytes: u64 = 0;
78 | while let Some(ref mut buf) = input.try_next().await? {
79 | num_bytes += buf.len() as u64;
80 | output.write_all_buf(buf).await?;
81 | }
82 | Ok(num_bytes)
83 | }
84 |
85 | pub async fn pipe_stream_to_outfile_or_stdout(
86 | input: &mut I,
87 | outfile: Option<&str>,
88 | ) -> anyhow::Result
89 | where
90 | I: tokio_stream::Stream
- > + std::marker::Unpin,
91 | E: std::error::Error + Send + Sync + 'static,
92 | {
93 | match outfile {
94 | Some(ref path) => {
95 | let mut file = tokio::fs::File::create(path).await?;
96 | pipe_stream(input, &mut file).await
97 | }
98 | None => {
99 | let mut out = tokio::io::stdout();
100 | pipe_stream(input, &mut out).await
101 | }
102 | }
103 | }
104 |
105 | pub async fn byte_stream_from_infile_or_stdin(infile: Option<&str>) -> anyhow::Result {
106 | let file = match infile {
107 | Some(ref path) => tokio::fs::File::open(path).await?,
108 | None => tokio::fs::File::from_std(unsafe { std::fs::File::from_raw_fd(0) }),
109 | };
110 | let stream = ByteStream::read_from().file(file).build().await?;
111 | Ok(stream)
112 | }
113 |
114 | pub async fn _read_yaml_file(path: &Path) -> anyhow::Result
115 | where
116 | T: for<'de> Deserialize<'de>,
117 | {
118 | Ok(serde_yaml::from_str(&read_to_string(path).await?)?)
119 | }
120 |
121 | pub fn to_internal_err>(err: F) -> T {
122 | InternalServerError {
123 | message: err.to_string(),
124 | }
125 | .into()
126 | }
127 |
--------------------------------------------------------------------------------
/src/write_queue.rs:
--------------------------------------------------------------------------------
1 | use crate::utils::{read_file_as_stream, to_internal_err, write_stream_to_file};
2 | use aws_smithy_http::byte_stream::ByteStream;
3 | use s3d_smithy_codegen_server_s3::{
4 | error::{GetObjectError, HeadObjectError, PutObjectError},
5 | input::{GetObjectInput, HeadObjectInput, PutObjectInput},
6 | output::{GetObjectOutput, HeadObjectOutput, PutObjectOutput},
7 | };
8 | use std::path::Path;
9 |
10 | pub struct WriteQueue {
11 | pub s3_client: &'static aws_sdk_s3::Client,
12 | pub write_queue_dir: String,
13 | }
14 |
15 | impl WriteQueue {
16 | pub fn start(&'static self) {
17 | tokio::spawn(self.worker());
18 | }
19 |
20 | pub async fn worker(&self) {
21 | loop {
22 | tokio::time::sleep(tokio::time::Duration::from_millis(5000)).await;
23 | if let Err(err) = self.work().await {
24 | debug!("{}", err);
25 | }
26 | }
27 | }
28 |
29 | pub async fn work(&self) -> anyhow::Result<()> {
30 | debug!("Write queue worker running ...");
31 | let mut queue = tokio::fs::read_dir(&self.write_queue_dir).await?;
32 | while let Some(entry) = queue.next_entry().await? {
33 | let entry_name_os = entry.file_name();
34 | let entry_name = entry_name_os.to_str().unwrap();
35 | if let Err(err) = self.push_file(entry_name).await {
36 | warn!("{}", err);
37 | }
38 | }
39 | Ok(())
40 | }
41 |
42 | pub async fn push_file(&self, entry_name: &str) -> anyhow::Result<()> {
43 | let bucket_path_cow = urlencoding::decode(entry_name).unwrap();
44 | let bucket_path = bucket_path_cow.as_ref();
45 | info!("Write queue item: {:?}", bucket_path);
46 | let mut parts = bucket_path.splitn(2, '/');
47 | let bucket = parts.next().unwrap();
48 | let key = parts.next().unwrap();
49 | let fname = format!("{}/{}", self.write_queue_dir, entry_name);
50 | let body = ByteStream::from_path(Path::new(&fname)).await?;
51 | self.s3_client
52 | .put_object()
53 | .bucket(bucket)
54 | .key(key)
55 | .body(body)
56 | .send()
57 | .await?;
58 | tokio::fs::remove_file(fname).await?;
59 | info!("Write queue item: {:?}", bucket_path);
60 | Ok(())
61 | }
62 |
63 | pub async fn put_object(
64 | &self,
65 | mut i: PutObjectInput,
66 | ) -> Result {
67 | let fname = self.to_file_name(i.bucket(), i.key());
68 | write_stream_to_file(&fname, &mut i.body)
69 | .await
70 | .map(|_| PutObjectOutput::builder().e_tag("s3d-etag").build())
71 | .map_err(to_internal_err)
72 | }
73 |
74 | pub async fn get_object(&self, i: GetObjectInput) -> Result {
75 | let fname = self.to_file_name(i.bucket(), i.key());
76 | read_file_as_stream(&fname)
77 | .await
78 | .map(|stream| GetObjectOutput::builder().set_body(Some(stream)).build())
79 | .map_err(to_internal_err)
80 | }
81 |
82 | pub async fn head_object(
83 | &self,
84 | _i: HeadObjectInput,
85 | ) -> Result {
86 | // let fname = self.to_file_name(_i.bucket(), _i.key()) + ".s3d-object-md.yaml";
87 | Ok(HeadObjectOutput::builder().build())
88 | }
89 |
90 | pub fn to_file_name(&self, bucket: &str, key: &str) -> String {
91 | format!(
92 | "{}/{}",
93 | self.write_queue_dir,
94 | urlencoding::encode(&format!("{}/{}", bucket, key))
95 | )
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/test/sanity.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # s3d should be running on localhost:33333 and then run this script.
4 |
5 | EP="http://localhost:33333"
6 | BKT="${1:-s3d-test-bucket}"
7 |
8 | function LOG() {
9 | { echo -e "\n----------> sanity: $@\n"; } 2>/dev/null
10 | }
11 |
12 | function S3() {
13 | LOG "🚀 s3 $@"
14 | eval "./target/debug/s3 $@"
15 | }
16 |
17 | function CURL() {
18 | local rc
19 | LOG "🚀 curl $@"
20 | curl -s -i "${EP}$@"
21 | rc="$?"
22 | if [ $rc -ne 0 ]; then
23 | LOG "Error\n\nError: CURL failed\nReturnCode: $rc\nCommand: curl -s -i ${EP}$@"
24 | exit 1
25 | fi
26 | }
27 |
28 | function AWSCLI() {
29 | LOG "🚀 aws $@"
30 | aws --endpoint $EP "$@"
31 | }
32 |
33 | function test_s3() {
34 | LOG "▶️ test_s3 ..."
35 | S3 ls
36 | S3 ls $BKT
37 | S3 put $BKT/README.md "/dev/null"
40 | S3 ls $BKT
41 | LOG "✅ test_s3 done"
42 | }
43 |
44 | function test_curl_client() {
45 | LOG "▶️ test_curl_client ..."
46 | CURL / # ListBuckets
47 | CURL /$BKT -X PUT # CreateBucket
48 | CURL / # ListBuckets
49 | CURL /$BKT -I # HeadBucket
50 | CURL /$BKT -X GET # ListObjects
51 | CURL /$BKT/README.md -X PUT -d @README.md # PutObject
52 | CURL /$BKT/README.md -I # HeadObject
53 | CURL /$BKT/README.md -X GET # GetObject
54 | CURL /$BKT/README.md -X DELETE # DeleteObject
55 | CURL /$BKT -X DELETE # DeleteBucket
56 | CURL / # ListBuckets
57 | LOG "✅ test_curl_client done"
58 | }
59 |
60 | function test_awscli_s3() {
61 | LOG "▶️ test_awscli_s3 ..."
62 | AWSCLI s3 ls
63 | AWSCLI s3 ls s3://$BKT
64 | AWSCLI s3 cp README.md s3://$BKT/README.md
65 | AWSCLI s3 cp s3://$BKT/README.md -
66 | AWSCLI s3 rm s3://$BKT/README.md
67 | AWSCLI s3 rb s3://$BKT
68 | AWSCLI s3 ls
69 | LOG "✅ test_awscli_s3 done"
70 | }
71 |
72 | function test_awscli_s3api() {
73 | LOG "▶️ test_awscli_s3api ..."
74 | AWSCLI s3api list-buckets
75 | AWSCLI s3api list-objects --bucket $BKT
76 | AWSCLI s3api put-object --bucket $BKT --key README.md --body README.md
77 | AWSCLI s3api get-object --bucket $BKT --key README.md /dev/null
78 | AWSCLI s3api delete-object --bucket $BKT --key README.md
79 | AWSCLI s3api list-objects --bucket $BKT
80 | LOG "✅ test_awscli_s3api done"
81 | }
82 |
83 |
84 | test_s3
85 | #test_curl_client
86 | #test_awscli_s3
87 | #test_awscli_s3api
88 |
--------------------------------------------------------------------------------
6 |
7 |
8 |
9 | 
10 |
11 |
16 |
17 |
18 | 
19 |
20 |
25 |
26 |
27 | 
28 |
29 |
30 | 
31 |
32 | 
37 |
38 |