├── .dockerignore ├── Dockerfile ├── README.md ├── benchmark.py ├── build_image.sh ├── fasts3 ├── .gitignore ├── Cargo.toml ├── rustfmt.toml └── src │ ├── fasts3_filesystem.rs │ └── lib.rs ├── regression_test.py ├── run_benchmark.sh └── run_regression.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | */target/debug 2 | */target/release 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8-buster AS builder 2 | 3 | RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y 4 | ENV PATH="/root/.cargo/bin:${PATH}" 5 | RUN rustup update 6 | RUN python3 -m pip install maturin 7 | 8 | COPY fasts3/ fasts3/ 9 | RUN cd fasts3 && cargo fmt && maturin build --release 10 | RUN ls fasts3/target/wheels/ 11 | 12 | #=========== 13 | FROM python:3.8-buster 14 | 15 | RUN python3 -m pip install boto3 fsspec s3fs 16 | 17 | COPY --from=builder fasts3/target/wheels/ / 18 | 19 | RUN python3 -m pip install /fasts3-*-cp38-*2_28*.whl 20 | 21 | ADD *.py / 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python_fasts3 2 | Fast S3 in Python using Rust 3 | 4 | A Rust library that can be called from Python to perform S3 operations. The goal is to be significantly faster than Python-only S3 code like boto3. Currently only supports very basic ls() and get_objects() functionality and is meant as a POC, not for production usage. This [blog post](https://joshua-robinson.medium.com/improving-python-s3-client-performance-with-rust-e9639359072f) provides more information about the motivation and initial performance results of FastS3. 5 | 6 | Using fasts3 from python should be simple and fast: 7 | ``` 8 | s = fasts3.FastS3FileSystem(endpoint=ENDPOINT_URL) 9 | 10 | contents = s.get_objects([OBJECTPATH1, OBJECTPATH2]) # Retrieve two objects in parallel 11 | ``` 12 | 13 | Compile the Rust library into wheel format using maturin: 14 | ``` 15 | cd fasts3/ && maturin build --release 16 | ``` 17 | 18 | Installation then follows as with any wheel: 19 | ``` 20 | python3 -m pip install fasts3/target/wheels/*.whl 21 | ``` 22 | 23 | Example output from benchmark program: 24 | ``` 25 | Benchmarking get_object operation 26 | ... 27 | Rust is 2.2x faster than Python 28 | Benchmarking list operation 29 | ... 30 | Rust is 1.8x faster than Boto3 and 1.8x faster than fsspec 31 | ``` 32 | -------------------------------------------------------------------------------- /benchmark.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import fasts3 3 | import fsspec 4 | import io 5 | import multiprocessing as mp 6 | import time 7 | 8 | 9 | # Constants used in testing. 10 | ENDPOINT_URL='http://10.62.64.207' 11 | BUCKET="joshuarobinson" 12 | 13 | # Small object download and group of object download 14 | SMALL_OBJECT="2021-06-04-17.03.28.jpg" 15 | SMALL_PATH=BUCKET + '/' + SMALL_OBJECT 16 | GROUPPATH="balloons/" 17 | 18 | # for list_objects_v2 and ls 19 | LISTPREFIX="opensky/staging1/movements/" 20 | LISTPATH=BUCKET + '/' + LISTPREFIX 21 | 22 | # Large objects of various sizes 23 | TEST_OBJECTS = ["foo64m.txt", "foo128m.txt", "foo256m.txt", "foo500m.txt", "foo1g.txt", "foo2g.txt", "foo3g.txt", "foo4g.txt", "foo.txt", "foo10G.txt"] 24 | 25 | 26 | # Initialize boto3, fsspec, and fasts3 27 | storage_options = {'endpoint_url': ENDPOINT_URL} 28 | fs = fsspec.filesystem('s3', client_kwargs=storage_options) 29 | 30 | s = fasts3.FastS3FileSystem(endpoint=ENDPOINT_URL) 31 | 32 | s3r = boto3.resource('s3', endpoint_url=ENDPOINT_URL) 33 | bucket = s3r.Bucket(BUCKET) 34 | 35 | 36 | print("Benchmarking small get_object operation") 37 | start = time.time() 38 | contents = s.get_objects([SMALL_PATH])[0] 39 | elapsed_rust = time.time() - start 40 | print("fasts3 small get_object, len={}, {}".format(len(contents), elapsed_rust)) 41 | 42 | start = time.time() 43 | bytes_buffer = io.BytesIO() 44 | s3r.meta.client.download_fileobj(Bucket=BUCKET, Key=SMALL_OBJECT, Fileobj=bytes_buffer) 45 | elapsed_b = time.time() - start 46 | print("boto3 small download, len={}, {}".format(bytes_buffer.getbuffer().nbytes, elapsed_b)) 47 | 48 | start = time.time() 49 | data = bucket.Object(SMALL_OBJECT).get().get('Body').read() 50 | elapsed_bg = time.time() - start 51 | print("boto3 small get, len={}, {}".format(len(data), elapsed_bg)) 52 | 53 | if bytes_buffer.getbuffer() != contents: 54 | print("Error, mismatched contents") 55 | exit() 56 | 57 | print("Rust is {:.1f}x faster than Python download_fileobj and {:.1f}x faster than Python get".format(elapsed_b / elapsed_rust, elapsed_bg / elapsed_rust)) 58 | 59 | 60 | print("Benchmarking group get_object operation") 61 | 62 | IMGPATH=BUCKET + "/" + GROUPPATH 63 | image_keys = s.ls(IMGPATH) 64 | image_paths = [BUCKET + '/' + p for p in image_keys] 65 | 66 | s.get_objects(image_paths) # Cache warming 67 | 68 | start = time.time() 69 | contents = s.get_objects(image_paths) 70 | elapsed_rust = time.time() - start 71 | print("fasts3 download group, len={}, {}".format(len(contents), elapsed_rust)) 72 | 73 | # Define boto3 download function so it can be used by Threadpool map() 74 | def do_boto3_download(key: str): 75 | bytes_buffer = io.BytesIO() 76 | s3r.meta.client.download_fileobj(Bucket=BUCKET, Key=key, Fileobj=bytes_buffer) 77 | return bytes_buffer.getbuffer() 78 | 79 | pool = mp.pool.ThreadPool() 80 | start = time.time() 81 | pycontents = pool.map(do_boto3_download, image_keys) 82 | elapsed_b = time.time() - start 83 | print("boto3 download group, len={}, {}".format(len(pycontents), elapsed_b)) 84 | 85 | if len(pycontents) != len(contents): 86 | print("ERROR, wrong number of files returned") 87 | exit() 88 | 89 | for a,b in zip(pycontents, contents): 90 | if a != b: 91 | print("ERROR, mismatched contents!") 92 | exit() 93 | 94 | print("Rust is {:.1f}x faster than Python".format(elapsed_b / elapsed_rust)) 95 | 96 | 97 | print("Benchmarking large get_object operation") 98 | 99 | for _ in range(3): 100 | for object_key in TEST_OBJECTS: 101 | object_path = BUCKET + '/' + object_key 102 | print(object_path) 103 | 104 | start = time.time() 105 | contents = s.get_objects([object_path])[0] 106 | elapsed_rust = time.time() - start 107 | print("fasts3 large get_object, len={}, {}".format(len(contents), elapsed_rust)) 108 | 109 | start = time.time() 110 | with fs.open('/' + object_path) as fp: 111 | foo = fp.read() 112 | elapsed_fs = time.time() - start 113 | print("Download-fsspec ", elapsed_fs) 114 | 115 | start = time.time() 116 | bytes_buffer = io.BytesIO() 117 | s3r.meta.client.download_fileobj(Bucket=BUCKET, Key=object_key, Fileobj=bytes_buffer) 118 | elapsed_b = time.time() - start 119 | print("boto3 large download, len={}, {}".format(bytes_buffer.getbuffer().nbytes, elapsed_b)) 120 | 121 | start = time.time() 122 | data = bucket.Object(object_key).get().get('Body').read() 123 | elapsed_bg = time.time() - start 124 | print("boto3 large get, len={}, {}".format(len(data), elapsed_bg)) 125 | 126 | if bytes_buffer.getbuffer() != contents: 127 | print("ERROR, mismatched contents") 128 | exit() 129 | 130 | object_size_mb = len(contents) / (1024 * 1024) 131 | print("Rust is {:.1f}x faster than Python download_fileobj and {:.1f}x faster than Python get".format(elapsed_b / elapsed_rust, elapsed_bg / elapsed_rust)) 132 | # Print in a manner that can be easily converted to CSV for plotting 133 | print("RESULT-get,{:.1f},{},{},{},{}".format(object_size_mb, elapsed_fs, elapsed_bg, elapsed_b, elapsed_rust)) 134 | 135 | 136 | 137 | print("Benchmarking list operation") 138 | start = time.time() 139 | listingc = fs.ls(LISTPATH) 140 | elapsed_fs = time.time() - start 141 | print("fsspec-s3 ls, len={}, {}".format(len(listingc), elapsed_fs)) 142 | 143 | start = time.time() 144 | listing = s.ls(LISTPATH) 145 | elapsed_rust = time.time() - start 146 | print("fasts3 ls, len={}, {}".format(len(listing), elapsed_rust)) 147 | 148 | start = time.time() 149 | paginator = s3r.meta.client.get_paginator('list_objects_v2') 150 | pages = paginator.paginate(Bucket=BUCKET, Prefix=LISTPREFIX) 151 | # The line below assumes at least one key returned. 152 | listingb = [obj['Key'] for page in pages for obj in page['Contents']] 153 | elapsed_py = time.time() - start 154 | print("boto3 ls, len={}, {}".format(len(listingb), elapsed_py)) 155 | 156 | print("Rust ls() is {:.1f}x faster than Boto3 and {:.1f}x faster than fsspec".format(elapsed_py / elapsed_rust, elapsed_fs / elapsed_rust)) 157 | print("RESULT-ls,{},{},{},{}".format(len(listing), elapsed_fs, elapsed_py, elapsed_rust)) 158 | -------------------------------------------------------------------------------- /build_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | TAG=pyo3-test 6 | REPONAME=joshuarobinson 7 | 8 | docker build -t $TAG --file Dockerfile . 9 | 10 | # Tag and push to a public docker repository. 11 | docker tag $TAG $REPONAME/$TAG 12 | #docker push $REPONAME/$TAG 13 | -------------------------------------------------------------------------------- /fasts3/.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | -------------------------------------------------------------------------------- /fasts3/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "fasts3" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | name = "fasts3" 8 | crate-type = ["cdylib"] 9 | 10 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 11 | 12 | [dependencies] 13 | pyo3 = { version = "0.15.1", features = ["extension-module"] } 14 | #pyo3-asyncio = { version = "0.15.0", features = ["attributes", "tokio-runtime"] } 15 | aws-config = "0.51.0" 16 | aws-sdk-s3 = "0.21.0" 17 | aws-endpoint = "0.51.0" 18 | tokio = { version = "1", features = ["full"] } 19 | tokio-stream = "0.1" 20 | http = "0.2" 21 | futures = "0.3" 22 | -------------------------------------------------------------------------------- /fasts3/rustfmt.toml: -------------------------------------------------------------------------------- 1 | max_width = 120 2 | -------------------------------------------------------------------------------- /fasts3/src/fasts3_filesystem.rs: -------------------------------------------------------------------------------- 1 | use pyo3::exceptions::PyIOError; 2 | use pyo3::prelude::*; 3 | use pyo3::types::{IntoPyDict, PyByteArray, PyDateTime, PyList}; 4 | 5 | use aws_sdk_s3::types::ByteStream; 6 | use aws_sdk_s3::{Client, Endpoint, Error, Region}; 7 | use futures::future::try_join_all; 8 | use http::Uri; 9 | use tokio_stream::StreamExt; 10 | 11 | const READCHUNK: usize = 1024 * 1024 * 128; 12 | 13 | #[allow(dead_code)] 14 | fn print_type_of(_: &T) { 15 | println!("{}", std::any::type_name::()) 16 | } 17 | 18 | #[pyclass] 19 | pub struct FastS3FileSystem { 20 | #[pyo3(get, set)] 21 | pub endpoint: String, 22 | 23 | s3_client: aws_sdk_s3::Client, 24 | } 25 | 26 | fn build_client(endpoint: &str) -> aws_sdk_s3::Client { 27 | let region = Region::new("us-west-2"); 28 | 29 | let rt = tokio::runtime::Runtime::new().unwrap(); 30 | let conf = rt.block_on(async { aws_config::load_from_env().await }); 31 | 32 | let s3_conf = match endpoint.is_empty() { 33 | true => aws_sdk_s3::config::Builder::from(&conf).region(region).build(), 34 | false => aws_sdk_s3::config::Builder::from(&conf) 35 | .endpoint_resolver(Endpoint::immutable(endpoint.parse::().unwrap())) 36 | .region(region) 37 | .build(), 38 | }; 39 | 40 | Client::from_conf(s3_conf) 41 | } 42 | 43 | impl FastS3FileSystem { 44 | fn get_client(&self) -> &aws_sdk_s3::Client { 45 | &self.s3_client 46 | } 47 | } 48 | 49 | // Extract path into bucket + prefix 50 | fn path_to_bucketprefix(path: &String) -> (String, String) { 51 | let s3path = std::path::Path::new(path); 52 | let mut path_it = s3path.iter(); 53 | let bucket = path_it.next().unwrap().to_str().unwrap(); 54 | let mut prefix_path = std::path::PathBuf::new(); 55 | for p in path_it { 56 | prefix_path.push(p); 57 | } 58 | let mut prefix = prefix_path.to_str().unwrap().to_string(); 59 | if path.ends_with('/') { 60 | prefix.push('/'); 61 | } 62 | 63 | (bucket.to_string(), prefix) 64 | } 65 | 66 | // Write contents of ByteStream into destination buffer. 67 | async fn drain_stream(mut s: ByteStream, dest: &mut [u8]) -> Result { 68 | let mut offset = 0; 69 | while let Ok(Some(bytes)) = s.try_next().await { 70 | let span = offset..offset + bytes.len(); 71 | dest[span].clone_from_slice(&bytes); 72 | offset += bytes.len(); 73 | } 74 | Ok(offset) 75 | } 76 | 77 | #[pymethods] 78 | impl FastS3FileSystem { 79 | #[new] 80 | pub fn new(endpoint: String) -> FastS3FileSystem { 81 | let c = build_client(&endpoint); 82 | FastS3FileSystem { 83 | endpoint: endpoint, 84 | s3_client: c, 85 | } 86 | } 87 | 88 | pub fn ls(&self, path: &str) -> PyResult> { 89 | let (bucket, prefix) = path_to_bucketprefix(&path.to_string()); 90 | 91 | let client = self.get_client(); 92 | let rt = tokio::runtime::Runtime::new().unwrap(); 93 | 94 | let listing = rt.block_on(async { 95 | let mut page_stream = client 96 | .list_objects_v2() 97 | .bucket(&bucket) 98 | .prefix(&prefix) 99 | .delimiter('/') 100 | .into_paginator() 101 | .send(); 102 | 103 | let mut listing: Vec = Vec::new(); 104 | while let Some(Ok(lp)) = page_stream.next().await { 105 | for object in lp.contents().unwrap_or_default() { 106 | let key = object.key().unwrap_or_default(); 107 | listing.push(key.to_string()); 108 | } 109 | } 110 | Ok(listing) 111 | }); 112 | listing 113 | } 114 | 115 | pub fn info(&self, py: Python, path: &str) -> PyResult { 116 | let (bucket, key) = path_to_bucketprefix(&path.to_string()); 117 | let mut key_vals: Vec<(&str, PyObject)> = vec![("Key", path.to_object(py))]; 118 | 119 | let client = self.get_client(); 120 | let rt = tokio::runtime::Runtime::new().unwrap(); 121 | 122 | let _info_result = rt.block_on(async { 123 | let res = match client.head_object().bucket(bucket).key(key).send().await { 124 | Ok(r) => r, 125 | Err(e) => return Err(PyIOError::new_err(e.to_string())), 126 | }; 127 | key_vals.push(("Size", res.content_length.to_object(py))); 128 | if let Some(etag) = res.e_tag { 129 | key_vals.push(("ETag", etag.to_object(py))); 130 | } 131 | if let Some(mtime) = res.last_modified { 132 | let pytime = PyDateTime::from_timestamp(py, mtime.secs() as f64, None)?; 133 | key_vals.push(("LastModified", pytime.to_object(py))); 134 | } 135 | if let Some(ver) = res.version_id { 136 | key_vals.push(("VersionId", ver.to_object(py))); 137 | } else { 138 | key_vals.push(("VersionId", py.None())); 139 | } 140 | if let Some(sc) = res.storage_class { 141 | key_vals.push(("StorageClass", sc.as_str().to_object(py))); 142 | } else { 143 | // Default to standard because s3fs does. 144 | key_vals.push(("StorageClass", "STANDARD".to_object(py))); 145 | } 146 | // Several fields not implemented, like custom metadata. 147 | 148 | Ok(()) 149 | }); 150 | let dict = key_vals.into_py_dict(py); 151 | Ok(dict.into()) 152 | } 153 | 154 | pub fn get_objects(&self, py: Python, paths: Vec) -> PyResult { 155 | let pathpairs: Vec<(String, String)> = paths.iter().map(path_to_bucketprefix).collect(); 156 | 157 | let client = self.get_client(); 158 | let rt = tokio::runtime::Runtime::new().unwrap(); 159 | 160 | let mut pybuf_list = Vec::new(); 161 | for _ in &pathpairs { 162 | pybuf_list.push(PyByteArray::new(py, &[])); 163 | } 164 | 165 | let return_buf = rt.block_on(async { 166 | let mut head_reqs = vec![]; 167 | for (bucket, key) in &pathpairs { 168 | head_reqs.push(client.head_object().bucket(bucket).key(key).send()); 169 | } 170 | let head_results = match try_join_all(head_reqs).await { 171 | Ok(r) => r, 172 | Err(e) => return Err(PyIOError::new_err(e.to_string())), 173 | }; 174 | let obj_sizes: Vec = head_results.iter().map(|x| x.content_length() as usize).collect(); 175 | 176 | for (p, o) in pybuf_list.iter_mut().zip(obj_sizes) { 177 | p.resize(o)?; 178 | } 179 | 180 | let mut read_reqs = vec![]; 181 | 182 | for (pybuf, (bucket, key)) in pybuf_list.iter_mut().zip(&pathpairs) { 183 | let obj_size = pybuf.len(); 184 | let landing_buf = unsafe { pybuf.as_bytes_mut() }; 185 | let mut landing_slices: Vec<&mut [u8]> = landing_buf.chunks_mut(READCHUNK).collect(); 186 | 187 | let mut read_offset = 0; 188 | while read_offset < obj_size { 189 | let read_upper = std::cmp::min(obj_size, read_offset + READCHUNK); 190 | let byte_range = format!("bytes={}-{}", read_offset, read_upper - 1); 191 | 192 | let resp = match client 193 | .get_object() 194 | .bucket(bucket) 195 | .key(key) 196 | .range(byte_range) 197 | .send() 198 | .await 199 | { 200 | Ok(r) => r, 201 | Err(e) => return Err(PyIOError::new_err(e.to_string())), 202 | }; 203 | 204 | read_reqs.push(drain_stream(resp.body, landing_slices.remove(0))); 205 | 206 | read_offset += READCHUNK; 207 | } 208 | } 209 | let _results = try_join_all(read_reqs).await.unwrap(); 210 | 211 | let pybufs: &PyList = PyList::new(py, pybuf_list); 212 | Ok(pybufs) 213 | }); 214 | 215 | match return_buf { 216 | Ok(b) => Ok(b.into()), 217 | Err(e) => Err(e), 218 | } 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /fasts3/src/lib.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | 3 | mod fasts3_filesystem; 4 | use crate::fasts3_filesystem::FastS3FileSystem; 5 | 6 | /// A Python module implemented in Rust. The name of this function must match 7 | /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to 8 | /// import the module. 9 | #[pymodule] 10 | fn fasts3(_py: Python, m: &PyModule) -> PyResult<()> { 11 | m.add_class::()?; 12 | 13 | Ok(()) 14 | } 15 | -------------------------------------------------------------------------------- /regression_test.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import fasts3 3 | import fsspec 4 | import random 5 | 6 | ENDPOINT_URL='http://local-minio:9000' 7 | BUCKET="testbucket" 8 | 9 | s3r = boto3.resource('s3', endpoint_url=ENDPOINT_URL) 10 | 11 | s3r.create_bucket(Bucket=BUCKET) 12 | 13 | storage_options = {'endpoint_url': ENDPOINT_URL} 14 | fs = fsspec.filesystem('s3', client_kwargs=storage_options) 15 | 16 | s = fasts3.FastS3FileSystem(endpoint=ENDPOINT_URL) 17 | 18 | ####### 19 | 20 | # Initialize data on the remote (test) object store 21 | random.seed(0) 22 | BUFFERSIZE = 2 * 1024 * 1024 23 | large_buffer = bytearray(random.getrandbits(8) for _ in range(BUFFERSIZE * 256 + 67891)) 24 | 25 | src_buffers = [bytearray(random.getrandbits(8) for _ in range(BUFFERSIZE)) for _ in range(10)] 26 | 27 | for i in range(len(src_buffers)): 28 | obj = s3r.Object(BUCKET, "test-" + str(i)) 29 | obj.put(Body=src_buffers[i]) 30 | 31 | 32 | ## BEGIN test scenarios 33 | paginator = s3r.meta.client.get_paginator('list_objects_v2') 34 | pages = paginator.paginate(Bucket=BUCKET, Prefix='') 35 | listingb = [obj['Key'] for page in pages for obj in page['Contents']] 36 | print(listingb) 37 | 38 | assert len(listingb) == len(src_buffers), "FAIL on returned number of objects from list" 39 | 40 | contents = s.get_objects([BUCKET + '/' + "test-" + str(i) for i in range(len(src_buffers))]) 41 | 42 | assert len(contents) == len(src_buffers), "FAIL on returning all object contents" 43 | 44 | for i in range(len(src_buffers)): 45 | assert contents[i] == src_buffers[i], "FAIL, content mismatch on object {}".format(i) 46 | 47 | obj = s3r.Object(BUCKET, "test-big") 48 | obj.put(Body=large_buffer) 49 | 50 | BIGPATH=BUCKET + "/test-big" 51 | contents = s.get_objects([BIGPATH])[0] 52 | assert contents == large_buffer, "FAIL on large buffer content match" 53 | 54 | stat = fs.info(BIGPATH) 55 | 56 | fast_stat = s.info(BIGPATH) 57 | print(fast_stat) 58 | 59 | # Skipping LastModified for now because of timezone support. 60 | keys_to_check = ["Key", "ETag", "Size", "StorageClass", "VersionId"] 61 | for k in keys_to_check: 62 | assert stat[k] == fast_stat[k], "FAIL on {}".format(k) 63 | -------------------------------------------------------------------------------- /run_benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | IMG="pyo3-test" 4 | 5 | docker run -it --rm --name fasts3-bench \ 6 | --env-file ./credentials \ 7 | $IMG \ 8 | python3 /benchmark.py 9 | -------------------------------------------------------------------------------- /run_regression.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | IMG="pyo3-test" 4 | MINIO_IMG=quay.io/minio/minio 5 | 6 | ACCESSKEY="AKIAIOSFODMM7EXAMPLE" 7 | SECRETKEY="wJalrXUtnFEMI/K7MDENG/bQxRfiCYEXAMPLEKEY" 8 | 9 | echo "Starting local minio instance" 10 | docker run --rm -d --name local-minio \ 11 | -p 9000:9000 \ 12 | -p 9001:9001 \ 13 | -e "MINIO_ROOT_USER=$ACCESSKEY" \ 14 | -e "MINIO_ROOT_PASSWORD=$SECRETKEY" \ 15 | $MINIO_IMG server /data --console-address ":9001" 16 | 17 | echo "Starting fasts3 test client" 18 | docker run -it --rm --name fasts3-test \ 19 | --link local-minio:local-minio \ 20 | -e "AWS_ACCESS_KEY_ID=$ACCESSKEY" \ 21 | -e "AWS_SECRET_ACCESS_KEY=$SECRETKEY" \ 22 | $IMG \ 23 | python3 /regression_test.py 24 | 25 | echo "Finished, now shutting down local minio." 26 | docker stop local-minio 27 | --------------------------------------------------------------------------------