├── src ├── Storage.Tests │ ├── GlobalUsings.cs │ ├── Utils │ │ ├── CollectionUtilsShould.cs │ │ └── ValueStringBuilderShould.cs │ ├── ClientShould.cs │ ├── BucketShould.cs │ ├── Storage.Tests.csproj │ ├── StorageFixture.cs │ ├── TestHelper.cs │ └── ObjectShould.cs ├── Storage.Benchmark │ ├── Program.cs │ ├── appsettings.json │ ├── InternalBenchmarks │ │ ├── HashBenchmark.cs │ │ ├── DownloadBenchmark.cs │ │ └── SignatureBenchmark.cs │ ├── Storage.Benchmark.csproj │ ├── Utils │ │ ├── InputStream.cs │ │ └── BenchmarkHelper.cs │ └── S3Benchmark.cs └── Storage │ ├── GlobalUsings.cs │ ├── IArrayPool.cs │ ├── DefaultArrayPool.cs │ ├── Utils │ ├── CollectionUtils.cs │ ├── StreamUtils.cs │ ├── Errors.cs │ ├── HashHelper.cs │ ├── XmlStreamReader.cs │ ├── StringUtils.cs │ ├── HttpDescription.cs │ ├── ValueStringBuilder.cs │ └── Signature.cs │ ├── S3Settings.cs │ ├── S3Client.Transport.cs │ ├── S3Client.Buckets.cs │ ├── S3Stream.cs │ ├── Storage.csproj │ ├── S3File.cs │ ├── S3Client.Multipart.cs │ ├── S3Upload.cs │ └── S3Client.cs ├── .editorconfig ├── .dockerignore ├── docker-compose.yml ├── Dockerfile ├── .github └── workflows │ └── dotnet.yml ├── LICENSE ├── Storage.sln ├── .gitignore └── README.md /src/Storage.Tests/GlobalUsings.cs: -------------------------------------------------------------------------------- 1 | global using System.Text.Json; 2 | global using FluentAssertions; 3 | global using Xunit; 4 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Running; 2 | using Storage.Benchmark; 3 | 4 | BenchmarkRunner.Run(); 5 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | # All Files 4 | [*] 5 | charset = utf-8 6 | indent_style = tab 7 | indent_size = 4 8 | tab_width = 4 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true -------------------------------------------------------------------------------- /src/Storage/GlobalUsings.cs: -------------------------------------------------------------------------------- 1 | global using System.Diagnostics; 2 | global using System.Diagnostics.CodeAnalysis; 3 | global using System.Net; 4 | global using System.Runtime.CompilerServices; 5 | global using System.Buffers; 6 | global using System.Globalization; 7 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "BigFilePath": "", 3 | "S3Storage": { 4 | "AccessKey": "ROOTUSER", 5 | "Bucket": "benchmark", 6 | "EndPoint": "localhost", 7 | "Port": "5300", 8 | "SecretKey": "ChangeMe123", 9 | "UseHttps": false 10 | } 11 | } -------------------------------------------------------------------------------- /src/Storage/IArrayPool.cs: -------------------------------------------------------------------------------- 1 | namespace Storage; 2 | 3 | /// 4 | /// Интерфейс класса, который берёт массивы из пула и возвращает их в пул. 5 | /// 6 | public interface IArrayPool 7 | { 8 | T[] Rent(int minimumLength); 9 | 10 | void Return(T[] array, bool clear = false); 11 | } 12 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.dockerignore 2 | **/.env 3 | **/.git 4 | **/.gitignore 5 | **/.project 6 | **/.settings 7 | **/.toolstarget 8 | **/.vs 9 | **/.vscode 10 | **/.idea 11 | **/*.*proj.user 12 | **/*.dbmdl 13 | **/*.jfm 14 | **/azds.yaml 15 | **/bin 16 | **/charts 17 | **/docker-compose* 18 | **/Dockerfile* 19 | **/node_modules 20 | **/npm-debug.log 21 | **/obj 22 | **/secrets.dev.yaml 23 | **/values.dev.yaml 24 | LICENSE 25 | README.md -------------------------------------------------------------------------------- /src/Storage/DefaultArrayPool.cs: -------------------------------------------------------------------------------- 1 | namespace Storage; 2 | 3 | internal sealed class DefaultArrayPool : IArrayPool 4 | { 5 | public static readonly IArrayPool Instance = new DefaultArrayPool(); 6 | 7 | private DefaultArrayPool() 8 | { 9 | } 10 | 11 | public T[] Rent(int minimumLength) 12 | { 13 | return ArrayPool.Shared.Rent(minimumLength); 14 | } 15 | 16 | public void Return(T[] array, bool clear) 17 | { 18 | ArrayPool.Shared.Return(array, clear); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Storage/Utils/CollectionUtils.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Utils; 2 | 3 | internal static class CollectionUtils 4 | { 5 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 6 | public static void Resize(ref T[] array, IArrayPool arrayPool, int newLength, bool clear = false) 7 | { 8 | var newArray = arrayPool.Rent(newLength); 9 | 10 | if (array.Length > 0) 11 | { 12 | Array.Copy(array, newArray, array.Length); 13 | arrayPool.Return(array, clear); 14 | } 15 | 16 | array = newArray; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Storage/S3Settings.cs: -------------------------------------------------------------------------------- 1 | namespace Storage; 2 | 3 | public sealed class S3Settings 4 | { 5 | public required string AccessKey { get; init; } 6 | 7 | public required string Bucket { get; init; } 8 | 9 | public required string EndPoint { get; init; } 10 | 11 | public int? Port { get; init; } 12 | 13 | public string Region { get; init; } = "us-east-1"; 14 | 15 | public required string SecretKey { get; init; } 16 | 17 | public string Service { get; init; } = "s3"; 18 | 19 | public bool UseHttp2 { get; init; } 20 | 21 | public required bool UseHttps { get; init; } 22 | } 23 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: quay.io/minio/minio 4 | command: server /data --console-address ":9090" 5 | expose: 6 | - "9000" 7 | - "9090" 8 | ports: 9 | - "5300:9000" 10 | - "5301:9090" 11 | environment: 12 | MINIO_ROOT_USER: "ROOTUSER" 13 | MINIO_ROOT_PASSWORD: "ChangeMe123" 14 | volumes: 15 | - minio-data:/data 16 | 17 | benchmark: 18 | image: s3-benchmark 19 | depends_on: 20 | - minio 21 | build: 22 | context: . 23 | dockerfile: Dockerfile 24 | 25 | volumes: 26 | minio-data: -------------------------------------------------------------------------------- /src/Storage.Tests/Utils/CollectionUtilsShould.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | 3 | namespace Storage.Tests.Utils; 4 | 5 | public class CollectionUtilsShould 6 | { 7 | private static readonly IArrayPool Pool = DefaultArrayPool.Instance; 8 | 9 | [Fact] 10 | public void ResizeArray() 11 | { 12 | var array = Pool.Rent(5); 13 | 14 | var newLength = array.Length * 2; 15 | CollectionUtils.Resize(ref array, Pool, newLength); 16 | 17 | array.Length 18 | .Should() 19 | .BeGreaterThanOrEqualTo(newLength); 20 | } 21 | 22 | [Fact] 23 | public void ResizeEmptyArray() 24 | { 25 | const int newLength = 5; 26 | 27 | var emptyArray = Array.Empty(); 28 | CollectionUtils.Resize(ref emptyArray, Pool, newLength); 29 | 30 | emptyArray.Length 31 | .Should() 32 | .BeGreaterThanOrEqualTo(newLength); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/sdk:8.0 2 | WORKDIR /src 3 | COPY . . 4 | RUN dotnet restore "./src/Storage.Benchmark/Storage.Benchmark.csproj" && dotnet publish "./src/Storage.Benchmark/Storage.Benchmark.csproj" -c Release -o "./src/publish" 5 | ENTRYPOINT ["dotnet", "./src/publish/Storage.Benchmark.dll"] 6 | 7 | #RUN apt-get update -y && apt-get install -y wget && \ 8 | #wget -O dotMemoryclt.zip https://www.nuget.org/api/v2/package/JetBrains.dotMemory.Console.linux-x64/2022.3.3 && \ 9 | #apt-get install -y unzip && \ 10 | #unzip dotMemoryclt.zip -d ./dotMemoryclt && \ 11 | #chmod +x -R ./dotMemoryclt/* 12 | # 13 | #ENTRYPOINT ./dotMemoryclt/tools/dotmemory start-net-core --temp-dir=./src/dotMemoryclt/tmp --timeout=16m --save-to-dir=./src/dotMemoryclt/workspaces --log-file=./src/dotMemoryclt/tmp/log.txt --trigger-timer=1m ./src/publish/Storage.Benchmark.dll 14 | -------------------------------------------------------------------------------- /.github/workflows/dotnet.yml: -------------------------------------------------------------------------------- 1 | name: .NET Core 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - name: Setup .NET 15 | uses: actions/setup-dotnet@v3 16 | with: 17 | dotnet-version: 8.x.x 18 | - name: Build with dotnet 19 | run: dotnet build -c Release 20 | - name: Run tests with coverage 21 | run: | 22 | cd ./src/Storage.Tests/ 23 | dotnet test -c Release --no-build /p:CollectCoverage=true /p:CoverletOutput=TestResults/ /p:CoverletOutputFormat=opencover /p:Exclude="[*]Storage.Benchmark.*" 24 | - name: Publish coverage report 25 | uses: codecov/codecov-action@v3 26 | with: 27 | token: ${{ secrets.CODECOV_TOKEN }} 28 | -------------------------------------------------------------------------------- /src/Storage/Utils/StreamUtils.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Utils; 2 | 3 | internal static class StreamUtils 4 | { 5 | public static async Task ReadTo(this Stream stream, byte[] buffer, CancellationToken cancellation) 6 | { 7 | var length = buffer.Length; 8 | var offset = 0; 9 | 10 | int written; 11 | do 12 | { 13 | written = await stream 14 | .ReadAsync(buffer.AsMemory(offset, length), cancellation) 15 | .ConfigureAwait(false); 16 | 17 | length -= written; 18 | offset += written; 19 | } 20 | while (written > 0 && length > 0); 21 | 22 | return offset; 23 | } 24 | 25 | [MethodImpl(MethodImplOptions.NoInlining)] 26 | public static long? TryGetLength(this Stream stream) 27 | { 28 | try 29 | { 30 | var length = stream.Length; 31 | return length == 0 ? null : length; 32 | } 33 | catch (NotSupportedException) 34 | { 35 | return null; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Storage/Utils/Errors.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Utils; 2 | 3 | internal static class Errors 4 | { 5 | [DoesNotReturn] 6 | [MethodImpl(MethodImplOptions.NoInlining)] 7 | public static void CantFormatToString(T value) 8 | where T : struct 9 | { 10 | throw new FormatException($"Can't format '{value}' to string"); 11 | } 12 | 13 | [DoesNotReturn] 14 | [MethodImpl(MethodImplOptions.NoInlining)] 15 | public static void Disposed() 16 | { 17 | throw new ObjectDisposedException(nameof(S3Client)); 18 | } 19 | 20 | [DoesNotReturn] 21 | [MethodImpl(MethodImplOptions.NoInlining)] 22 | public static void UnexpectedResult(HttpResponseMessage response) 23 | { 24 | var reason = response.ReasonPhrase ?? response.ToString(); 25 | var exception = new HttpRequestException($"Storage has returned an unexpected result: {response.StatusCode} ({reason})"); 26 | 27 | response.Dispose(); 28 | 29 | throw exception; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/InternalBenchmarks/HashBenchmark.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using BenchmarkDotNet.Jobs; 3 | using Storage.Benchmark.Utils; 4 | using Storage.Utils; 5 | 6 | namespace Storage.Benchmark.InternalBenchmarks; 7 | 8 | [SimpleJob(RuntimeMoniker.Net70)] 9 | [MeanColumn] 10 | [MemoryDiagnoser] 11 | public class HashBenchmark 12 | { 13 | private byte[] _byteData = null!; 14 | private string _stringData = null!; 15 | 16 | [GlobalSetup] 17 | public void Config() 18 | { 19 | var config = BenchmarkHelper.ReadConfiguration(); 20 | 21 | _byteData = BenchmarkHelper.ReadBigArray(config); 22 | _stringData = "C10F4FFD-BB46-452C-B054-C595EB92248E"; 23 | } 24 | 25 | [Benchmark] 26 | public int ByteHash() 27 | { 28 | return HashHelper 29 | .GetPayloadHash(_byteData, DefaultArrayPool.Instance) 30 | .Length; 31 | } 32 | 33 | [Benchmark] 34 | public int StringHash() 35 | { 36 | return HashHelper 37 | .GetPayloadHash(_stringData, DefaultArrayPool.Instance) 38 | .Length; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Storage.Tests/ClientShould.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Tests; 2 | 3 | public sealed class ClientShould(StorageFixture fixture) : IClassFixture 4 | { 5 | private readonly CancellationToken _ct = CancellationToken.None; 6 | private readonly S3Client _client = fixture.S3Client; 7 | 8 | [Fact] 9 | public void DeserializeSettingsJson() 10 | { 11 | var expected = fixture.Settings; 12 | 13 | var json = JsonSerializer.Serialize(expected); 14 | var actual = JsonSerializer.Deserialize(json); 15 | 16 | actual.Should().BeEquivalentTo(expected); 17 | } 18 | 19 | [Fact] 20 | public void HasValidInfo() 21 | { 22 | _client 23 | .Bucket 24 | .Should().Be(fixture.Settings.Bucket); 25 | } 26 | 27 | [Fact] 28 | public Task ThrowIfDisposed() 29 | { 30 | var client = TestHelper.CloneClient(fixture, null, new HttpClient()); 31 | 32 | client.Dispose(); 33 | 34 | return client 35 | .Invoking(c => c.CreateBucket(_ct)) 36 | .Should().ThrowExactlyAsync(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/Storage.Tests/Utils/ValueStringBuilderShould.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | 3 | namespace Storage.Tests.Utils; 4 | 5 | public class ValueStringBuilderShould 6 | { 7 | [Fact] 8 | public void Grow() 9 | { 10 | const int stringLength = 256; 11 | var chars = Enumerable.Range(0, stringLength).Select(i => (char)i); 12 | 13 | var builder = new ValueStringBuilder(stackalloc char[64], DefaultArrayPool.Instance); 14 | foreach (var c in chars) 15 | { 16 | builder.Append(c); 17 | } 18 | 19 | builder.Length.Should().Be(stringLength); 20 | } 21 | 22 | [Fact] 23 | public void NotCreateEmptyString() 24 | { 25 | var builder = new ValueStringBuilder(stackalloc char[64], DefaultArrayPool.Instance); 26 | builder 27 | .ToString() 28 | .Should().BeEmpty(); 29 | } 30 | 31 | [Fact] 32 | public void RemoveLastCorrectly() 33 | { 34 | var builder = new ValueStringBuilder(stackalloc char[64], DefaultArrayPool.Instance); 35 | builder.RemoveLast(); 36 | 37 | builder.Length 38 | .Should().BeGreaterThan(-1); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Kirill Bazhaykin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /src/Storage.Benchmark/Storage.Benchmark.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net8.0 6 | latest 7 | enable 8 | enable 9 | false 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | PreserveNewest 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/Utils/InputStream.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Benchmark.Utils; 2 | 3 | internal sealed class InputStream : Stream 4 | { 5 | private readonly Stream _baseStream; 6 | 7 | public InputStream(byte[] data) 8 | { 9 | _baseStream = new MemoryStream(); 10 | _baseStream.Write(data); 11 | } 12 | 13 | public override long Position 14 | { 15 | get => _baseStream.Position; 16 | set => _baseStream.Position = value; 17 | } 18 | 19 | public override bool CanRead => true; 20 | 21 | public override bool CanSeek => true; 22 | 23 | public override bool CanWrite => false; 24 | 25 | public override long Length => _baseStream.Length; 26 | 27 | public override void Close() 28 | { 29 | _baseStream.Seek(0, SeekOrigin.Begin); 30 | } 31 | 32 | public override void Flush() 33 | { 34 | _baseStream.Flush(); 35 | } 36 | 37 | public override int Read(byte[] buffer, int offset, int count) 38 | { 39 | return _baseStream.Read(buffer, offset, count); 40 | } 41 | 42 | public override long Seek(long offset, SeekOrigin origin) 43 | { 44 | return _baseStream.Seek(offset, origin); 45 | } 46 | 47 | public override void SetLength(long value) 48 | { 49 | _baseStream.SetLength(value); 50 | } 51 | 52 | public override void Write(byte[] buffer, int offset, int count) 53 | { 54 | _baseStream.Write(buffer, offset, count); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/InternalBenchmarks/DownloadBenchmark.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using Storage.Benchmark.Utils; 3 | 4 | namespace Storage.Benchmark.InternalBenchmarks; 5 | 6 | [MeanColumn] 7 | [MemoryDiagnoser] 8 | [InProcess] 9 | public class DownloadBenchmark 10 | { 11 | private CancellationToken _cancellation; 12 | private string _fileId = null!; 13 | private S3Client _s3Client = null!; 14 | 15 | [GlobalSetup] 16 | public void Config() 17 | { 18 | var config = BenchmarkHelper.ReadConfiguration(); 19 | var settings = BenchmarkHelper.ReadSettings(config); 20 | 21 | _cancellation = CancellationToken.None; 22 | _fileId = "привет-как-делаdcd156a8-b6bd-4130-a2c7-8a38dbfebbc7"; 23 | _s3Client = BenchmarkHelper.CreateStoragesClient(settings); 24 | 25 | // BenchmarkHelper.EnsureBucketExists(_storageClient, _cancellation); 26 | // BenchmarkHelper.EnsureFileExists(config, _storageClient, _fileId, _cancellation); 27 | } 28 | 29 | [GlobalCleanup] 30 | public void Clear() 31 | { 32 | _s3Client.Dispose(); 33 | } 34 | 35 | [Benchmark] 36 | public async Task JustDownload() 37 | { 38 | using var file = await _s3Client.GetFile(_fileId, _cancellation); 39 | 40 | return await BenchmarkHelper.ReadStreamMock( 41 | await file.GetStream(_cancellation), 42 | BenchmarkHelper.StreamBuffer, 43 | _cancellation); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/InternalBenchmarks/SignatureBenchmark.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Attributes; 2 | using BenchmarkDotNet.Jobs; 3 | using Storage.Benchmark.Utils; 4 | using Storage.Utils; 5 | 6 | namespace Storage.Benchmark.InternalBenchmarks; 7 | 8 | [SimpleJob(RuntimeMoniker.Net70)] 9 | [MeanColumn] 10 | [MemoryDiagnoser] 11 | public class SignatureBenchmark 12 | { 13 | private string[] _headers = null!; 14 | private DateTime _now; 15 | private HttpRequestMessage _request = null!; 16 | private string _payloadHash = null!; 17 | private Signature _signature = null!; 18 | 19 | [GlobalSetup] 20 | public void Config() 21 | { 22 | var config = BenchmarkHelper.ReadConfiguration(); 23 | var data = BenchmarkHelper.ReadBigArray(config); 24 | var settings = BenchmarkHelper.ReadSettings(config); 25 | 26 | _headers = ["host", "x-amz-content-sha256", "x-amz-date"]; 27 | _now = DateTime.UtcNow; 28 | _request = new HttpRequestMessage(HttpMethod.Post, "http://company-name.com/controller"); 29 | _payloadHash = HashHelper.GetPayloadHash(data, DefaultArrayPool.Instance); 30 | _signature = new Signature( 31 | new HttpDescription(DefaultArrayPool.Instance, "", "", "", []), 32 | settings.SecretKey, settings.Region, settings.Service); 33 | } 34 | 35 | [Benchmark] 36 | public int Content() 37 | { 38 | return _signature 39 | .Calculate(_request, _payloadHash, _headers, _now) 40 | .Length; 41 | } 42 | 43 | [Benchmark] 44 | public int Url() 45 | { 46 | return _signature 47 | .Calculate("http://subdomain-name.company-name.com/controller/method?arg=eto-arg", _now) 48 | .Length; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Storage/S3Client.Transport.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | 3 | namespace Storage; 4 | 5 | /// 6 | /// Transport functions 7 | /// 8 | public sealed partial class S3Client 9 | { 10 | [SkipLocalsInit] 11 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 12 | private HttpRequestMessage CreateRequest(HttpMethod method, string? fileName = null) 13 | { 14 | var url = new ValueStringBuilder(stackalloc char[512], ArrayPool); 15 | url.Append(_bucket); 16 | 17 | // ReSharper disable once InvertIf 18 | if (!string.IsNullOrEmpty(fileName)) 19 | { 20 | url.Append('/'); 21 | _httpDescription.AppendEncodedName(ref url, fileName); 22 | } 23 | 24 | return new HttpRequestMessage(method, new Uri(url.Flush(), UriKind.Absolute)); 25 | } 26 | 27 | private Task Send(HttpRequestMessage request, string payloadHash, CancellationToken ct) 28 | { 29 | if (_disposed) 30 | { 31 | Errors.Disposed(); 32 | } 33 | 34 | var now = DateTime.UtcNow; 35 | 36 | var headers = request.Headers; 37 | headers.Add("host", _endpoint); 38 | headers.Add("x-amz-content-sha256", payloadHash); 39 | headers.Add("x-amz-date", now.ToString(Signature.Iso8601DateTime, CultureInfo.InvariantCulture)); 40 | 41 | if (_useHttp2) 42 | { 43 | request.Version = HttpVersion.Version20; 44 | } 45 | 46 | var signature = _signature.Calculate(request, payloadHash, S3Headers, now); 47 | headers.TryAddWithoutValidation("Authorization", _httpDescription.BuildHeader(now, signature)); 48 | 49 | return _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Storage.Tests/BucketShould.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Tests; 2 | 3 | public sealed class BucketShould(StorageFixture fixture) : IClassFixture 4 | { 5 | private readonly CancellationToken _ct = CancellationToken.None; 6 | private readonly S3Client _client = fixture.S3Client; 7 | 8 | [Fact] 9 | public async Task CreateBucket() 10 | { 11 | // don't dispose it 12 | var client = TestHelper.CloneClient(fixture); 13 | 14 | var bucketCreateResult = await client.CreateBucket(_ct); 15 | 16 | bucketCreateResult 17 | .Should().BeTrue(); 18 | 19 | await DeleteTestBucket(client); 20 | } 21 | 22 | [Fact] 23 | public async Task BeExists() 24 | { 25 | var bucketExistsResult = await _client.IsBucketExists(_ct); 26 | 27 | bucketExistsResult 28 | .Should().BeTrue(); 29 | } 30 | 31 | [Fact] 32 | public async Task BeNotExists() 33 | { 34 | // don't dispose it 35 | var client = TestHelper.CloneClient(fixture); 36 | 37 | var bucketExistsResult = await client.IsBucketExists(_ct); 38 | 39 | bucketExistsResult 40 | .Should().BeFalse(); 41 | } 42 | 43 | [Fact] 44 | public Task NotThrowIfCreateBucketAlreadyExists() 45 | { 46 | return _client 47 | .Invoking(client => client.CreateBucket(_ct)) 48 | .Should().NotThrowAsync(); 49 | } 50 | 51 | [Fact] 52 | public Task NotThrowIfDeleteNotExistsBucket() 53 | { 54 | // don't dispose it 55 | var client = TestHelper.CloneClient(fixture); 56 | 57 | return client 58 | .Invoking(c => c.DeleteBucket(_ct)) 59 | .Should().NotThrowAsync(); 60 | } 61 | 62 | private async Task DeleteTestBucket(S3Client client) 63 | { 64 | var bucketDeleteResult = await client.DeleteBucket(_ct); 65 | 66 | bucketDeleteResult 67 | .Should().BeTrue(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/Storage.Tests/Storage.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | latest 6 | enable 7 | enable 8 | false 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | all 21 | runtime; build; native; contentfiles; analyzers; buildtransitive 22 | 23 | 24 | all 25 | runtime; build; native; contentfiles; analyzers; buildtransitive 26 | 27 | 28 | all 29 | runtime; build; native; contentfiles; analyzers; buildtransitive 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /Storage.sln: -------------------------------------------------------------------------------- 1 | 2 | Microsoft Visual Studio Solution File, Format Version 12.00 3 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Storage", "src\Storage\Storage.csproj", "{A44AEC9A-8F03-48EF-8AF4-10C1A5F47E01}" 4 | EndProject 5 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Storage.Benchmark", "src\Storage.Benchmark\Storage.Benchmark.csproj", "{9306915C-AF76-42E9-B94B-CE87E2B341DF}" 6 | EndProject 7 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Storage.Tests", "src\Storage.Tests\Storage.Tests.csproj", "{4CF65AA7-268C-4F47-935F-DD55A1E2A295}" 8 | EndProject 9 | Global 10 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 11 | Debug|Any CPU = Debug|Any CPU 12 | Release|Any CPU = Release|Any CPU 13 | EndGlobalSection 14 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 15 | {A44AEC9A-8F03-48EF-8AF4-10C1A5F47E01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 16 | {A44AEC9A-8F03-48EF-8AF4-10C1A5F47E01}.Debug|Any CPU.Build.0 = Debug|Any CPU 17 | {A44AEC9A-8F03-48EF-8AF4-10C1A5F47E01}.Release|Any CPU.ActiveCfg = Release|Any CPU 18 | {A44AEC9A-8F03-48EF-8AF4-10C1A5F47E01}.Release|Any CPU.Build.0 = Release|Any CPU 19 | {9306915C-AF76-42E9-B94B-CE87E2B341DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 20 | {9306915C-AF76-42E9-B94B-CE87E2B341DF}.Debug|Any CPU.Build.0 = Debug|Any CPU 21 | {9306915C-AF76-42E9-B94B-CE87E2B341DF}.Release|Any CPU.ActiveCfg = Release|Any CPU 22 | {9306915C-AF76-42E9-B94B-CE87E2B341DF}.Release|Any CPU.Build.0 = Release|Any CPU 23 | {4CF65AA7-268C-4F47-935F-DD55A1E2A295}.Debug|Any CPU.ActiveCfg = Debug|Any CPU 24 | {4CF65AA7-268C-4F47-935F-DD55A1E2A295}.Debug|Any CPU.Build.0 = Debug|Any CPU 25 | {4CF65AA7-268C-4F47-935F-DD55A1E2A295}.Release|Any CPU.ActiveCfg = Release|Any CPU 26 | {4CF65AA7-268C-4F47-935F-DD55A1E2A295}.Release|Any CPU.Build.0 = Release|Any CPU 27 | EndGlobalSection 28 | EndGlobal 29 | -------------------------------------------------------------------------------- /src/Storage/Utils/HashHelper.cs: -------------------------------------------------------------------------------- 1 | using System.Security.Cryptography; 2 | using System.Text; 3 | 4 | namespace Storage.Utils; 5 | 6 | internal static class HashHelper 7 | { 8 | public static readonly string EmptyPayloadHash = Sha256ToHex(string.Empty, DefaultArrayPool.Instance); 9 | 10 | [SkipLocalsInit] 11 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 12 | public static string GetPayloadHash(ReadOnlySpan data, IArrayPool arrayPool) 13 | { 14 | Span hashBuffer = stackalloc byte[32]; 15 | return SHA256.TryHashData(data, hashBuffer, out _) 16 | ? ToHex(hashBuffer, arrayPool) 17 | : string.Empty; 18 | } 19 | 20 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 21 | public static string GetPayloadHash(string data, IArrayPool arrayPool) 22 | { 23 | return Sha256ToHex(data, arrayPool); 24 | } 25 | 26 | [SkipLocalsInit] 27 | public static string ToHex(ReadOnlySpan data, IArrayPool arrayPool) 28 | { 29 | Span buffer = stackalloc char[2]; 30 | using var builder = new ValueStringBuilder(stackalloc char[64], arrayPool); 31 | 32 | foreach (ref readonly var element in data) 33 | { 34 | builder.Append(buffer[..StringUtils.FormatX2(ref buffer, element)]); 35 | } 36 | 37 | return builder.Flush(); 38 | } 39 | 40 | [SkipLocalsInit] 41 | private static string Sha256ToHex(ReadOnlySpan value, IArrayPool arrayPool) 42 | { 43 | var count = Encoding.UTF8.GetByteCount(value); 44 | 45 | var byteBuffer = arrayPool.Rent(count); 46 | 47 | var encoded = Encoding.UTF8.GetBytes(value, byteBuffer); 48 | Span hashBuffer = stackalloc byte[64]; 49 | var result = SHA256.TryHashData(byteBuffer.AsSpan(0, encoded), hashBuffer, out var written) 50 | ? ToHex(hashBuffer[..written], arrayPool) 51 | : string.Empty; 52 | 53 | arrayPool.Return(byteBuffer); 54 | 55 | return result; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Storage.Tests/StorageFixture.cs: -------------------------------------------------------------------------------- 1 | using AutoFixture; 2 | using DotNet.Testcontainers.Containers; 3 | 4 | namespace Storage.Tests; 5 | 6 | public sealed class StorageFixture : IDisposable, IAsyncDisposable 7 | { 8 | public const string StreamContentType = "application/octet-stream"; 9 | 10 | private const int DefaultByteArraySize = 1 * 1024 * 1024; // 7Mb 11 | 12 | private readonly IContainer? _container; 13 | private Fixture? _fixture; 14 | 15 | public StorageFixture() 16 | { 17 | _container = TestHelper.CreateContainer(); 18 | 19 | Settings = TestHelper.CreateSettings(_container); 20 | HttpClient = new HttpClient(); 21 | S3Client = new S3Client(Settings); 22 | 23 | TestHelper.EnsureBucketExists(S3Client); 24 | } 25 | 26 | internal S3Settings Settings { get; } 27 | 28 | internal S3Client S3Client { get; } 29 | 30 | internal HttpClient HttpClient { get; } 31 | 32 | internal Fixture Mocks => _fixture ??= new Fixture(); 33 | 34 | public static byte[] GetByteArray(int size = DefaultByteArraySize) 35 | { 36 | var random = Random.Shared; 37 | var bytes = new byte[size]; 38 | for (var i = 0; i < bytes.Length; i++) 39 | { 40 | bytes[i] = (byte)random.Next(); 41 | } 42 | 43 | return bytes; 44 | } 45 | 46 | public static MemoryStream GetByteStream(int size = DefaultByteArraySize) 47 | { 48 | return new MemoryStream(GetByteArray(size)); 49 | } 50 | 51 | public static MemoryStream GetEmptyByteStream(long? size = null) 52 | { 53 | return size.HasValue 54 | ? new MemoryStream(new byte[(int)size]) 55 | : new MemoryStream(); 56 | } 57 | 58 | public T Create() 59 | { 60 | return Mocks.Create(); 61 | } 62 | 63 | public void Dispose() 64 | { 65 | HttpClient.Dispose(); 66 | S3Client.Dispose(); 67 | } 68 | 69 | public async ValueTask DisposeAsync() 70 | { 71 | if (_container != null) 72 | { 73 | await _container.DisposeAsync(); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Storage/Utils/XmlStreamReader.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Utils; 2 | 3 | internal static class XmlStreamReader 4 | { 5 | [SkipLocalsInit] 6 | public static string ReadString(Stream stream, ReadOnlySpan elementName, int valueBufferLength = 256) 7 | { 8 | Span buffer = stackalloc char[valueBufferLength]; 9 | 10 | var written = ReadTo(stream, elementName, ref buffer); 11 | return written is -1 12 | ? string.Empty 13 | : buffer[..written].ToString(); 14 | } 15 | 16 | private static int ReadTo(Stream stream, ReadOnlySpan elementName, ref Span valueBuffer) 17 | { 18 | var expectedIndex = 0; 19 | var propertyLength = elementName.Length; 20 | var sectionStarted = false; 21 | 22 | while (true) 23 | { 24 | var nextByte = stream.ReadByte(); 25 | if (nextByte is -1) 26 | { 27 | break; 28 | } 29 | 30 | var nextChar = (char)nextByte; 31 | if (sectionStarted) 32 | { 33 | if (nextChar == elementName[expectedIndex]) 34 | { 35 | if (++expectedIndex == propertyLength) 36 | { 37 | if ((char)stream.ReadByte() is '>') 38 | { 39 | return ReadValue(stream, ref valueBuffer); 40 | } 41 | 42 | expectedIndex = 0; 43 | sectionStarted = false; 44 | } 45 | } 46 | else 47 | { 48 | sectionStarted = false; 49 | } 50 | 51 | continue; 52 | } 53 | 54 | if (nextChar is '<') 55 | { 56 | sectionStarted = true; 57 | } 58 | } 59 | 60 | return -1; 61 | } 62 | 63 | private static int ReadValue(Stream stream, ref Span valueBuffer) 64 | { 65 | var index = 0; 66 | while (true) 67 | { 68 | var nextByte = stream.ReadByte(); 69 | if (nextByte is -1) 70 | { 71 | break; 72 | } 73 | 74 | var nextChar = (char)nextByte; 75 | if (nextChar is '<') 76 | { 77 | return index; 78 | } 79 | 80 | valueBuffer[index++] = nextChar; 81 | } 82 | 83 | return -1; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/Storage/S3Client.Buckets.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | using static Storage.Utils.HashHelper; 3 | 4 | namespace Storage; 5 | 6 | /// 7 | /// Функции управления бакетом 8 | /// 9 | public sealed partial class S3Client 10 | { 11 | public async Task CreateBucket(CancellationToken ct) 12 | { 13 | HttpResponseMessage response; 14 | using (var request = CreateRequest(HttpMethod.Put)) 15 | { 16 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 17 | } 18 | 19 | switch (response.StatusCode) 20 | { 21 | case HttpStatusCode.OK: 22 | response.Dispose(); 23 | return true; 24 | case HttpStatusCode.Conflict: // already exists 25 | response.Dispose(); 26 | return false; 27 | default: 28 | Errors.UnexpectedResult(response); 29 | return false; 30 | } 31 | } 32 | 33 | public async Task DeleteBucket(CancellationToken ct) 34 | { 35 | HttpResponseMessage response; 36 | using (var request = CreateRequest(HttpMethod.Delete)) 37 | { 38 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 39 | } 40 | 41 | switch (response.StatusCode) 42 | { 43 | case HttpStatusCode.NoContent: 44 | response.Dispose(); 45 | return true; 46 | case HttpStatusCode.NotFound: 47 | response.Dispose(); 48 | return false; 49 | default: 50 | Errors.UnexpectedResult(response); 51 | return false; 52 | } 53 | } 54 | 55 | public async Task IsBucketExists(CancellationToken ct) 56 | { 57 | HttpResponseMessage response; 58 | using (var request = CreateRequest(HttpMethod.Head)) 59 | { 60 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 61 | } 62 | 63 | switch (response.StatusCode) 64 | { 65 | case HttpStatusCode.OK: 66 | response.Dispose(); 67 | return true; 68 | case HttpStatusCode.NotFound: 69 | response.Dispose(); 70 | return false; 71 | default: 72 | Errors.UnexpectedResult(response); 73 | return false; 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Storage/Utils/StringUtils.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace Storage.Utils; 4 | 5 | internal static class StringUtils 6 | { 7 | [ThreadStatic] 8 | private static StringBuilder? _sharedBuilder; 9 | 10 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 11 | public static StringBuilder Append(this StringBuilder builder, string start, int middle, string end) 12 | { 13 | return builder 14 | .Append(start) 15 | .Append(middle) 16 | .Append(end); 17 | } 18 | 19 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 20 | public static StringBuilder Append(this StringBuilder builder, string start, string middle, string end) 21 | { 22 | return builder 23 | .Append(start) 24 | .Append(middle) 25 | .Append(end); 26 | } 27 | 28 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 29 | public static StringBuilder GetBuilder() 30 | { 31 | return Interlocked.Exchange(ref _sharedBuilder, null) 32 | ?? new StringBuilder(4096); 33 | } 34 | 35 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 36 | public static int Format(ref Span buffer, DateTime dateTime, string format) 37 | { 38 | return dateTime.TryFormat(buffer, out var written, format, CultureInfo.InvariantCulture) 39 | ? written 40 | : -1; 41 | } 42 | 43 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 44 | public static int FormatX2(ref Span buffer, byte value) 45 | { 46 | return value.TryFormat(buffer, out var written, "x2", CultureInfo.InvariantCulture) 47 | ? written 48 | : -1; 49 | } 50 | 51 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 52 | public static int FormatX2(ref Span buffer, char value) 53 | { 54 | return ((int)value).TryFormat(buffer, out var written, "x2", CultureInfo.InvariantCulture) 55 | ? written 56 | : -1; 57 | } 58 | 59 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 60 | public static string Flush(this StringBuilder builder) 61 | { 62 | var result = builder.ToString(); 63 | builder.Clear(); 64 | 65 | Interlocked.Exchange(ref _sharedBuilder, builder); 66 | return result; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Storage/S3Stream.cs: -------------------------------------------------------------------------------- 1 | namespace Storage; 2 | 3 | internal sealed class S3Stream(HttpResponseMessage response, Stream stream) : Stream 4 | { 5 | private long? _length; 6 | 7 | public override bool CanRead => stream.CanRead; 8 | 9 | public override bool CanSeek => stream.CanSeek; 10 | 11 | public override bool CanWrite => stream.CanWrite; 12 | 13 | public override long Length 14 | { 15 | get 16 | { 17 | _length ??= response.Content.Headers.ContentLength ?? stream.Length; 18 | return _length.Value; 19 | } 20 | } 21 | 22 | [ExcludeFromCodeCoverage] 23 | public override long Position 24 | { 25 | get => stream.Position; 26 | set => stream.Position = value; 27 | } 28 | 29 | public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) 30 | { 31 | return stream.ReadAsync(buffer, offset, count, cancellationToken); 32 | } 33 | 34 | public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) 35 | { 36 | return stream.ReadAsync(buffer, cancellationToken); 37 | } 38 | 39 | public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) 40 | { 41 | return stream.CopyToAsync(destination, bufferSize, cancellationToken); 42 | } 43 | 44 | [ExcludeFromCodeCoverage] 45 | public override void Flush() 46 | { 47 | stream.Flush(); 48 | } 49 | 50 | public override int Read(byte[] buffer, int offset, int count) 51 | { 52 | return stream.Read(buffer, offset, count); 53 | } 54 | 55 | [ExcludeFromCodeCoverage] 56 | public override long Seek(long offset, SeekOrigin origin) 57 | { 58 | return stream.Seek(offset, origin); 59 | } 60 | 61 | [ExcludeFromCodeCoverage] 62 | public override void SetLength(long value) 63 | { 64 | stream.SetLength(value); 65 | } 66 | 67 | [ExcludeFromCodeCoverage] 68 | public override void Write(byte[] buffer, int offset, int count) 69 | { 70 | stream.Write(buffer, offset, count); 71 | } 72 | 73 | protected override void Dispose(bool disposing) 74 | { 75 | stream.Dispose(); 76 | response.Dispose(); 77 | 78 | base.Dispose(true); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/Storage/Storage.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0 5 | latest 6 | enable 7 | enable 8 | true 9 | false 10 | true 11 | 12 | 13 | 14 | Storage 15 | Simple client for S3-storage 16 | 0.6.4 17 | git 18 | git://github.com/teoadal/Storage 19 | Storages3 20 | s3;s3storage;objectS3;bucketS3;performance 21 | https://github.com/teoadal/Storage 22 | MIT 23 | Kirill Bazhaykin 24 | README.md 25 | true 26 | snupkg 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | S3Client.cs 48 | 49 | 50 | S3Client.cs 51 | 52 | 53 | S3Client.cs 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /src/Storage/S3File.cs: -------------------------------------------------------------------------------- 1 | namespace Storage; 2 | 3 | /// 4 | /// Обёртка над , позволяющая удобно работать с загруженным файлом 5 | /// 6 | [DebuggerDisplay("{ToString()}")] 7 | public sealed class S3File : IDisposable 8 | { 9 | private readonly HttpResponseMessage _response; 10 | 11 | internal S3File(HttpResponseMessage response) 12 | { 13 | _response = response; 14 | } 15 | 16 | /// 17 | /// Тип файла в MIME 18 | /// 19 | /// Берётся из заголовка "Content-Type" из 20 | public string? ContentType 21 | { 22 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 23 | get => _response.Content.Headers.ContentType?.MediaType; 24 | } 25 | 26 | /// 27 | /// Файл существовал? 28 | /// 29 | public bool Exists 30 | { 31 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 32 | get => _response.IsSuccessStatusCode; 33 | } 34 | 35 | /// 36 | /// Размер файла 37 | /// 38 | /// Берётся из заголовка "Content-Length" из 39 | public long? Length 40 | { 41 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 42 | get => _response.Content.Headers.ContentLength; 43 | } 44 | 45 | /// 46 | /// Ответ сервера 47 | /// 48 | public HttpStatusCode StatusCode 49 | { 50 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 51 | get => _response.StatusCode; 52 | } 53 | 54 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 55 | public static implicit operator bool(S3File file) 56 | { 57 | return file._response.IsSuccessStatusCode; 58 | } 59 | 60 | public void Dispose() 61 | { 62 | _response.Dispose(); 63 | } 64 | 65 | /// 66 | /// Возвращает Stream с данными файла из 67 | /// 68 | /// Stream of data 69 | /// Когда Stream будет закрыт, будет уничтожен 70 | public async Task GetStream(CancellationToken ct) 71 | { 72 | if (!_response.IsSuccessStatusCode) 73 | { 74 | return Stream.Null; 75 | } 76 | 77 | var stream = await _response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); 78 | return new S3Stream(_response, stream); 79 | } 80 | 81 | [ExcludeFromCodeCoverage] 82 | public override string ToString() 83 | { 84 | if (_response.IsSuccessStatusCode) 85 | { 86 | return $"OK (Length = {Length})"; 87 | } 88 | 89 | var reasonPhrase = _response.ReasonPhrase; 90 | var statusCode = _response.StatusCode; 91 | return string.IsNullOrEmpty(reasonPhrase) 92 | ? $"{statusCode} ({(int)statusCode})" 93 | : $"{statusCode} ({(int)statusCode}, '{reasonPhrase}')"; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/Storage.Tests/TestHelper.cs: -------------------------------------------------------------------------------- 1 | using System.Globalization; 2 | using DotNet.Testcontainers.Builders; 3 | using DotNet.Testcontainers.Containers; 4 | 5 | namespace Storage.Tests; 6 | 7 | public static class TestHelper 8 | { 9 | public const int MinioInternalPort = 9000; 10 | public const string SecretKey = "ChangeMe123"; 11 | public const string Username = "ROOTUSER"; 12 | 13 | public static S3Client CloneClient(StorageFixture fixture, string? bucket = null, HttpClient? client = null) 14 | { 15 | var settings = fixture.Settings; 16 | var clonedSettings = new S3Settings 17 | { 18 | AccessKey = settings.AccessKey, 19 | Bucket = bucket ?? fixture.Create(), 20 | EndPoint = settings.EndPoint, 21 | Port = settings.Port, 22 | SecretKey = settings.SecretKey, 23 | UseHttps = settings.UseHttps, 24 | }; 25 | 26 | return new S3Client(clonedSettings, client ?? fixture.HttpClient); 27 | } 28 | 29 | public static S3Settings CreateSettings(IContainer? container = null) 30 | { 31 | var environmentPort = Environment.GetEnvironmentVariable("STORAGE_PORT"); 32 | int? port = string.IsNullOrEmpty(environmentPort) 33 | ? container?.GetMappedPublicPort(MinioInternalPort) ?? 5300 34 | : environmentPort is "null" 35 | ? null 36 | : int.Parse(environmentPort, CultureInfo.InvariantCulture); 37 | 38 | var environmentHttps = Environment.GetEnvironmentVariable("STORAGE_HTTPS"); 39 | var https = !string.IsNullOrEmpty(environmentHttps) && bool.Parse(environmentHttps); 40 | 41 | return new S3Settings 42 | { 43 | AccessKey = Environment.GetEnvironmentVariable("STORAGE_KEY") ?? Username, 44 | Bucket = Environment.GetEnvironmentVariable("STORAGE_BUCKET") ?? "reconfig", 45 | EndPoint = Environment.GetEnvironmentVariable("STORAGE_ENDPOINT") ?? "localhost", 46 | Port = port, 47 | SecretKey = Environment.GetEnvironmentVariable("STORAGE_SECRET") ?? SecretKey, 48 | UseHttps = https, 49 | }; 50 | } 51 | 52 | public static IContainer CreateContainer(bool run = true) 53 | { 54 | var container = new ContainerBuilder() 55 | .WithImage("minio/minio:latest") 56 | .WithEnvironment("MINIO_ROOT_USER", Username) 57 | .WithEnvironment("MINIO_ROOT_PASSWORD", SecretKey) 58 | .WithPortBinding(MinioInternalPort, true) 59 | .WithCommand("server", "/data") 60 | .WithWaitStrategy(Wait 61 | .ForUnixContainer() 62 | .UntilHttpRequestIsSucceeded(static request => 63 | request.ForPath("/minio/health/ready").ForPort(MinioInternalPort))) 64 | .Build(); 65 | 66 | if (run) 67 | { 68 | container 69 | .StartAsync() 70 | .ConfigureAwait(false) 71 | .GetAwaiter() 72 | .GetResult(); 73 | } 74 | 75 | return container; 76 | } 77 | 78 | public static void EnsureBucketExists(S3Client client) 79 | { 80 | EnsureBucketExists(client, CancellationToken.None) 81 | .ConfigureAwait(false) 82 | .GetAwaiter() 83 | .GetResult(); 84 | } 85 | 86 | public static async Task EnsureBucketExists(S3Client client, CancellationToken cancellation) 87 | { 88 | if (await client.IsBucketExists(cancellation).ConfigureAwait(false)) 89 | { 90 | return; 91 | } 92 | 93 | await client.CreateBucket(cancellation).ConfigureAwait(false); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/Storage/Utils/HttpDescription.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Frozen; 2 | using System.Text; 3 | 4 | namespace Storage.Utils; 5 | 6 | internal sealed class HttpDescription 7 | { 8 | private static readonly FrozenSet ValidUrlCharacters = 9 | "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~/".ToFrozenSet(); 10 | 11 | public readonly IArrayPool ArrayPool; 12 | 13 | private readonly string _headerEnd; 14 | private readonly string _headerStart; 15 | 16 | private readonly string _urlMiddle; 17 | private readonly string _urlStart; 18 | 19 | public HttpDescription( 20 | IArrayPool arrayPool, 21 | string accessKey, 22 | string region, 23 | string service, 24 | string[] signedHeaders) 25 | { 26 | ArrayPool = arrayPool; 27 | 28 | _headerStart = $"AWS4-HMAC-SHA256 Credential={accessKey}/"; 29 | _headerEnd = $"/{region}/{service}/aws4_request, SignedHeaders={string.Join(';', signedHeaders)}, Signature="; 30 | 31 | _urlStart = $"?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential={accessKey}%2F"; 32 | _urlMiddle = $"%2F{region}%2F{service}%2Faws4_request"; 33 | } 34 | 35 | [SkipLocalsInit] 36 | public bool AppendEncodedName(scoped ref ValueStringBuilder builder, ReadOnlySpan name) 37 | { 38 | var count = Encoding.UTF8.GetByteCount(name); 39 | var hasEncoded = false; 40 | 41 | var byteBuffer = ArrayPool.Rent(count); 42 | 43 | Span charBuffer = stackalloc char[2]; 44 | Span upperBuffer = stackalloc char[2]; 45 | 46 | var validCharacters = ValidUrlCharacters; 47 | var encoded = Encoding.UTF8.GetBytes(name, byteBuffer); 48 | 49 | try 50 | { 51 | var span = byteBuffer.AsSpan(0, encoded); 52 | foreach (var element in span) 53 | { 54 | var symbol = (char)element; 55 | if (validCharacters.Contains(symbol)) 56 | { 57 | builder.Append(symbol); 58 | } 59 | else 60 | { 61 | builder.Append('%'); 62 | 63 | StringUtils.FormatX2(ref charBuffer, symbol); 64 | MemoryExtensions.ToUpperInvariant(charBuffer, upperBuffer); 65 | builder.Append(upperBuffer); 66 | 67 | hasEncoded = true; 68 | } 69 | } 70 | 71 | return hasEncoded; 72 | } 73 | finally 74 | { 75 | ArrayPool.Return(byteBuffer); 76 | } 77 | } 78 | 79 | [SkipLocalsInit] 80 | public string EncodeName(string fileName) 81 | { 82 | var builder = new ValueStringBuilder(stackalloc char[fileName.Length], ArrayPool); 83 | var encoded = AppendEncodedName(ref builder, fileName); 84 | 85 | return encoded 86 | ? builder.Flush() 87 | : fileName; 88 | } 89 | 90 | [SkipLocalsInit] 91 | public string BuildHeader(DateTime now, string signature) 92 | { 93 | using var builder = new ValueStringBuilder(stackalloc char[512], ArrayPool); 94 | 95 | builder.Append(_headerStart); 96 | builder.Append(now, Signature.Iso8601Date); 97 | builder.Append(_headerEnd); 98 | builder.Append(signature); 99 | 100 | return builder.Flush(); 101 | } 102 | 103 | [SkipLocalsInit] 104 | public string BuildUrl(string bucket, string fileName, DateTime now, TimeSpan expires) 105 | { 106 | var builder = new ValueStringBuilder(stackalloc char[512], ArrayPool); 107 | 108 | builder.Append(bucket); 109 | builder.Append('/'); 110 | 111 | AppendEncodedName(ref builder, fileName); 112 | 113 | builder.Append(_urlStart); 114 | builder.Append(now, Signature.Iso8601Date); 115 | builder.Append(_urlMiddle); 116 | 117 | builder.Append("&X-Amz-Date="); 118 | builder.Append(now, Signature.Iso8601DateTime); 119 | builder.Append("&X-Amz-Expires="); 120 | builder.Append(expires.TotalSeconds); 121 | 122 | builder.Append("&X-Amz-SignedHeaders=host"); 123 | 124 | return builder.Flush(); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/Utils/BenchmarkHelper.cs: -------------------------------------------------------------------------------- 1 | using Amazon; 2 | using Amazon.Runtime; 3 | using Amazon.S3; 4 | using Microsoft.Extensions.Configuration; 5 | using Minio; 6 | 7 | namespace Storage.Benchmark.Utils; 8 | 9 | internal static class BenchmarkHelper 10 | { 11 | public static readonly byte[] StreamBuffer = new byte[2048]; 12 | 13 | // ReSharper disable once InconsistentNaming 14 | public static AmazonS3Client CreateAWSClient(S3Settings settings) 15 | { 16 | var scheme = settings.UseHttps ? Uri.UriSchemeHttps : Uri.UriSchemeHttp; 17 | var port = settings.Port.HasValue ? $":{settings.Port}" : string.Empty; 18 | 19 | return new AmazonS3Client( 20 | new BasicAWSCredentials(settings.AccessKey, settings.SecretKey), 21 | new AmazonS3Config 22 | { 23 | RegionEndpoint = RegionEndpoint.USEast1, 24 | ServiceURL = $"{scheme}://{settings.EndPoint}{port}", 25 | ForcePathStyle = true, // MUST be true to work correctly with MinIO server 26 | }); 27 | } 28 | 29 | public static IMinioClient CreateMinioClient(S3Settings settings) 30 | { 31 | var builder = new MinioClient(); 32 | var port = settings.Port; 33 | if (port.HasValue) 34 | { 35 | builder.WithEndpoint(settings.EndPoint, port.Value); 36 | } 37 | else 38 | { 39 | builder.WithEndpoint(settings.EndPoint); 40 | } 41 | 42 | return builder 43 | .WithCredentials(settings.AccessKey, settings.SecretKey) 44 | .WithSSL(settings.UseHttps) 45 | .Build(); 46 | } 47 | 48 | public static S3Client CreateStoragesClient(S3Settings settings) 49 | { 50 | return new S3Client(settings); 51 | } 52 | 53 | public static void EnsureBucketExists(S3Client client, CancellationToken cancellation) 54 | { 55 | if (client.IsBucketExists(cancellation).GetAwaiter().GetResult()) 56 | { 57 | return; 58 | } 59 | 60 | client.CreateBucket(cancellation).GetAwaiter().GetResult(); 61 | } 62 | 63 | public static void EnsureFileExists( 64 | IConfiguration config, 65 | S3Client client, 66 | string fileName, 67 | CancellationToken cancellation) 68 | { 69 | var fileData = ReadBigFile(config); 70 | fileData.Seek(0, SeekOrigin.Begin); 71 | 72 | var result = client 73 | .UploadFile(fileName, "application/pdf", fileData, cancellation) 74 | .GetAwaiter() 75 | .GetResult(); 76 | 77 | if (!result) 78 | { 79 | throw new Exception("File isn't uploaded"); 80 | } 81 | } 82 | 83 | public static byte[] ReadBigArray(IConfiguration config) 84 | { 85 | var filePath = config.GetValue("BigFilePath"); 86 | 87 | return !string.IsNullOrEmpty(filePath) && File.Exists(filePath) 88 | ? File.ReadAllBytes(filePath) 89 | : GetByteArray(123 * 1024 * 1024); // 123 Mb 90 | } 91 | 92 | public static InputStream ReadBigFile(IConfiguration config) 93 | { 94 | return new InputStream(ReadBigArray(config)); 95 | } 96 | 97 | public static IConfiguration ReadConfiguration() 98 | { 99 | return new ConfigurationBuilder() 100 | .AddJsonFile("appsettings.json", false) 101 | .Build(); 102 | } 103 | 104 | public static async Task ReadStreamMock(Stream input, byte[] buffer, CancellationToken cancellation) 105 | { 106 | var result = 0; 107 | while (await input.ReadAsync(buffer, cancellation) != 0) 108 | { 109 | result++; 110 | } 111 | 112 | await input.DisposeAsync(); 113 | 114 | return result; 115 | } 116 | 117 | public static S3Settings ReadSettings(IConfiguration config) 118 | { 119 | var settings = config.GetRequiredSection("S3Storage").Get(); 120 | if (settings == null || string.IsNullOrEmpty(settings.EndPoint)) 121 | { 122 | throw new Exception("S3Storage configuration is not found"); 123 | } 124 | 125 | var isContainer = Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER"); 126 | if (isContainer != null && bool.TryParse(isContainer, out var value) && value) 127 | { 128 | settings = new S3Settings 129 | { 130 | AccessKey = settings.AccessKey, 131 | Bucket = settings.Bucket, 132 | EndPoint = "host.docker.internal", 133 | Port = settings.Port, 134 | Region = settings.Region, 135 | SecretKey = settings.SecretKey, 136 | Service = settings.Service, 137 | UseHttp2 = settings.UseHttp2, 138 | UseHttps = settings.UseHttps, 139 | }; 140 | } 141 | 142 | return settings; 143 | } 144 | 145 | private static byte[] GetByteArray(int size) 146 | { 147 | var random = Random.Shared; 148 | var bytes = new byte[size]; 149 | for (var i = 0; i < bytes.Length; i++) 150 | { 151 | bytes[i] = (byte)random.Next(); 152 | } 153 | 154 | return bytes; 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/Storage/Utils/ValueStringBuilder.cs: -------------------------------------------------------------------------------- 1 | namespace Storage.Utils; 2 | 3 | internal ref struct ValueStringBuilder(Span initialBuffer, IArrayPool arrayPool) 4 | { 5 | private Span _buffer = initialBuffer; 6 | private int _length = 0; 7 | private char[]? _array = null; 8 | 9 | // ReSharper disable once ReplaceWithPrimaryConstructorParameter 10 | private readonly IArrayPool _arrayPool = arrayPool; 11 | 12 | // ReSharper disable once ConvertToAutoPropertyWithPrivateSetter 13 | public readonly int Length 14 | { 15 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 16 | get => _length; 17 | } 18 | 19 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 20 | public void Append(char c) 21 | { 22 | var pos = _length; 23 | if ((uint)pos < (uint)_buffer.Length) 24 | { 25 | _buffer[pos] = c; 26 | _length = pos + 1; 27 | } 28 | else 29 | { 30 | GrowAndAppend(c); 31 | } 32 | } 33 | 34 | [SkipLocalsInit] 35 | public void Append(int value) 36 | { 37 | Span buffer = stackalloc char[10]; 38 | var pos = _length; 39 | if (value.TryFormat(buffer, out var written, provider: CultureInfo.InvariantCulture)) 40 | { 41 | if (pos > _buffer.Length - written) 42 | { 43 | Grow(written); 44 | } 45 | 46 | buffer.CopyTo(_buffer[pos..]); 47 | 48 | _length = pos + written; 49 | } 50 | else 51 | { 52 | Errors.CantFormatToString(value); 53 | } 54 | } 55 | 56 | [SkipLocalsInit] 57 | public void Append(DateTime value, string format) 58 | { 59 | Span buffer = stackalloc char[18]; 60 | var pos = _length; 61 | if (value.TryFormat(buffer, out var written, format, CultureInfo.InvariantCulture)) 62 | { 63 | if (pos > _buffer.Length - written) 64 | { 65 | Grow(written); 66 | } 67 | 68 | buffer.CopyTo(_buffer[pos..]); 69 | 70 | _length = pos + written; 71 | } 72 | else 73 | { 74 | Errors.CantFormatToString(value); 75 | } 76 | } 77 | 78 | [SkipLocalsInit] 79 | public void Append(double value) 80 | { 81 | Span buffer = stackalloc char[33]; 82 | var pos = _length; 83 | if (value.TryFormat(buffer, out var written, default, CultureInfo.InvariantCulture)) 84 | { 85 | if (pos > _buffer.Length - written) 86 | { 87 | Grow(written); 88 | } 89 | 90 | buffer.CopyTo(_buffer[pos..]); 91 | 92 | _length = pos + written; 93 | } 94 | else 95 | { 96 | Errors.CantFormatToString(value); 97 | } 98 | } 99 | 100 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 101 | public void Append(string? s) 102 | { 103 | if (string.IsNullOrEmpty(s)) 104 | { 105 | return; 106 | } 107 | 108 | var pos = _length; 109 | if (s.Length == 1 && (uint)pos < (uint)_buffer.Length) 110 | { 111 | _buffer[pos] = s[0]; 112 | _length = pos + 1; 113 | } 114 | else 115 | { 116 | Append(s.AsSpan()); 117 | } 118 | } 119 | 120 | public void Append(scoped Span value) 121 | { 122 | var pos = _length; 123 | var valueLength = value.Length; 124 | 125 | if (pos > _buffer.Length - valueLength) 126 | { 127 | Grow(valueLength); 128 | } 129 | 130 | value.CopyTo(_buffer[pos..]); 131 | 132 | _length = pos + valueLength; 133 | } 134 | 135 | public void Append(ReadOnlySpan value) 136 | { 137 | var pos = _length; 138 | var valueLength = value.Length; 139 | 140 | if (pos > _buffer.Length - valueLength) 141 | { 142 | Grow(valueLength); 143 | } 144 | 145 | value.CopyTo(_buffer[pos..]); 146 | 147 | _length = pos + valueLength; 148 | } 149 | 150 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 151 | public readonly ReadOnlySpan AsReadonlySpan() 152 | { 153 | return _length == 0 154 | ? ReadOnlySpan.Empty 155 | : _buffer[.._length]; 156 | } 157 | 158 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 159 | public readonly void Dispose() 160 | { 161 | var toReturn = _array; 162 | if (toReturn is not null) 163 | { 164 | ArrayPool.Shared.Return(toReturn); 165 | } 166 | } 167 | 168 | public readonly string Flush() 169 | { 170 | var result = _length == 0 171 | ? string.Empty 172 | : _buffer[.._length].ToString(); 173 | 174 | Dispose(); 175 | 176 | return result; 177 | } 178 | 179 | public void RemoveLast() 180 | { 181 | if (_length == 0) 182 | { 183 | return; 184 | } 185 | 186 | _length--; 187 | } 188 | 189 | [ExcludeFromCodeCoverage] 190 | public readonly override string ToString() 191 | { 192 | return _length is 0 193 | ? string.Empty 194 | : _buffer[.._length].ToString(); 195 | } 196 | 197 | [MethodImpl(MethodImplOptions.NoInlining)] 198 | private void GrowAndAppend(char c) 199 | { 200 | Grow(1); 201 | Append(c); 202 | } 203 | 204 | [MethodImpl(MethodImplOptions.NoInlining)] 205 | private void Grow(int additionalCapacityBeyondPos) 206 | { 207 | const uint arrayMaxLength = 0x7FFFFFC7; // same as Array.MaxLength 208 | 209 | var newCapacity = (int)Math.Max( 210 | (uint)(_length + additionalCapacityBeyondPos), 211 | Math.Min((uint)_buffer.Length * 2, arrayMaxLength)); 212 | 213 | var poolArray = _arrayPool.Rent(newCapacity); 214 | 215 | _buffer[.._length].CopyTo(poolArray); 216 | 217 | var toReturn = _array; 218 | _buffer = _array = poolArray; 219 | if (toReturn is not null) 220 | { 221 | _arrayPool.Return(toReturn); 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /src/Storage/S3Client.Multipart.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | using Storage.Utils; 3 | using static Storage.Utils.HashHelper; 4 | 5 | namespace Storage; 6 | 7 | /// 8 | /// Функции управления multipart-загрузкой 9 | /// 10 | public sealed partial class S3Client 11 | { 12 | internal async Task MultipartAbort(string encodedFileName, string uploadId, CancellationToken ct) 13 | { 14 | var url = $"{_bucket}/{encodedFileName}?uploadId={uploadId}"; 15 | 16 | HttpResponseMessage? response = null; 17 | using (var request = new HttpRequestMessage(HttpMethod.Delete, url)) 18 | { 19 | try 20 | { 21 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 22 | } 23 | catch 24 | { 25 | // ignored 26 | } 27 | } 28 | 29 | if (response is null) 30 | { 31 | return false; 32 | } 33 | 34 | var result = response is 35 | { 36 | IsSuccessStatusCode: true, 37 | StatusCode: HttpStatusCode.NoContent, 38 | }; 39 | 40 | response.Dispose(); 41 | 42 | return result; 43 | } 44 | 45 | internal async Task MultipartComplete( 46 | string encodedFileName, 47 | string uploadId, 48 | string[] partTags, 49 | int tagsCount, 50 | CancellationToken ct) 51 | { 52 | var builder = StringUtils.GetBuilder(); 53 | 54 | builder.Append(""); 55 | for (var i = 0; i < partTags.Length; i++) 56 | { 57 | if (i == tagsCount) 58 | { 59 | break; 60 | } 61 | 62 | builder.Append(""); 63 | builder.Append("", i + 1, ""); 64 | builder.Append("", partTags[i], ""); 65 | builder.Append(""); 66 | } 67 | 68 | var data = builder 69 | .Append("") 70 | .Flush(); 71 | 72 | var payloadHash = GetPayloadHash(data, ArrayPool); 73 | 74 | HttpResponseMessage response; 75 | using (var request = new HttpRequestMessage( 76 | HttpMethod.Post, 77 | $"{_bucket}/{encodedFileName}?uploadId={uploadId}")) 78 | { 79 | using (var content = new StringContent(data, Encoding.UTF8)) 80 | { 81 | request.Content = content; 82 | response = await Send(request, payloadHash, ct).ConfigureAwait(false); 83 | } 84 | } 85 | 86 | var result = response is { IsSuccessStatusCode: true, StatusCode: HttpStatusCode.OK }; 87 | 88 | response.Dispose(); 89 | return result; 90 | } 91 | 92 | internal async Task MultipartPutPart( 93 | string encodedFileName, 94 | string uploadId, 95 | int partNumber, 96 | byte[] partData, 97 | int partSize, 98 | CancellationToken ct) 99 | { 100 | var payloadHash = GetPayloadHash(partData.AsSpan(0, partSize), ArrayPool); 101 | var url = $"{_bucket}/{encodedFileName}?partNumber={partNumber}&uploadId={uploadId}"; 102 | 103 | HttpResponseMessage response; 104 | using (var request = new HttpRequestMessage(HttpMethod.Put, url)) 105 | { 106 | using (var content = new ByteArrayContent(partData, 0, partSize)) 107 | { 108 | content.Headers.Add("content-length", partSize.ToString()); 109 | request.Content = content; 110 | 111 | response = await Send(request, payloadHash, ct).ConfigureAwait(false); 112 | } 113 | } 114 | 115 | var result = response is { IsSuccessStatusCode: true, StatusCode: HttpStatusCode.OK } 116 | ? response.Headers.ETag?.Tag 117 | : null; 118 | 119 | response.Dispose(); 120 | return result; 121 | } 122 | 123 | private async Task ExecuteMultipartUpload( 124 | string fileName, 125 | string contentType, 126 | Stream data, 127 | CancellationToken ct) 128 | { 129 | using var upload = await UploadFile(fileName, contentType, ct).ConfigureAwait(false); 130 | 131 | if (await upload.AddParts(data, ct).ConfigureAwait(false) && await upload.Complete(ct).ConfigureAwait(false)) 132 | { 133 | return true; 134 | } 135 | 136 | await upload.Abort(ct).ConfigureAwait(false); 137 | return false; 138 | } 139 | 140 | private async Task ExecuteMultipartUpload( 141 | string fileName, 142 | string contentType, 143 | byte[] data, 144 | CancellationToken ct) 145 | { 146 | using var upload = await UploadFile(fileName, contentType, ct).ConfigureAwait(false); 147 | 148 | if (await upload.AddParts(data, ct).ConfigureAwait(false) && await upload.Complete(ct).ConfigureAwait(false)) 149 | { 150 | return true; 151 | } 152 | 153 | await upload.Abort(ct).ConfigureAwait(false); 154 | return false; 155 | } 156 | 157 | private async Task MultipartStart(string encodedFileName, string contentType, CancellationToken ct) 158 | { 159 | HttpResponseMessage response; 160 | using (var request = new HttpRequestMessage(HttpMethod.Post, $"{_bucket}/{encodedFileName}?uploads")) 161 | { 162 | using (var content = new ByteArrayContent([])) 163 | { 164 | content.Headers.Add("content-type", contentType); 165 | request.Content = content; 166 | 167 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 168 | } 169 | } 170 | 171 | if (response.StatusCode is HttpStatusCode.OK) 172 | { 173 | var responseStream = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); 174 | var result = XmlStreamReader.ReadString(responseStream, "UploadId"); 175 | 176 | await responseStream.DisposeAsync().ConfigureAwait(false); 177 | response.Dispose(); 178 | 179 | return result; 180 | } 181 | 182 | Errors.UnexpectedResult(response); 183 | return string.Empty; 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /src/Storage/S3Upload.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | 3 | namespace Storage; 4 | 5 | /// 6 | /// Структура управления Multipart-загрузкой 7 | /// 8 | public sealed class S3Upload : IDisposable 9 | { 10 | private readonly IArrayPool _arrayPool; 11 | private readonly S3Client _client; 12 | private readonly string _encodedFileName; 13 | 14 | private byte[]? _byteBuffer; 15 | private bool _disposed; 16 | private int _partCount; 17 | private string[] _parts; 18 | 19 | internal S3Upload(S3Client client, string fileName, string encodedFileName, string uploadId) 20 | { 21 | FileName = fileName; 22 | UploadId = uploadId; 23 | 24 | _arrayPool = client.ArrayPool; 25 | _client = client; 26 | _encodedFileName = encodedFileName; 27 | 28 | _parts = client.ArrayPool.Rent(16); 29 | } 30 | 31 | /// 32 | /// Название загружаемого файла 33 | /// 34 | public string FileName 35 | { 36 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 37 | get; 38 | } 39 | 40 | /// 41 | /// Идентификатор загрузки 42 | /// 43 | public string UploadId 44 | { 45 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 46 | get; 47 | } 48 | 49 | /// 50 | /// Сколько байт загружено 51 | /// 52 | public long Written 53 | { 54 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 55 | get; 56 | private set; 57 | } 58 | 59 | /// 60 | /// Прерывает загрузку и удалить временные данные с сервера 61 | /// 62 | /// Токен отмены операции 63 | /// Возвращает результат отмены загрузки 64 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 65 | public Task Abort(CancellationToken ct) 66 | { 67 | if (_disposed) 68 | { 69 | Errors.Disposed(); 70 | } 71 | 72 | return _client.MultipartAbort(_encodedFileName, UploadId, ct); 73 | } 74 | 75 | /// 76 | /// Загружает блок данных на сервер 77 | /// 78 | /// Блок данных 79 | /// Токен отмены операции 80 | /// Возвращает результат загрузки 81 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 82 | public Task AddPart(byte[] data, CancellationToken ct) 83 | { 84 | return AddPart(data, data.Length, ct); 85 | } 86 | 87 | /// 88 | /// Загружает блок данных указанного размера на сервер 89 | /// 90 | /// Блок данных 91 | /// Количество данных, которые нужно взять из блока 92 | /// Токен отмены 93 | /// Возвращает результат загрузки 94 | public async Task AddPart(byte[] data, int length, CancellationToken ct) 95 | { 96 | if (_disposed) 97 | { 98 | Errors.Disposed(); 99 | } 100 | 101 | var partId = await _client.MultipartPutPart( 102 | _encodedFileName, UploadId, _partCount + 1, data, length, ct); 103 | 104 | if (string.IsNullOrEmpty(partId)) 105 | { 106 | return false; 107 | } 108 | 109 | if (_parts.Length == _partCount) 110 | { 111 | CollectionUtils.Resize(ref _parts, _arrayPool, _partCount * 2); 112 | } 113 | 114 | _parts[_partCount++] = partId; 115 | 116 | Written += length; 117 | 118 | return true; 119 | } 120 | 121 | /// 122 | /// Разделяет данные на блоки и загружает их на сервер 123 | /// 124 | /// Блок данных 125 | /// Токен отмены операции 126 | /// Возвращает результат загрузки 127 | public async Task AddParts(Stream data, CancellationToken ct) 128 | { 129 | _byteBuffer ??= _arrayPool.Rent(S3Client.DefaultPartSize); 130 | 131 | while (true) 132 | { 133 | var written = await data.ReadTo(_byteBuffer, ct).ConfigureAwait(false); 134 | if (written is 0) 135 | { 136 | break; 137 | } 138 | 139 | if (!await AddPart(_byteBuffer, written, ct).ConfigureAwait(false)) 140 | { 141 | return false; 142 | } 143 | } 144 | 145 | return true; 146 | } 147 | 148 | /// 149 | /// Разделяет данные на блоки и загружает их на сервер 150 | /// 151 | /// Блок данных 152 | /// Токен отмены операции 153 | /// Возвращает результат загрузки 154 | public async Task AddParts(byte[] data, CancellationToken ct) 155 | { 156 | _byteBuffer ??= ArrayPool.Shared.Rent(S3Client.DefaultPartSize); 157 | 158 | var bufferLength = _byteBuffer.Length; 159 | var offset = 0; 160 | while (offset < data.Length) 161 | { 162 | var partSize = Math.Min(bufferLength, data.Length - offset); 163 | Array.Copy(data, offset, _byteBuffer, 0, partSize); 164 | 165 | if (!await AddPart(_byteBuffer, partSize, ct).ConfigureAwait(false)) 166 | { 167 | return false; 168 | } 169 | 170 | offset += partSize; 171 | } 172 | 173 | return true; 174 | } 175 | 176 | /// 177 | /// Указывает серверу, что загрузка завершена и блоки данных можно объединить в файл 178 | /// 179 | /// Токен отмены операции 180 | /// Возвращает результат завершения загрузки 181 | public Task Complete(CancellationToken ct) 182 | { 183 | if (_disposed) 184 | { 185 | Errors.Disposed(); 186 | } 187 | 188 | return _partCount == 0 189 | ? Task.FromResult(false) 190 | : _client.MultipartComplete(_encodedFileName, UploadId, _parts, _partCount, ct); 191 | } 192 | 193 | public void Dispose() 194 | { 195 | if (_disposed) 196 | { 197 | return; 198 | } 199 | 200 | Array.Clear(_parts, 0, _partCount); 201 | _arrayPool.Return(_parts); 202 | _parts = null!; 203 | 204 | if (_byteBuffer is not null) 205 | { 206 | _arrayPool.Return(_byteBuffer); 207 | _byteBuffer = null; 208 | } 209 | 210 | _disposed = true; 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/Storage.Benchmark/S3Benchmark.cs: -------------------------------------------------------------------------------- 1 | using Amazon.S3; 2 | using Amazon.S3.Model; 3 | using Amazon.S3.Transfer; 4 | using Amazon.S3.Util; 5 | using BenchmarkDotNet.Attributes; 6 | using BenchmarkDotNet.Jobs; 7 | using Minio; 8 | using Minio.DataModel.Args; 9 | using Storage.Benchmark.Utils; 10 | 11 | namespace Storage.Benchmark; 12 | 13 | [SimpleJob(RuntimeMoniker.Net80)] 14 | [MeanColumn] 15 | [MemoryDiagnoser] 16 | public class S3Benchmark 17 | { 18 | private string _bucket = null!; 19 | private CancellationToken _cancellation; 20 | private InputStream _inputData = null!; 21 | private string _fileId = null!; 22 | private MemoryStream _outputData = null!; 23 | 24 | private IAmazonS3 _amazonClient = null!; 25 | private TransferUtility _amazonTransfer = null!; 26 | private IMinioClient _minioClient = null!; 27 | private S3Client _s3Client = null!; 28 | 29 | [GlobalSetup] 30 | public void Config() 31 | { 32 | var config = BenchmarkHelper.ReadConfiguration(); 33 | var settings = BenchmarkHelper.ReadSettings(config); 34 | 35 | _bucket = settings.Bucket; 36 | _cancellation = default; 37 | _fileId = $"привет-как-дела{Guid.NewGuid()}"; 38 | _inputData = BenchmarkHelper.ReadBigFile(config); 39 | _outputData = new MemoryStream(new byte[_inputData.Length]); 40 | 41 | _amazonClient = BenchmarkHelper.CreateAWSClient(settings); 42 | _amazonTransfer = new TransferUtility(_amazonClient); 43 | _minioClient = BenchmarkHelper.CreateMinioClient(settings); 44 | _s3Client = BenchmarkHelper.CreateStoragesClient(settings); 45 | 46 | BenchmarkHelper.EnsureBucketExists(_s3Client, _cancellation); 47 | } 48 | 49 | [GlobalCleanup] 50 | public void Clear() 51 | { 52 | _s3Client.Dispose(); 53 | _inputData.Dispose(); 54 | _outputData.Dispose(); 55 | } 56 | 57 | [Benchmark] 58 | public async Task Aws() 59 | { 60 | var result = 0; 61 | 62 | await AmazonS3Util.DoesS3BucketExistV2Async(_amazonClient, _bucket); 63 | 64 | try 65 | { 66 | await _amazonClient.GetObjectMetadataAsync(_bucket, _fileId, _cancellation); 67 | } 68 | catch (Exception) 69 | { 70 | result++; // it's OK - file not found 71 | } 72 | 73 | _inputData.Seek(0, SeekOrigin.Begin); 74 | 75 | await _amazonTransfer.UploadAsync(_inputData, _bucket, _fileId, _cancellation); 76 | result++; 77 | 78 | await _amazonClient.GetObjectMetadataAsync(_bucket, _fileId, _cancellation); 79 | result++; 80 | 81 | _outputData.Seek(0, SeekOrigin.Begin); 82 | var fileDownload = await _amazonClient.GetObjectAsync(_bucket, _fileId, _cancellation); 83 | await fileDownload.ResponseStream.CopyToAsync(_outputData, _cancellation); 84 | result++; 85 | 86 | await _amazonClient.DeleteObjectAsync( 87 | new DeleteObjectRequest { BucketName = _bucket, Key = _fileId }, 88 | _cancellation); 89 | 90 | return ++result; 91 | } 92 | 93 | [Benchmark] 94 | public async Task Minio() 95 | { 96 | var result = 0; 97 | 98 | if (!await _minioClient.BucketExistsAsync(new BucketExistsArgs().WithBucket(_bucket), _cancellation)) 99 | { 100 | ThrowException(); 101 | } 102 | 103 | result++; 104 | 105 | try 106 | { 107 | await _minioClient.StatObjectAsync( 108 | new StatObjectArgs() 109 | .WithBucket(_bucket) 110 | .WithObject(_fileId), 111 | _cancellation); 112 | } 113 | catch (Exception) 114 | { 115 | result++; // it's OK - file not found 116 | } 117 | 118 | _inputData.Seek(0, SeekOrigin.Begin); 119 | await _minioClient.PutObjectAsync( 120 | new PutObjectArgs() 121 | .WithBucket(_bucket) 122 | .WithObject(_fileId) 123 | .WithObjectSize(_inputData.Length) 124 | .WithStreamData(_inputData) 125 | .WithContentType("application/pdf"), 126 | _cancellation); 127 | 128 | result++; 129 | 130 | await _minioClient.StatObjectAsync( 131 | new StatObjectArgs() 132 | .WithBucket(_bucket) 133 | .WithObject(_fileId), 134 | _cancellation); 135 | 136 | result++; 137 | 138 | _outputData.Seek(0, SeekOrigin.Begin); 139 | await _minioClient.GetObjectAsync( 140 | new GetObjectArgs() 141 | .WithBucket(_bucket) 142 | .WithObject(_fileId) 143 | .WithCallbackStream((file, ct) => file.CopyToAsync(_outputData, ct)), 144 | _cancellation); 145 | 146 | result++; 147 | 148 | await _minioClient.RemoveObjectAsync( 149 | new RemoveObjectArgs() 150 | .WithBucket(_bucket) 151 | .WithObject(_fileId), 152 | _cancellation); 153 | 154 | return ++result; 155 | } 156 | 157 | [Benchmark(Baseline = true)] 158 | public async Task Storage() 159 | { 160 | var result = 0; 161 | 162 | var bucketExistsResult = await _s3Client.IsBucketExists(_cancellation); 163 | if (!bucketExistsResult) 164 | { 165 | ThrowException(); 166 | } 167 | 168 | result++; 169 | 170 | var fileExistsResult = await _s3Client.IsFileExists(_fileId, _cancellation); 171 | if (fileExistsResult) 172 | { 173 | ThrowException(); 174 | } 175 | 176 | result++; 177 | 178 | _inputData.Seek(0, SeekOrigin.Begin); 179 | var fileUploadResult = await _s3Client.UploadFile(_fileId, "application/pdf", _inputData, _cancellation); 180 | if (!fileUploadResult) 181 | { 182 | ThrowException(); 183 | } 184 | 185 | result++; 186 | 187 | fileExistsResult = await _s3Client.IsFileExists(_fileId, _cancellation); 188 | if (!fileExistsResult) 189 | { 190 | ThrowException(); 191 | } 192 | 193 | result++; 194 | 195 | _outputData.Seek(0, SeekOrigin.Begin); 196 | var storageFile = await _s3Client.GetFile(_fileId, _cancellation); 197 | if (!storageFile) 198 | { 199 | ThrowException(storageFile.ToString()); 200 | } 201 | 202 | var fileStream = await storageFile.GetStream(_cancellation); 203 | await fileStream.CopyToAsync(_outputData, _cancellation); 204 | 205 | storageFile.Dispose(); 206 | 207 | result++; 208 | 209 | await _s3Client.DeleteFile(_fileId, _cancellation); 210 | return ++result; 211 | } 212 | 213 | private static void ThrowException(string? message = null) 214 | { 215 | throw new Exception(message); 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | ## https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 4 | 5 | # User-specific files 6 | *.suo 7 | *.user 8 | *.userosscache 9 | *.sln.docstates 10 | 11 | # User-specific files (MonoDevelop/Xamarin Studio) 12 | *.userprefs 13 | 14 | # Build results 15 | [Dd]ebug/ 16 | [Dd]ebugPublic/ 17 | [Rr]elease/ 18 | [Rr]eleases/ 19 | x64/ 20 | x86/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | [Ll]og/ 25 | 26 | # Visual Studio 2015 cache/options directory 27 | .vs/ 28 | # Uncomment if you have tasks that create the project's static files in wwwroot 29 | #wwwroot/ 30 | 31 | # MSTest test Results 32 | [Tt]est[Rr]esult*/ 33 | [Bb]uild[Ll]og.* 34 | 35 | # NUNIT 36 | *.VisualState.xml 37 | TestResult.xml 38 | 39 | # Build Results of an ATL Project 40 | [Dd]ebugPS/ 41 | [Rr]eleasePS/ 42 | dlldata.c 43 | 44 | # DNX 45 | project.lock.json 46 | project.fragment.lock.json 47 | artifacts/ 48 | 49 | *_i.c 50 | *_p.c 51 | *_i.h 52 | *.ilk 53 | *.meta 54 | *.obj 55 | *.pch 56 | *.pdb 57 | *.pgc 58 | *.pgd 59 | *.rsp 60 | *.sbr 61 | *.tlb 62 | *.tli 63 | *.tlh 64 | *.tmp 65 | *.tmp_proj 66 | *.log 67 | *.vspscc 68 | *.vssscc 69 | .builds 70 | *.pidb 71 | *.svclog 72 | *.scc 73 | 74 | # Chutzpah Test files 75 | _Chutzpah* 76 | 77 | # Visual C++ cache files 78 | ipch/ 79 | *.aps 80 | *.ncb 81 | *.opendb 82 | *.opensdf 83 | *.sdf 84 | *.cachefile 85 | *.VC.db 86 | *.VC.VC.opendb 87 | 88 | # Visual Studio profiler 89 | *.psess 90 | *.vsp 91 | *.vspx 92 | *.sap 93 | 94 | # TFS 2012 Local Workspace 95 | $tf/ 96 | 97 | # Guidance Automation Toolkit 98 | *.gpState 99 | 100 | # ReSharper is a .NET coding add-in 101 | _ReSharper*/ 102 | *.[Rr]e[Ss]harper 103 | *.DotSettings.user 104 | *.DotSettings 105 | 106 | # JustCode is a .NET coding add-in 107 | .JustCode 108 | 109 | # TeamCity is a build add-in 110 | _TeamCity* 111 | 112 | # DotCover is a Code Coverage Tool 113 | *.dotCover 114 | 115 | # NCrunch 116 | _NCrunch_* 117 | .*crunch*.local.xml 118 | nCrunchTemp_* 119 | 120 | # MightyMoose 121 | *.mm.* 122 | AutoTest.Net/ 123 | 124 | # Web workbench (sass) 125 | .sass-cache/ 126 | 127 | # Installshield output folder 128 | [Ee]xpress/ 129 | 130 | # DocProject is a documentation generator add-in 131 | DocProject/buildhelp/ 132 | DocProject/Help/*.HxT 133 | DocProject/Help/*.HxC 134 | DocProject/Help/*.hhc 135 | DocProject/Help/*.hhk 136 | DocProject/Help/*.hhp 137 | DocProject/Help/Html2 138 | DocProject/Help/html 139 | 140 | # Click-Once directory 141 | publish/ 142 | 143 | # Publish Web Output 144 | *.[Pp]ublish.xml 145 | *.azurePubxml 146 | # but database connection strings (with potential passwords) will be unencrypted 147 | *.pubxml 148 | *.publishproj 149 | 150 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 151 | # checkin your Azure Web App publish settings, but sensitive information contained 152 | # in these scripts will be unencrypted 153 | PublishScripts/ 154 | 155 | # NuGet Packages 156 | *.nupkg 157 | # The packages folder can be ignored because of Package Restore 158 | **/packages/* 159 | # except build/, which is used as an MSBuild target. 160 | !**/packages/build/ 161 | # Uncomment if necessary however generally it will be regenerated when needed 162 | #!**/packages/repositories.config 163 | # NuGet v3's project.json files produces more ignoreable files 164 | *.nuget.props 165 | *.nuget.targets 166 | 167 | # Microsoft Azure Build Output 168 | csx/ 169 | *.build.csdef 170 | 171 | # Microsoft Azure Emulator 172 | ecf/ 173 | rcf/ 174 | 175 | # Windows Store app package directories and files 176 | AppPackages/ 177 | BundleArtifacts/ 178 | Package.StoreAssociation.xml 179 | _pkginfo.txt 180 | 181 | # Visual Studio cache files 182 | # files ending in .cache can be ignored 183 | *.[Cc]ache 184 | # but keep track of directories ending in .cache 185 | !*.[Cc]ache/ 186 | 187 | # Others 188 | ClientBin/ 189 | ~$* 190 | *~ 191 | *.dbmdl 192 | *.dbproj.schemaview 193 | *.pfx 194 | *.publishsettings 195 | node_modules/ 196 | orleans.codegen.cs 197 | 198 | # Since there are multiple workflows, uncomment next line to ignore bower_components 199 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 200 | #bower_components/ 201 | 202 | # RIA/Silverlight projects 203 | Generated_Code/ 204 | 205 | # Backup & report files from converting an old project file 206 | # to a newer Visual Studio version. Backup files are not needed, 207 | # because we have git ;-) 208 | _UpgradeReport_Files/ 209 | Backup*/ 210 | UpgradeLog*.XML 211 | UpgradeLog*.htm 212 | 213 | # SQL Server files 214 | *.mdf 215 | *.ldf 216 | 217 | # Business Intelligence projects 218 | *.rdl.data 219 | *.bim.layout 220 | *.bim_*.settings 221 | 222 | # Microsoft Fakes 223 | FakesAssemblies/ 224 | 225 | # GhostDoc plugin setting file 226 | *.GhostDoc.xml 227 | 228 | # Node.js Tools for Visual Studio 229 | .ntvs_analysis.dat 230 | 231 | # Visual Studio 6 build log 232 | *.plg 233 | 234 | # Visual Studio 6 workspace options file 235 | *.opt 236 | 237 | # Visual Studio LightSwitch build output 238 | **/*.HTMLClient/GeneratedArtifacts 239 | **/*.DesktopClient/GeneratedArtifacts 240 | **/*.DesktopClient/ModelManifest.xml 241 | **/*.Server/GeneratedArtifacts 242 | **/*.Server/ModelManifest.xml 243 | _Pvt_Extensions 244 | 245 | # Paket dependency manager 246 | .paket/paket.exe 247 | paket-files/ 248 | 249 | # FAKE - F# Make 250 | .fake/ 251 | 252 | # JetBrains Rider 253 | .idea/ 254 | *.sln.iml 255 | 256 | # CodeRush 257 | .cr/ 258 | 259 | # Python Tools for Visual Studio (PTVS) 260 | __pycache__/ 261 | *.pyc 262 | 263 | *.orig 264 | .sonarqube/ 265 | 266 | #Ignore thumbnails created by Windows 267 | Thumbs.db 268 | #Ignore files built by Visual Studio 269 | *.exe 270 | *.bak 271 | *.cache 272 | [Bb]in 273 | [Dd]ebug*/ 274 | obj/ 275 | [Rr]elease*/ 276 | [Tt]est[Rr]esult* 277 | #Nuget packages folder 278 | packages/ 279 | 280 | #Visual Studio Code 281 | .vscode/ 282 | 283 | # Custom ignore 284 | .DS_Store 285 | 286 | coverage.json -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![.NET Core](https://github.com/teoadal/Storage/actions/workflows/dotnet.yml/badge.svg?branch=master)](https://github.com/teoadal/Storage/actions/workflows/dotnet.yml) 2 | [![NuGet](https://img.shields.io/nuget/v/Storages3.svg)](https://www.nuget.org/packages/Storages3) 3 | [![NuGet](https://img.shields.io/nuget/dt/Storages3.svg)](https://www.nuget.org/packages/Storages3) 4 | [![codecov](https://codecov.io/gh/teoadal/Storage/branch/master/graph/badge.svg?token=8L4HN9FAIV)](https://codecov.io/gh/teoadal/Storage) 5 | [![CodeFactor](https://www.codefactor.io/repository/github/teoadal/storage/badge)](https://www.codefactor.io/repository/github/teoadal/storage) 6 | 7 | # Клиент для S3 8 | 9 | Привет! Это обертка над HttpClient для работы с S3 хранилищами. Мотивация создания была простейшей - я не понимал, 10 | почему клиенты [AWS](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/welcome.html) (4.0.0) 11 | и [Minio](https://github.com/minio/minio-dotnet) (6.0.4) потребляют так много памяти. Результат экспериментов: скорость 12 | почти как у AWS, а потребление памяти почти в 150 раз меньше, чем клиент для Minio (и в 17 для AWS). 13 | 14 | ```ini 15 | BenchmarkDotNet v0.14.0, Debian GNU/Linux 12 (bookworm) (container) 16 | Intel Xeon CPU E5-2697 v3 2.60GHz, 1 CPU, 56 logical and 28 physical cores 17 | .NET SDK 8.0.408 18 | [Host] : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 19 | .NET 8.0 : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX2 20 | 21 | Job=.NET 8.0 Runtime=.NET 8.0 22 | ``` 23 | | Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio | 24 | |-------- |--------:|---------:|---------:|------:|--------:|--------:|----------:|------------:| 25 | | Aws | 5.788 s | 0.0297 s | 0.0264 s | 1.05 | 0.02 | 1000 | 25.78 MB | 17.14 | 26 | | Minio | 7.016 s | 0.0824 s | 0.0730 s | 1.27 | 0.03 | - | 274.03 MB | 182.13 | 27 | | Storage | 5.510 s | 0.1063 s | 0.0994 s | 1.00 | 0.02 | - | 1.5 MB | 1.00 | 28 | 29 | 30 | ## Создание клиента 31 | 32 | Для работы с хранилищем необходимо создать клиент. 33 | 34 | ```csharp 35 | var storageClient = new S3Client(new S3Settings 36 | { 37 | AccessKey = "ROOTUSER", 38 | Bucket = "mybucket", 39 | EndPoint = "localhost", // для Yandex.Objects это "storage.yandexcloud.net" 40 | Port = 9000, // стандартный порт Minio - 9000, для Yandex.Objects указывать не нужно 41 | SecretKey = "ChangeMe123", 42 | UseHttps = false, // для Yandex.Objects укажите true 43 | UseHttp2 = false // Yandex.Objects позволяет работать по HTTP2, можете указать true 44 | }) 45 | ``` 46 | Также, в конструктор клиента можно передать имплементацию интерфейса `IArrayPool`, которая позволяет тонко настроить переиспользование массивов клиента. 47 | 48 | Minio предоставляет [playground](https://play.min.io:9443) для тестирования (порт для запросов всё тот же - 9000). Ключи 49 | можно найти [в документации](https://min.io/docs/minio/linux/developers/python/minio-py.html#file-uploader-py). Доступ к 50 | Amazon S3 не тестировался. 51 | 52 | ## Операции с S3 bucket 53 | 54 | ### Создание bucket'a 55 | 56 | Мы передаём название bucket'a в настройках, поэтому дополнительно его вводить не надо. 57 | 58 | ```csharp 59 | bool bucketCreateResult = await storageClient.CreateBucket(cancellationToken); 60 | Console.WriteLine(bucketCreateResult 61 | ? "Bucket создан" 62 | : "Bucket не был создан"); 63 | ``` 64 | 65 | ### Проверка существования bucket'a 66 | 67 | Как и в прошлый раз, мы знаем название bucket'a, так как мы передаём его в настройках клиента. 68 | 69 | ```csharp 70 | bool bucketCheckResult = await storageClient.IsBucketExists(cancellationToken); 71 | if (bucketCheckResult) Console.WriteLine("Bucket существует"); 72 | ``` 73 | 74 | ### Удаление bucket'a 75 | 76 | ```csharp 77 | bool bucketDeleteResult = await storageClient.DeleteBucket(cancellationToken); 78 | if (bucketDeleteResult) Console.WriteLine("Bucket удалён"); 79 | ``` 80 | 81 | ## Операции с S3 object 82 | 83 | Напомню, что объект в смысле S3 это и есть файл. 84 | 85 | ### Создание файла 86 | 87 | Создание, то есть загрузка файла в S3 хранилище, возможна двумя путями: можно разбить исходные данных на кусочки ( 88 | multipart), а можно не разбивать. Самый простой способ загрузки файла - воспользоваться следующим методом (если файл 89 | будет больше 5 МБ, то применяется multipart): 90 | 91 | ```csharp 92 | bool fileUploadResult = await storageClient.UploadFile(fileName, fileContentType, fileStream, cancellationToken); 93 | if (fileUploadResult) Console.WriteLine("Файл загружен"); 94 | ``` 95 | 96 | 97 | #### Управление Multipart-загрузкой 98 | 99 | Для самостоятельного управления multipart-загрузкой, можно воспользоваться методом `UploadFile` без указания данных. Получится примеоно такой код: 100 | 101 | ```csharp 102 | 103 | using S3Upload upload = await storageClient.UploadFile(fileName, fileType, cancellationToken); 104 | 105 | await upload.AddParts(stream, cancellationToken); // загружаем части документа 106 | if (!await upload.AddParts(byteArray, cancellationToken)) { // загружаем другую часть документа 107 | await upload.Abort(cancellationToken); // отменяем загрузку 108 | } 109 | else { 110 | await upload.Complete(cancellationToken); // завершаем загрузку 111 | } 112 | 113 | ``` 114 | 115 | В коде клиента именно эту логику использует метод PutFileMultipart. Конкретную реализацию можно подсмотреть в нём. 116 | 117 | ### Получение файла 118 | 119 | ```csharp 120 | StorageFile fileGetResult = await storageClient.GetFile(fileName, cancellationToken); 121 | if (fileGetResult) { 122 | Console.WriteLine($"Размер файла {fileGetResult.Length}, контент {fileGetResult.ContetType}"); 123 | return await fileGetResult.GetStream(cancellationToken); 124 | } 125 | else { 126 | Console.WriteLine($"Файл не может быть загружен, так как {fileGetResult}"); 127 | } 128 | ``` 129 | 130 | ### Получение файла как Stream 131 | 132 | ```csharp 133 | var fileStream = await storageClient.GetFileStream(fileName, cancellationToken); 134 | ``` 135 | 136 | В случае, если файл не существует, возвратится `Stream.Null`. 137 | 138 | ### Проверка существования файла 139 | 140 | ```csharp 141 | bool fileExistsResult = await storageClient.IsFileExists(fileName, cancellationToken); 142 | if (fileExistsResult) { 143 | Console.WriteLine("Файл существует"); 144 | } 145 | ``` 146 | 147 | ### Создание подписанной ссылки на файл 148 | 149 | Метод проверяет наличие файла в хранилище S3 и формирует GET запрос файла. Параметр `expiration` должен содержать время 150 | валидности ссылки начиная с даты формирования ссылки. 151 | 152 | ```csharp 153 | string? preSignedFileUrl = storageClient.GetFileUrl(fileName, expiration); 154 | if (preSignedFileUrl != null) { 155 | Console.WriteLine($"URL получен: {preSignedFileUrl}"); 156 | } 157 | ``` 158 | 159 | Существует не безопасный способ создать ссылку, без проверки наличия файла в S3. 160 | 161 | ```csharp 162 | string preSignedFileUrl = await storageClient.BuildFileUrl(fileName, expiration, cancellationToken); 163 | ``` 164 | 165 | ### Удаление 166 | 167 | Удаление объекта из S3 происходит почти мгновенно. На самом деле в S3 хранилище просто ставится задача на удаление и 168 | клиенту возвращается результат. Кстати, если удалить файл, который не существует, то ответ будет такой же, как если бы 169 | файл существовал. Поэтому этот метод ничего не возвращает. 170 | 171 | ```csharp 172 | await storageClient.DeleteFile(fileName, cancellationToken); 173 | Console.WriteLine("Файл удалён, если он, конечно, существовал"); 174 | ``` 175 | 176 | ## Измерение производительности и тестирование 177 | 178 | Локальное измерение производительности и тестирование осуществляется с помощью Minio в Docker'e по http. Понимаю, что 179 | это не самый хороший способ, но зато он самый доступный и простой. 180 | 181 | 1. Файл `docker-compose` для локального тестирования можно найти в репозитории. 182 | 2. Запускаем `docker-compose up -d`. Если всё хорошо, то бенчмарк заработает в Docker'e. 183 | 3. Если нужно запустить бенчмарк локально, то обращаем внимание на файл `appsettings.json`. В нём содержатся основные 184 | настройки для подключения к Minio. 185 | 4. Свойство `BigFilePath` файла `appsettings.json` сейчас не заполнено. Его можно использовать для загрузки реального 186 | файла (больше 100МБ). Если свойство не заполнено, то тест сгенерирует случайную последовательность байт размером 187 | 123МБ в памяти. 188 | 189 | ## Вопросы 190 | 191 | У меня есть канал в TG: [@csharp_gepard](https://t.me/csharp_gepard/91). К нему привязан чат - вопросы можно задавать в чате, либо в любом из последних постов. 192 | -------------------------------------------------------------------------------- /src/Storage/Utils/Signature.cs: -------------------------------------------------------------------------------- 1 | using System.Security.Cryptography; 2 | using System.Text; 3 | 4 | // ReSharper disable ReplaceWithPrimaryConstructorParameter 5 | 6 | namespace Storage.Utils; 7 | 8 | internal sealed class Signature(HttpDescription description, string secretKey, string region, string service) 9 | { 10 | public const string Iso8601DateTime = "yyyyMMddTHHmmssZ"; 11 | public const string Iso8601Date = "yyyyMMdd"; 12 | 13 | private static SortedDictionary? _headerSort = new(); 14 | 15 | private readonly IArrayPool _arrayPool = description.ArrayPool; 16 | private readonly HttpDescription _description = description; 17 | private readonly byte[] _secretKey = Encoding.UTF8.GetBytes($"AWS4{secretKey}"); 18 | private readonly string _scope = $"/{region}/{service}/aws4_request\n"; 19 | 20 | [SkipLocalsInit] 21 | public string Calculate( 22 | HttpRequestMessage request, 23 | string payloadHash, 24 | string[] signedHeaders, 25 | DateTime requestDate) 26 | { 27 | var builder = new ValueStringBuilder(stackalloc char[512], _arrayPool); 28 | 29 | AppendStringToSign(ref builder, requestDate); 30 | AppendCanonicalRequestHash(ref builder, request, signedHeaders, payloadHash); 31 | 32 | Span signature = stackalloc byte[32]; 33 | CreateSigningKey(ref signature, requestDate); 34 | 35 | signature = signature[..Sign(ref signature, signature, builder.AsReadonlySpan())]; 36 | builder.Dispose(); 37 | 38 | return HashHelper.ToHex(signature, _arrayPool); 39 | } 40 | 41 | [SkipLocalsInit] 42 | public string Calculate(string url, DateTime requestDate) 43 | { 44 | var builder = new ValueStringBuilder(stackalloc char[512], _arrayPool); 45 | 46 | AppendStringToSign(ref builder, requestDate); 47 | AppendCanonicalRequestHash(ref builder, url); 48 | 49 | Span signature = stackalloc byte[32]; 50 | CreateSigningKey(ref signature, requestDate); 51 | 52 | signature = signature[..Sign(ref signature, signature, builder.AsReadonlySpan())]; 53 | builder.Dispose(); 54 | 55 | return HashHelper.ToHex(signature, _arrayPool); 56 | } 57 | 58 | private void AppendCanonicalHeaders( 59 | scoped ref ValueStringBuilder builder, 60 | HttpRequestMessage request, 61 | string[] signedHeaders) 62 | { 63 | var sortedHeaders = Interlocked.Exchange(ref _headerSort, null) ?? new SortedDictionary(); 64 | foreach (var requestHeader in request.Headers) 65 | { 66 | var header = NormalizeHeader(requestHeader.Key); 67 | if (signedHeaders.Contains(header)) 68 | { 69 | sortedHeaders.Add(header, string.Join(' ', requestHeader.Value).Trim()); 70 | } 71 | } 72 | 73 | var content = request.Content; 74 | if (content != null) 75 | { 76 | foreach (var contentHeader in content.Headers) 77 | { 78 | var header = NormalizeHeader(contentHeader.Key); 79 | if (signedHeaders.Contains(header)) 80 | { 81 | sortedHeaders.Add(header, string.Join(' ', contentHeader.Value).Trim()); 82 | } 83 | } 84 | } 85 | 86 | foreach (var (header, value) in sortedHeaders) 87 | { 88 | builder.Append(header); 89 | builder.Append(':'); 90 | builder.Append(value); 91 | builder.Append('\n'); 92 | } 93 | 94 | sortedHeaders.Clear(); 95 | Interlocked.Exchange(ref _headerSort, sortedHeaders); 96 | } 97 | 98 | private void AppendCanonicalQueryParameters(scoped ref ValueStringBuilder builder, string? query) 99 | { 100 | if (string.IsNullOrEmpty(query) || query == "?") 101 | { 102 | return; 103 | } 104 | 105 | var scanIndex = 0; 106 | if (query[0] is '?') 107 | { 108 | scanIndex = 1; 109 | } 110 | 111 | var textLength = query.Length; 112 | var equalIndex = query.IndexOf('=', StringComparison.Ordinal); 113 | if (equalIndex is -1) 114 | { 115 | equalIndex = textLength; 116 | } 117 | 118 | while (scanIndex < textLength) 119 | { 120 | var delimiter = query.IndexOf('&', scanIndex); 121 | if (delimiter is -1) 122 | { 123 | delimiter = textLength; 124 | } 125 | 126 | if (equalIndex < delimiter) 127 | { 128 | while (scanIndex != equalIndex && char.IsWhiteSpace(query[scanIndex])) 129 | { 130 | ++scanIndex; 131 | } 132 | 133 | var name = UnescapeString(query.AsSpan(scanIndex, equalIndex - scanIndex)); 134 | _description.AppendEncodedName(ref builder, name); 135 | builder.Append('='); 136 | 137 | var value = UnescapeString(query.AsSpan(equalIndex + 1, delimiter - equalIndex - 1)); 138 | _description.AppendEncodedName(ref builder, value); 139 | builder.Append('&'); 140 | 141 | equalIndex = query.IndexOf('=', delimiter); 142 | if (equalIndex is -1) 143 | { 144 | equalIndex = textLength; 145 | } 146 | } 147 | else 148 | { 149 | if (delimiter > scanIndex) 150 | { 151 | _description.AppendEncodedName(ref builder, query.AsSpan(scanIndex, delimiter - scanIndex)); 152 | builder.Append('='); 153 | _description.AppendEncodedName(ref builder, string.Empty); 154 | builder.Append('&'); 155 | } 156 | } 157 | 158 | scanIndex = delimiter + 1; 159 | } 160 | 161 | builder.RemoveLast(); 162 | } 163 | 164 | [SkipLocalsInit] 165 | private void AppendCanonicalRequestHash( 166 | scoped ref ValueStringBuilder builder, 167 | HttpRequestMessage request, 168 | string[] signedHeaders, 169 | string payload) 170 | { 171 | var canonical = new ValueStringBuilder(stackalloc char[512], _arrayPool); 172 | var uri = request.RequestUri!; 173 | 174 | const char newLine = '\n'; 175 | 176 | canonical.Append(request.Method.Method); 177 | canonical.Append(newLine); 178 | canonical.Append(uri.AbsolutePath); 179 | canonical.Append(newLine); 180 | 181 | AppendCanonicalQueryParameters(ref canonical, uri.Query); 182 | canonical.Append(newLine); 183 | 184 | AppendCanonicalHeaders(ref canonical, request, signedHeaders); 185 | canonical.Append(newLine); 186 | 187 | var first = true; 188 | var span = signedHeaders.AsSpan(); 189 | for (var index = 0; index < span.Length; index++) 190 | { 191 | var header = span[index]; 192 | if (first) 193 | { 194 | first = false; 195 | } 196 | else 197 | { 198 | canonical.Append(';'); 199 | } 200 | 201 | canonical.Append(header); 202 | } 203 | 204 | canonical.Append(newLine); 205 | canonical.Append(payload); 206 | 207 | AppendSha256ToHex(ref builder, canonical.AsReadonlySpan()); 208 | 209 | canonical.Dispose(); 210 | } 211 | 212 | [SkipLocalsInit] 213 | private void AppendCanonicalRequestHash(scoped ref ValueStringBuilder builder, string url) 214 | { 215 | var uri = new Uri(url); 216 | 217 | var canonical = new ValueStringBuilder(stackalloc char[256], _arrayPool); 218 | canonical.Append("GET\n"); // canonical request 219 | canonical.Append(uri.AbsolutePath); 220 | canonical.Append('\n'); 221 | canonical.Append(uri.Query.AsSpan(1)); 222 | canonical.Append('\n'); 223 | canonical.Append("host:"); 224 | canonical.Append(uri.Host); 225 | 226 | if (!uri.IsDefaultPort) 227 | { 228 | canonical.Append(':'); 229 | canonical.Append(uri.Port); 230 | } 231 | 232 | canonical.Append("\n\n"); 233 | canonical.Append("host\n"); 234 | canonical.Append("UNSIGNED-PAYLOAD"); 235 | 236 | AppendSha256ToHex(ref builder, canonical.AsReadonlySpan()); 237 | 238 | canonical.Dispose(); 239 | } 240 | 241 | [SkipLocalsInit] 242 | private void AppendSha256ToHex(ref ValueStringBuilder builder, scoped ReadOnlySpan value) 243 | { 244 | var count = Encoding.UTF8.GetByteCount(value); 245 | 246 | var byteBuffer = _arrayPool.Rent(count); 247 | 248 | var encoded = Encoding.UTF8.GetBytes(value, byteBuffer); 249 | 250 | Span hashBuffer = stackalloc byte[32]; 251 | if (SHA256.TryHashData(byteBuffer.AsSpan(0, encoded), hashBuffer, out var written)) 252 | { 253 | Span buffer = stackalloc char[2]; 254 | for (var index = 0; index < hashBuffer[..written].Length; index++) 255 | { 256 | var element = hashBuffer[..written][index]; 257 | builder.Append(buffer[..StringUtils.FormatX2(ref buffer, element)]); 258 | } 259 | } 260 | 261 | _arrayPool.Return(byteBuffer); 262 | } 263 | 264 | [SkipLocalsInit] 265 | private string NormalizeHeader(string header) 266 | { 267 | using var builder = new ValueStringBuilder(stackalloc char[header.Length], _arrayPool); 268 | var culture = CultureInfo.InvariantCulture; 269 | 270 | // ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator 271 | var span = header.AsSpan(); 272 | foreach (var ch in span) 273 | { 274 | if (ch is ' ') continue; 275 | builder.Append(char.ToLower(ch, culture)); 276 | } 277 | 278 | return string.Intern(builder.Flush()); 279 | } 280 | 281 | [SuppressMessage("ReSharper", "InvertIf", Justification = "Approved")] 282 | private int Sign(ref Span buffer, ReadOnlySpan key, scoped ReadOnlySpan content) 283 | { 284 | var count = Encoding.UTF8.GetByteCount(content); 285 | 286 | var byteBuffer = _arrayPool.Rent(count); 287 | 288 | var encoded = Encoding.UTF8.GetBytes(content, byteBuffer); 289 | var result = HMACSHA256.TryHashData(key, byteBuffer.AsSpan(0, encoded), buffer, out var written) 290 | ? written 291 | : -1; 292 | 293 | _arrayPool.Return(byteBuffer); 294 | 295 | return result; 296 | } 297 | 298 | [SkipLocalsInit] 299 | private string UnescapeString(ReadOnlySpan query) 300 | { 301 | using var data = new ValueStringBuilder(stackalloc char[query.Length], _arrayPool); 302 | foreach (var ch in query) 303 | { 304 | data.Append(ch is '+' ? ' ' : ch); 305 | } 306 | 307 | return Uri.UnescapeDataString(data.Flush()); 308 | } 309 | 310 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 311 | private void AppendStringToSign(ref ValueStringBuilder builder, DateTime requestDate) 312 | { 313 | builder.Append("AWS4-HMAC-SHA256\n"); 314 | builder.Append(requestDate, Iso8601DateTime); 315 | builder.Append("\n"); 316 | builder.Append(requestDate, Iso8601Date); 317 | builder.Append(_scope); 318 | } 319 | 320 | [SkipLocalsInit] 321 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 322 | private void CreateSigningKey(ref Span buffer, DateTime requestDate) 323 | { 324 | Span dateBuffer = stackalloc char[16]; 325 | 326 | Sign(ref buffer, _secretKey, dateBuffer[..StringUtils.Format(ref dateBuffer, requestDate, Iso8601Date)]); 327 | Sign(ref buffer, buffer, region); 328 | Sign(ref buffer, buffer, service); 329 | Sign(ref buffer, buffer, "aws4_request"); 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /src/Storage/S3Client.cs: -------------------------------------------------------------------------------- 1 | using Storage.Utils; 2 | using static Storage.Utils.HashHelper; 3 | 4 | namespace Storage; 5 | 6 | /// 7 | /// Клиент для загрузки данных в S3 и их получения 8 | /// 9 | [DebuggerDisplay("Client for '{Bucket}'")] 10 | [SuppressMessage("ReSharper", "SwitchStatementHandlesSomeKnownEnumValuesWithDefault", Justification = "Approved")] 11 | [SuppressMessage("StyleCop.CSharp.MaintainabilityRules", "SA1401:Fields should be private", Justification = "Approved")] 12 | public sealed partial class S3Client 13 | { 14 | internal const int DefaultPartSize = 5 * 1024 * 1024; // 5 Mb 15 | 16 | private static readonly string[] S3Headers = // trimmed, lower invariant, ordered 17 | [ 18 | "host", 19 | "x-amz-content-sha256", 20 | "x-amz-date", 21 | ]; 22 | 23 | public readonly IArrayPool ArrayPool; 24 | public readonly string Bucket; 25 | 26 | private readonly string _bucket; 27 | private readonly HttpClient _client; 28 | private readonly string _endpoint; 29 | private readonly HttpDescription _httpDescription; 30 | private readonly Signature _signature; 31 | private readonly bool _useHttp2; 32 | 33 | private bool _disposed; 34 | 35 | public S3Client(S3Settings settings, HttpClient? client = null, IArrayPool? arrayProvider = null) 36 | { 37 | Bucket = settings.Bucket; 38 | 39 | var bucket = Bucket.ToLowerInvariant(); 40 | var scheme = settings.UseHttps ? Uri.UriSchemeHttps : Uri.UriSchemeHttp; 41 | var port = settings.Port.HasValue ? $":{settings.Port}" : string.Empty; 42 | 43 | ArrayPool = arrayProvider ?? DefaultArrayPool.Instance; 44 | _bucket = $"{scheme}://{settings.EndPoint}{port}/{bucket}"; 45 | _client = client ?? new HttpClient(); 46 | _endpoint = $"{settings.EndPoint}{port}"; 47 | _httpDescription = new HttpDescription( 48 | ArrayPool, 49 | settings.AccessKey, 50 | settings.Region, 51 | settings.Service, 52 | S3Headers); 53 | _signature = new Signature(_httpDescription, settings.SecretKey, settings.Region, settings.Service); 54 | _useHttp2 = settings.UseHttp2; 55 | } 56 | 57 | /// 58 | /// Создаёт подписанную ссылку на файл без проверки наличия файла на сервере 59 | /// 60 | /// Название файла 61 | /// Время жизни ссылки 62 | /// Возвращает подписанную ссылку на файл 63 | public string BuildFileUrl(string fileName, TimeSpan expiration) 64 | { 65 | var now = DateTime.UtcNow; 66 | var url = _httpDescription.BuildUrl(_bucket, fileName, now, expiration); 67 | var signature = _signature.Calculate(url, now); 68 | 69 | return $"{url}&X-Amz-Signature={signature}"; 70 | } 71 | 72 | public async Task DeleteFile(string fileName, CancellationToken ct) 73 | { 74 | HttpResponseMessage response; 75 | using (var request = CreateRequest(HttpMethod.Delete, fileName)) 76 | { 77 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 78 | } 79 | 80 | if (response.StatusCode is not HttpStatusCode.NoContent) 81 | { 82 | Errors.UnexpectedResult(response); 83 | } 84 | 85 | response.Dispose(); 86 | } 87 | 88 | public void Dispose() 89 | { 90 | if (_disposed) 91 | { 92 | return; 93 | } 94 | 95 | _client.Dispose(); 96 | 97 | _disposed = true; 98 | } 99 | 100 | /// 101 | /// Получает объектное представление файла на сервере 102 | /// 103 | /// Название файла 104 | /// Токен отмены операции 105 | /// Возвращает объектное представление файла на сервере 106 | public async Task GetFile(string fileName, CancellationToken ct) 107 | { 108 | HttpResponseMessage response; 109 | using (var request = CreateRequest(HttpMethod.Get, fileName)) 110 | { 111 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 112 | } 113 | 114 | switch (response.StatusCode) 115 | { 116 | case HttpStatusCode.OK: 117 | return new S3File(response); 118 | case HttpStatusCode.NotFound: 119 | response.Dispose(); 120 | return new S3File(response); 121 | default: 122 | Errors.UnexpectedResult(response); 123 | return new S3File(null!); // никогда не будет вызвано 124 | } 125 | } 126 | 127 | public async Task GetFileStream(string fileName, CancellationToken ct) 128 | { 129 | HttpResponseMessage response; 130 | using (var request = CreateRequest(HttpMethod.Get, fileName)) 131 | { 132 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 133 | } 134 | 135 | switch (response.StatusCode) 136 | { 137 | case HttpStatusCode.OK: 138 | return await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false); 139 | case HttpStatusCode.NotFound: 140 | response.Dispose(); 141 | return Stream.Null; 142 | default: 143 | Errors.UnexpectedResult(response); 144 | return Stream.Null; 145 | } 146 | } 147 | 148 | /// 149 | /// Создаёт подписанную ссылку на файл после проверки наличия файла на сервере 150 | /// 151 | /// Название файла 152 | /// Время жизни ссылки 153 | /// Токен отмены операции 154 | /// Возвращает подписанную ссылку на файл 155 | public async Task GetFileUrl(string fileName, TimeSpan expiration, CancellationToken ct) 156 | { 157 | return await IsFileExists(fileName, ct).ConfigureAwait(false) 158 | ? BuildFileUrl(fileName, expiration) 159 | : null; 160 | } 161 | 162 | public async Task IsFileExists(string fileName, CancellationToken ct) 163 | { 164 | HttpResponseMessage response; 165 | using (var request = CreateRequest(HttpMethod.Head, fileName)) 166 | { 167 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 168 | } 169 | 170 | switch (response.StatusCode) 171 | { 172 | case HttpStatusCode.OK: 173 | response.Dispose(); 174 | return true; 175 | case HttpStatusCode.NotFound: 176 | response.Dispose(); 177 | return false; 178 | default: 179 | Errors.UnexpectedResult(response); 180 | return false; 181 | } 182 | } 183 | 184 | public async IAsyncEnumerable List(string? prefix, [EnumeratorCancellation] CancellationToken ct) 185 | { 186 | var url = string.IsNullOrEmpty(prefix) 187 | ? $"{_bucket}?list-type=2" 188 | : $"{_bucket}?list-type=2&prefix={_httpDescription.EncodeName(prefix)}"; 189 | 190 | HttpResponseMessage response; 191 | using (var request = new HttpRequestMessage(HttpMethod.Get, url)) 192 | { 193 | response = await Send(request, EmptyPayloadHash, ct).ConfigureAwait(false); 194 | } 195 | 196 | if (response.StatusCode is HttpStatusCode.OK) 197 | { 198 | var responseStream = await response.Content 199 | .ReadAsStreamAsync(ct) 200 | .ConfigureAwait(false); 201 | 202 | while (responseStream.CanRead) 203 | { 204 | var readString = XmlStreamReader.ReadString(responseStream, "Key"); 205 | if (string.IsNullOrEmpty(readString)) 206 | { 207 | break; 208 | } 209 | 210 | yield return readString; 211 | } 212 | 213 | await responseStream.DisposeAsync().ConfigureAwait(false); 214 | response.Dispose(); 215 | 216 | yield break; 217 | } 218 | 219 | Errors.UnexpectedResult(response); 220 | } 221 | 222 | /// 223 | /// Загружает файл с ручным управлением загрузкой блоков файла 224 | /// 225 | /// Название файла 226 | /// Тип загружаемого файла 227 | /// Токен отмены операции 228 | /// Возвращает объект управления загрузкой 229 | public async Task UploadFile(string fileName, string contentType, CancellationToken ct) 230 | { 231 | var encodedFileName = _httpDescription.EncodeName(fileName); 232 | var uploadId = await MultipartStart(encodedFileName, contentType, ct).ConfigureAwait(false); 233 | 234 | return new S3Upload(this, fileName, encodedFileName, uploadId); 235 | } 236 | 237 | /// 238 | /// Загружает файл на сервер 239 | /// 240 | /// Название файла 241 | /// Тип загружаемого файла 242 | /// Данные файл 243 | /// Токен отмены операции 244 | /// Если файл превышает 5 МБ, то будет применена Multipart-загрузка 245 | /// Возвращает результат загрузки файла 246 | public Task UploadFile(string fileName, string contentType, Stream data, CancellationToken ct) 247 | { 248 | var length = data.TryGetLength(); 249 | 250 | return length is null or 0 or > DefaultPartSize 251 | ? ExecuteMultipartUpload(fileName, contentType, data, ct) 252 | : PutFile(fileName, contentType, data, ct); 253 | } 254 | 255 | /// 256 | /// Загружает файл на сервер 257 | /// 258 | /// Название файла 259 | /// Тип загружаемого файла 260 | /// Данные файл 261 | /// Токен отмены операции 262 | /// Если файл превышает 5 МБ, то будет применена Multipart-загрузка 263 | /// Возвращает результат загрузки файла 264 | public Task UploadFile(string fileName, string contentType, byte[] data, CancellationToken ct) 265 | { 266 | var length = data.Length; 267 | 268 | return length > DefaultPartSize 269 | ? ExecuteMultipartUpload(fileName, contentType, data, ct) 270 | : PutFile(fileName, contentType, data, data.Length, ct); 271 | } 272 | 273 | private async Task PutFile( 274 | string fileName, 275 | string contentType, 276 | Stream data, 277 | CancellationToken ct) 278 | { 279 | var buffer = ArrayPool.Rent((int)data.Length); // размер точно есть 280 | var dataSize = await data.ReadAsync(buffer, ct).ConfigureAwait(false); 281 | 282 | var payloadHash = GetPayloadHash(buffer.AsSpan(0, dataSize), ArrayPool); 283 | 284 | HttpResponseMessage response; 285 | using (var request = CreateRequest(HttpMethod.Put, fileName)) 286 | { 287 | using (var content = new ByteArrayContent(buffer, 0, dataSize)) 288 | { 289 | content.Headers.Add("content-type", contentType); 290 | request.Content = content; 291 | 292 | try 293 | { 294 | response = await Send(request, payloadHash, ct).ConfigureAwait(false); 295 | } 296 | finally 297 | { 298 | ArrayPool.Return(buffer); 299 | } 300 | } 301 | } 302 | 303 | if (response.StatusCode == HttpStatusCode.OK) 304 | { 305 | response.Dispose(); 306 | return true; 307 | } 308 | 309 | Errors.UnexpectedResult(response); 310 | return false; 311 | } 312 | 313 | private async Task PutFile( 314 | string fileName, 315 | string contentType, 316 | byte[] data, 317 | int length, 318 | CancellationToken ct) 319 | { 320 | var payloadHash = GetPayloadHash(data, ArrayPool); 321 | 322 | HttpResponseMessage response; 323 | using (var request = CreateRequest(HttpMethod.Put, fileName)) 324 | { 325 | using (var content = new ByteArrayContent(data, 0, length)) 326 | { 327 | content.Headers.Add("content-type", contentType); 328 | request.Content = content; 329 | 330 | response = await Send(request, payloadHash, ct).ConfigureAwait(false); 331 | } 332 | } 333 | 334 | if (response.StatusCode is HttpStatusCode.OK) 335 | { 336 | response.Dispose(); 337 | return true; 338 | } 339 | 340 | Errors.UnexpectedResult(response); 341 | return false; 342 | } 343 | } 344 | -------------------------------------------------------------------------------- /src/Storage.Tests/ObjectShould.cs: -------------------------------------------------------------------------------- 1 | using System.Net; 2 | using static Storage.Tests.StorageFixture; 3 | 4 | namespace Storage.Tests; 5 | 6 | public sealed class ObjectShould : IClassFixture 7 | { 8 | private readonly S3Client _client; 9 | private readonly CancellationToken _ct; 10 | private readonly StorageFixture _fixture; 11 | private readonly S3Client _notExistsBucketClient; // don't dispose it 12 | 13 | public ObjectShould(StorageFixture fixture) 14 | { 15 | _ct = CancellationToken.None; 16 | _client = fixture.S3Client; 17 | _fixture = fixture; 18 | 19 | _notExistsBucketClient = TestHelper.CloneClient(_fixture); 20 | } 21 | 22 | [Fact] 23 | public async Task AbortMultipart() 24 | { 25 | var fileName = _fixture.Create(); 26 | var data = GetByteArray(50 * 1024 * 1024); 27 | 28 | using var uploader = await _client.UploadFile(fileName, StreamContentType, _ct); 29 | 30 | var addResult = await uploader.AddPart(data, _ct); 31 | addResult 32 | .Should().BeTrue(); 33 | 34 | var abortResult = await uploader.Abort(_ct); 35 | abortResult 36 | .Should().BeTrue(); 37 | } 38 | 39 | [Fact] 40 | public async Task AllowParallelUploadMultipleFiles() 41 | { 42 | const int parallelization = 10; 43 | 44 | var file = _fixture.Create(); 45 | var tasks = new Task[parallelization]; 46 | for (var i = 0; i < parallelization; i++) 47 | { 48 | var fileData = GetByteStream(12 * 1024 * 1024); 49 | var fileName = $"{file}-{i}"; 50 | tasks[i] = Task.Run( 51 | async () => 52 | { 53 | await _client.UploadFile(fileName, StreamContentType, fileData, _ct); 54 | if (!await _client.IsFileExists(fileName, _ct)) 55 | { 56 | return false; 57 | } 58 | 59 | await _client.DeleteFile(fileName, _ct); 60 | return true; 61 | }, 62 | _ct); 63 | } 64 | 65 | await Task.WhenAll(tasks); 66 | 67 | foreach (var task in tasks) 68 | { 69 | task 70 | .IsCompletedSuccessfully 71 | .Should().BeTrue(); 72 | 73 | task 74 | .Result 75 | .Should().BeTrue(); 76 | 77 | task.Dispose(); 78 | } 79 | } 80 | 81 | [Fact] 82 | public void BuildUrl() 83 | { 84 | var fileName = _fixture.Create(); 85 | 86 | _client 87 | .Invoking(client => client.BuildFileUrl(fileName, TimeSpan.FromSeconds(100))) 88 | .Should().NotThrow() 89 | .Which.Should().NotBeNull(); 90 | } 91 | 92 | [Fact] 93 | public async Task BeExists() 94 | { 95 | var fileName = await CreateTestFile(); 96 | var fileExistsResult = await _client.IsFileExists(fileName, _ct); 97 | 98 | fileExistsResult 99 | .Should().BeTrue(); 100 | 101 | await DeleteTestFile(fileName); 102 | } 103 | 104 | [Fact] 105 | public async Task BeNotExists() 106 | { 107 | var fileExistsResult = await _client.IsFileExists(_fixture.Create(), _ct); 108 | 109 | fileExistsResult 110 | .Should().BeFalse(); 111 | } 112 | 113 | [Fact] 114 | public async Task DeleteFile() 115 | { 116 | var fileName = await CreateTestFile(); 117 | 118 | await _client 119 | .Invoking(client => client.DeleteFile(fileName, _ct)) 120 | .Should().NotThrowAsync(); 121 | } 122 | 123 | [Fact] 124 | public async Task DisposeFileStream() 125 | { 126 | var fileName = await CreateTestFile(); 127 | using var fileGetResult = await _client.GetFile(fileName, _ct); 128 | 129 | var fileStream = await fileGetResult.GetStream(_ct); 130 | await fileStream.DisposeAsync(); 131 | 132 | await DeleteTestFile(fileName); 133 | } 134 | 135 | [Fact] 136 | public async Task DisposeStorageFile() 137 | { 138 | var fileName = await CreateTestFile(); 139 | using var fileGetResult = await _client.GetFile(fileName, _ct); 140 | 141 | // ReSharper disable once DisposeOnUsingVariable 142 | fileGetResult.Dispose(); 143 | 144 | await DeleteTestFile(fileName); 145 | } 146 | 147 | [Fact] 148 | public async Task GetFileStream() 149 | { 150 | var fileName = await CreateTestFile(); 151 | 152 | var fileStream = await _client.GetFileStream(fileName, _ct); 153 | 154 | using var bufferStream = GetEmptyByteStream(); 155 | await fileStream.CopyToAsync(bufferStream, _ct); 156 | 157 | await EnsureFileSame(fileName, bufferStream.ToArray()); 158 | await DeleteTestFile(fileName); 159 | } 160 | 161 | [Fact] 162 | public async Task GetFileUrl() 163 | { 164 | var fileName = await CreateTestFile(); 165 | 166 | var url = await _client.GetFileUrl(fileName, TimeSpan.FromSeconds(600), _ct); 167 | 168 | url.Should().NotBeNull(); 169 | 170 | using var response = await _fixture.HttpClient.GetAsync(url, _ct); 171 | 172 | await DeleteTestFile(fileName); 173 | 174 | response 175 | .IsSuccessStatusCode 176 | .Should().BeTrue(response.ReasonPhrase); 177 | } 178 | 179 | [Fact] 180 | public async Task GetFileUrlWithCyrillicName() 181 | { 182 | var fileName = await CreateTestFile($"при(ве)+т_как23дела{Guid.NewGuid()}.pdf"); 183 | 184 | var url = await _client.GetFileUrl(fileName, TimeSpan.FromSeconds(600), _ct); 185 | 186 | url.Should().NotBeNull(); 187 | 188 | using var response = await _fixture.HttpClient.GetAsync(url, _ct); 189 | 190 | await DeleteTestFile(fileName); 191 | 192 | response 193 | .IsSuccessStatusCode 194 | .Should().BeTrue(response.ReasonPhrase); 195 | } 196 | 197 | [Fact] 198 | public async Task HasValidUploadInformation() 199 | { 200 | var fileName = _fixture.Create(); 201 | 202 | using var uploader = await _client.UploadFile(fileName, StreamContentType, _ct); 203 | 204 | uploader 205 | .FileName 206 | .Should().Be(fileName); 207 | 208 | uploader 209 | .UploadId 210 | .Should().NotBeEmpty(); 211 | 212 | uploader 213 | .Written 214 | .Should().Be(0); 215 | 216 | var partData = new byte[] { 1, 2, 3 }; 217 | await uploader.AddPart(partData, _ct); 218 | 219 | uploader 220 | .Written 221 | .Should().Be(partData.Length); 222 | 223 | await uploader.Abort(_ct); 224 | } 225 | 226 | [Fact] 227 | public async Task HasValidInformation() 228 | { 229 | const int length = 1 * 1024 * 1024; 230 | const string contentType = "video/mp4"; 231 | 232 | var fileName = await CreateTestFile(contentType: contentType); 233 | using var fileGetResult = await _client.GetFile(fileName, _ct); 234 | 235 | ((bool)fileGetResult).Should().BeTrue(); 236 | 237 | fileGetResult 238 | .ContentType 239 | .Should().Be(contentType); 240 | 241 | fileGetResult 242 | .Exists 243 | .Should().BeTrue(); 244 | 245 | fileGetResult 246 | .Length 247 | .Should().Be(length); 248 | 249 | fileGetResult 250 | .StatusCode 251 | .Should().Be(HttpStatusCode.OK); 252 | 253 | await DeleteTestFile(fileName); 254 | } 255 | 256 | [Fact] 257 | public async Task HasValidStreamInformation() 258 | { 259 | const int length = 1 * 1024 * 1024; 260 | var fileName = await CreateTestFile(size: length); 261 | using var fileGetResult = await _client.GetFile(fileName, _ct); 262 | 263 | var fileStream = await fileGetResult.GetStream(_ct); 264 | 265 | fileStream.CanRead.Should().BeTrue(); 266 | fileStream.CanSeek.Should().BeFalse(); 267 | fileStream.CanWrite.Should().BeFalse(); 268 | fileStream.Length.Should().Be(length); 269 | 270 | fileStream 271 | .Invoking(stream => stream.Position) 272 | .Should().Throw(); 273 | 274 | await fileStream.DisposeAsync(); 275 | await fileStream.DisposeAsync(); 276 | 277 | await DeleteTestFile(fileName); 278 | } 279 | 280 | [Fact] 281 | public async Task ListFiles() 282 | { 283 | const int count = 2; 284 | var noiseFiles = new List(); 285 | var expectedFileNames = new string[count]; 286 | var prefix = _fixture.Create(); 287 | 288 | for (var i = 0; i < count; i++) 289 | { 290 | var fileName = $"{prefix}#{_fixture.Create()}"; 291 | expectedFileNames[i] = await CreateTestFile(fileName, size: 1024); 292 | noiseFiles.Add(await CreateTestFile()); 293 | } 294 | 295 | var actualFileNames = new List(); 296 | await foreach (var file in _client.List(prefix, _ct)) 297 | { 298 | actualFileNames.Add(file); 299 | } 300 | 301 | actualFileNames 302 | .Should().Contain(expectedFileNames); 303 | 304 | foreach (var fileName in expectedFileNames.Concat(noiseFiles)) 305 | { 306 | await DeleteTestFile(fileName); 307 | } 308 | } 309 | 310 | [Fact] 311 | public async Task PutByteArray() 312 | { 313 | var fileName = _fixture.Create(); 314 | var data = GetByteArray(15000); 315 | var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 316 | 317 | filePutResult 318 | .Should().BeTrue(); 319 | 320 | await EnsureFileSame(fileName, data); 321 | await DeleteTestFile(fileName); 322 | } 323 | 324 | [Fact] 325 | public async Task PutBigByteArray() 326 | { 327 | var fileName = _fixture.Create(); 328 | var data = GetByteArray(50 * 1024 * 1024); 329 | var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 330 | 331 | filePutResult 332 | .Should().BeTrue(); 333 | 334 | await EnsureFileSame(fileName, data); 335 | await DeleteTestFile(fileName); 336 | } 337 | 338 | [Fact] 339 | public async Task PutStream() 340 | { 341 | var fileName = _fixture.Create(); 342 | var data = GetByteStream(15000); 343 | var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 344 | 345 | filePutResult 346 | .Should().BeTrue(); 347 | 348 | await EnsureFileSame(fileName, data); 349 | await DeleteTestFile(fileName); 350 | } 351 | 352 | [Fact] 353 | public async Task PutBigStream() 354 | { 355 | var fileName = _fixture.Create(); 356 | var data = GetByteStream(50 * 1024 * 1024); 357 | var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 358 | 359 | filePutResult 360 | .Should().BeTrue(); 361 | 362 | await EnsureFileSame(fileName, data); 363 | await DeleteTestFile(fileName); 364 | } 365 | 366 | [Fact] 367 | public async Task ReadFileStream() 368 | { 369 | var fileName = await CreateTestFile(); 370 | 371 | var buffer = new byte[1024]; 372 | var file = await _client.GetFile(fileName, _ct); 373 | var fileStream = await file.GetStream(_ct); 374 | 375 | #pragma warning disable CA1835 376 | var read = await fileStream.ReadAsync(buffer, _ct); 377 | read.Should().BeGreaterThan(0); 378 | 379 | read = await fileStream.ReadAsync(buffer, 10, 20, _ct); 380 | read.Should().BeGreaterThan(0); 381 | 382 | // ReSharper disable once MethodHasAsyncOverloadWithCancellation 383 | read = fileStream.Read(buffer, 10, 20); 384 | read.Should().BeGreaterThan(0); 385 | #pragma warning restore CA1835 386 | 387 | await DeleteTestFile(fileName); 388 | } 389 | 390 | [Fact] 391 | public async Task Upload() 392 | { 393 | var fileName = _fixture.Create(); 394 | using var data = GetByteStream(12 * 1024 * 1024); // 12 Mb 395 | var filePutResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 396 | 397 | filePutResult 398 | .Should().BeTrue(); 399 | 400 | await EnsureFileSame(fileName, data.ToArray()); 401 | await DeleteTestFile(fileName); 402 | } 403 | 404 | [Fact] 405 | public async Task UploadCyrillicName() 406 | { 407 | var fileName = $"при(ве)+т_как23дела{Guid.NewGuid()}.pdf"; 408 | using var data = GetByteStream(); 409 | var uploadResult = await _client.UploadFile(fileName, StreamContentType, data, _ct); 410 | 411 | await DeleteTestFile(fileName); 412 | 413 | uploadResult 414 | .Should().BeTrue(); 415 | } 416 | 417 | [Fact] 418 | public async Task NotThrowIfFileAlreadyExists() 419 | { 420 | var fileName = await CreateTestFile(); 421 | await _client 422 | .Invoking(client => client.UploadFile(fileName, StreamContentType, GetByteStream(), _ct)) 423 | .Should().NotThrowAsync(); 424 | 425 | await DeleteTestFile(fileName); 426 | } 427 | 428 | [Fact] 429 | public Task NotThrowIfFileExistsWithNotExistsBucket() 430 | { 431 | return _notExistsBucketClient 432 | .Invoking(client => client.IsFileExists(_fixture.Create(), _ct)) 433 | .Should().NotThrowAsync(); 434 | } 435 | 436 | [Fact] 437 | public async Task NotThrowIfFileGetUrlWithNotExistsBucket() 438 | { 439 | var result = await _notExistsBucketClient 440 | .Invoking(client => client.GetFileUrl(_fixture.Create(), TimeSpan.FromSeconds(100), _ct)) 441 | .Should().NotThrowAsync(); 442 | 443 | result 444 | .Which 445 | .Should().BeNull(); 446 | } 447 | 448 | [Fact] 449 | public async Task NotThrowIfFileGetWithNotExistsBucket() 450 | { 451 | var result = await _notExistsBucketClient 452 | .Invoking(client => client.GetFile(_fixture.Create(), _ct)) 453 | .Should().NotThrowAsync(); 454 | 455 | var getFileResult = result.Which; 456 | 457 | ((bool)getFileResult).Should().BeFalse(); 458 | 459 | getFileResult 460 | .Exists 461 | .Should().BeFalse(); 462 | } 463 | 464 | [Fact] 465 | public async Task NotThrowIfGetNotExistsFile() 466 | { 467 | var fileName = _fixture.Create(); 468 | await _client 469 | .Invoking(client => client.GetFile(fileName, _ct)) 470 | .Should().NotThrowAsync(); 471 | } 472 | 473 | [Fact] 474 | public Task NotThrowIfDeleteFileNotExists() 475 | { 476 | return _client 477 | .Invoking(client => client.DeleteFile(_fixture.Create(), _ct)) 478 | .Should().NotThrowAsync(); 479 | } 480 | 481 | [Fact] 482 | public async Task NotThrowIfGetFileStreamNotFound() 483 | { 484 | var fileName = _fixture.Create(); 485 | await _client 486 | .Invoking(client => client.GetFileStream(fileName, _ct)) 487 | .Should().NotThrowAsync(); 488 | } 489 | 490 | [Fact] 491 | public async Task ThrowIfBucketNotExists() 492 | { 493 | var fileArray = GetByteArray(); 494 | var fileName = _fixture.Create(); 495 | var fileStream = GetByteStream(); 496 | 497 | await _notExistsBucketClient 498 | .Invoking(client => client.DeleteFile(fileName, _ct)) 499 | .Should().ThrowAsync(); 500 | 501 | await _notExistsBucketClient 502 | .Invoking(client => client.UploadFile(fileName, StreamContentType, fileStream, _ct)) 503 | .Should().ThrowAsync(); 504 | 505 | await _notExistsBucketClient 506 | .Invoking(client => client.UploadFile(fileName, StreamContentType, fileArray, _ct)) 507 | .Should().ThrowAsync(); 508 | } 509 | 510 | [Theory] 511 | [InlineData("some/foo/audio.wav", 1024)] 512 | [InlineData("another/path/test.mp3", 2048)] 513 | public async Task UploadFileWithNestedPath(string nestedFileName, int dataSize) 514 | { 515 | var data = GetByteArray(dataSize); // Пример данных, которые вы хотите загрузить 516 | 517 | // Act 518 | var result = await _client.UploadFile(nestedFileName, StreamContentType, data, _ct); 519 | Assert.True(result); 520 | 521 | // Assert 522 | var exists = await _client.IsFileExists(nestedFileName, _ct); 523 | Assert.True(exists, "The object should exist in the S3 bucket"); 524 | 525 | await DeleteTestFile(nestedFileName); 526 | } 527 | 528 | 529 | [Fact] 530 | public async Task ThrowIfUploadDisposed() 531 | { 532 | var fileName = _fixture.Create(); 533 | 534 | var uploader = await _client.UploadFile(fileName, StreamContentType, _ct); 535 | uploader.Dispose(); 536 | 537 | await uploader 538 | .Invoking(u => u.Abort(_ct)) 539 | .Should().ThrowExactlyAsync(); 540 | 541 | await uploader 542 | .Invoking(u => u.AddPart([], 0, _ct)) 543 | .Should().ThrowExactlyAsync(); 544 | 545 | await uploader 546 | .Invoking(u => u.Complete(_ct)) 547 | .Should().ThrowExactlyAsync(); 548 | } 549 | 550 | private async Task CreateTestFile( 551 | string? fileName = null, 552 | string contentType = StreamContentType, 553 | int? size = null) 554 | { 555 | fileName ??= _fixture.Create(); 556 | using var data = GetByteStream(size ?? 1 * 1024 * 1024); // 1 Mb 557 | 558 | var uploadResult = await _client.UploadFile(fileName, contentType, data, _ct); 559 | 560 | uploadResult 561 | .Should().BeTrue(); 562 | 563 | return fileName; 564 | } 565 | 566 | private async Task EnsureFileSame(string fileName, byte[] expectedBytes) 567 | { 568 | using var getFileResult = await _client.GetFile(fileName, _ct); 569 | 570 | using var memoryStream = GetEmptyByteStream(getFileResult.Length); 571 | var stream = await getFileResult.GetStream(_ct); 572 | await stream.CopyToAsync(memoryStream, _ct); 573 | 574 | memoryStream 575 | .ToArray().SequenceEqual(expectedBytes) 576 | .Should().BeTrue(); 577 | } 578 | 579 | private async Task EnsureFileSame(string fileName, MemoryStream expectedBytes) 580 | { 581 | expectedBytes.Seek(0, SeekOrigin.Begin); 582 | 583 | using var getFileResult = await _client.GetFile(fileName, _ct); 584 | 585 | using var memoryStream = GetEmptyByteStream(getFileResult.Length); 586 | var stream = await getFileResult.GetStream(_ct); 587 | await stream.CopyToAsync(memoryStream, _ct); 588 | 589 | memoryStream 590 | .ToArray().SequenceEqual(expectedBytes.ToArray()) 591 | .Should().BeTrue(); 592 | } 593 | 594 | private Task DeleteTestFile(string fileName) 595 | { 596 | return _client.DeleteFile(fileName, _ct); 597 | } 598 | } 599 | --------------------------------------------------------------------------------