├── Snappier.snk ├── images ├── icon.png └── icon-48.png ├── AUTHORS ├── global.json ├── Snappier.Tests ├── TestData │ ├── alice29.snappy │ ├── fireworks.jpeg │ ├── geo.protodata │ ├── paper-100k.pdf │ ├── baddata1.snappy │ ├── baddata2.snappy │ ├── baddata3.snappy │ └── html_x_4.snappy ├── Internal │ ├── Crc32CAlgorithmTests.cs │ ├── SnappyStreamCompressorTests.cs │ ├── VarIntEncodingWriteTests.cs │ ├── VarIntEncodingReadTests.cs │ ├── SnappyCompressorTests.cs │ └── SnappyDecompressorTests.cs ├── SequenceHelpers.cs ├── HelpersTests.cs ├── Snappier.Tests.csproj └── SnappyStreamTests.cs ├── GitVersion.yml ├── docs ├── toc.yml ├── getting-started.md ├── stream.md └── block.md ├── toc.yml ├── Directory.Build.props ├── Snappier.Benchmarks ├── Configuration │ ├── BasicConfig.cs │ ├── JobExtensions.cs │ ├── StandardConfig.cs │ ├── FrameworkCompareConfig.cs │ ├── X86X64Config.cs │ ├── PgoColumn.cs │ └── VersionComparisonConfig.cs ├── Log2FloorHelper.cs ├── LeftShiftOverflows.cs ├── VarIntEncodingWrite.cs ├── Crc32CAlgorithm.cs ├── GetHashTable.cs ├── UnalignedCopy64.cs ├── UnalignedCopy128.cs ├── VarIntEncodingRead.cs ├── IncrementalCopy.cs ├── BlockCompressHtml.cs ├── CompressHtml.cs ├── DecompressHtml.cs ├── BlockDecompressHtml.cs ├── CompressAll.cs ├── Program.cs ├── DecompressAll.cs ├── Snappier.Benchmarks.csproj ├── FindMatchLength.cs └── Overview.cs ├── Snappier.slnx ├── Snappier ├── Internal │ ├── IsExternalInit.cs │ ├── CallerArgumentExpressionAttribute.cs │ ├── DebugExtensions.cs │ ├── ByteArrayPoolMemoryOwner.cs │ ├── VarIntEncoding.Write.cs │ ├── ThrowHelper.cs │ ├── UnsafeReadonly.cs │ ├── Constants.cs │ ├── VarIntEncoding.Read.cs │ ├── HashTable.cs │ ├── Crc32CAlgorithm.cs │ ├── VarIntEncoding.WriteFast.cs │ ├── Helpers.cs │ ├── NullableAttributes.cs │ ├── SnappyStreamCompressor.cs │ ├── SnappyStreamDecompressor.cs │ └── CopyHelpers.cs ├── Properties │ └── AssemblyInfo.cs ├── Snappier.csproj └── Snappy.cs ├── .config └── dotnet-tools.json ├── .gitignore ├── nuget.config ├── Snappier.slnx.DotSettings ├── SECURITY.md ├── .github ├── FUNDING.yml └── workflows │ ├── pages.yml │ ├── codeql-analysis.yml │ └── main.yml ├── docfx.json ├── CONTRIBUTING.md ├── COPYING.txt ├── index.md ├── README.md └── .editorconfig /Snappier.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.snk -------------------------------------------------------------------------------- /images/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/images/icon.png -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | opensource@google.com 2 | bburnett@centeredgesoftware.com 3 | info@couchbase.com 4 | -------------------------------------------------------------------------------- /images/icon-48.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/images/icon-48.png -------------------------------------------------------------------------------- /global.json: -------------------------------------------------------------------------------- 1 | { 2 | "sdk": { 3 | "version": "10.0.100", 4 | "rollForward": "latestFeature" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /Snappier.Tests/TestData/alice29.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/alice29.snappy -------------------------------------------------------------------------------- /Snappier.Tests/TestData/fireworks.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/fireworks.jpeg -------------------------------------------------------------------------------- /Snappier.Tests/TestData/geo.protodata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/geo.protodata -------------------------------------------------------------------------------- /Snappier.Tests/TestData/paper-100k.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/paper-100k.pdf -------------------------------------------------------------------------------- /Snappier.Tests/TestData/baddata1.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/baddata1.snappy -------------------------------------------------------------------------------- /Snappier.Tests/TestData/baddata2.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/baddata2.snappy -------------------------------------------------------------------------------- /Snappier.Tests/TestData/baddata3.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/baddata3.snappy -------------------------------------------------------------------------------- /Snappier.Tests/TestData/html_x_4.snappy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/brantburnett/Snappier/HEAD/Snappier.Tests/TestData/html_x_4.snappy -------------------------------------------------------------------------------- /GitVersion.yml: -------------------------------------------------------------------------------- 1 | workflow: GitHubFlow/v1 2 | tag-prefix: release/[vV]? 3 | mode: ManualDeployment 4 | branches: 5 | main: 6 | label: beta 7 | -------------------------------------------------------------------------------- /docs/toc.yml: -------------------------------------------------------------------------------- 1 | - name: Getting Started 2 | href: getting-started.md 3 | - name: Block Compression 4 | href: block.md 5 | - name: Stream Compression 6 | href: stream.md 7 | -------------------------------------------------------------------------------- /toc.yml: -------------------------------------------------------------------------------- 1 | - name: Documentation 2 | href: docs/ 3 | - name: API 4 | href: api/ 5 | - name: Contributing 6 | href: CONTRIBUTING.md 7 | - name: GitHub 8 | href: https://github.com/brantburnett/Snappier 9 | -------------------------------------------------------------------------------- /Directory.Build.props: -------------------------------------------------------------------------------- 1 | 2 | 3 | 14 4 | false 5 | true 6 | 7 | 8 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/BasicConfig.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Jobs; 2 | 3 | namespace Snappier.Benchmarks.Configuration; 4 | 5 | public class BasicConfig : StandardConfig 6 | { 7 | public BasicConfig(Job baseJob) 8 | { 9 | AddJob(baseJob); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Log2FloorHelper.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | [DisassemblyDiagnoser(2)] 4 | public class Log2FloorHelper 5 | { 6 | private uint _n = 5; 7 | 8 | [Benchmark] 9 | public int Log2Floor() 10 | { 11 | return Helpers.Log2Floor(_n); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | ## Installing 4 | 5 | Simply add a NuGet package reference to the latest version of Snappier. 6 | 7 | ```xml 8 | 9 | ``` 10 | 11 | or 12 | 13 | ```sh 14 | dotnet add package Snappier 15 | ``` 16 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/LeftShiftOverflows.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class LeftShiftOverflows 4 | { 5 | private byte _value = 24; 6 | private int _shift = 7; 7 | 8 | [Benchmark(Baseline = true)] 9 | public bool Current() 10 | { 11 | return Helpers.LeftShiftOverflows(_value, _shift); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/JobExtensions.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks.Configuration; 2 | 3 | public static class JobExtensions 4 | { 5 | extension(Job job) 6 | { 7 | public Job WithPgo(bool enabled = true) => 8 | job.WithEnvironmentVariable(PgoColumn.PgoEnvironmentVariableName, enabled ? "1" : "0"); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Snappier.slnx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/VarIntEncodingWrite.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class VarIntEncodingWrite 4 | { 5 | [Params(0u, 256u, 65536u)] 6 | public uint Value { get; set; } 7 | 8 | readonly byte[] _dest = new byte[8]; 9 | 10 | [Benchmark(Baseline = true)] 11 | public bool Baseline() 12 | { 13 | return VarIntEncoding.TryWrite(_dest, Value, out _); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Snappier/Internal/IsExternalInit.cs: -------------------------------------------------------------------------------- 1 | #if !NET8_0_OR_GREATER 2 | 3 | // Licensed to the .NET Foundation under one or more agreements. 4 | // The .NET Foundation licenses this file to you under the MIT license. 5 | 6 | #pragma warning disable IDE0079 7 | #pragma warning disable S3903 8 | 9 | namespace System.Runtime.CompilerServices; 10 | 11 | internal static class IsExternalInit 12 | { 13 | } 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /.config/dotnet-tools.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "isRoot": true, 4 | "tools": { 5 | "dotnet-reportgenerator-globaltool": { 6 | "version": "5.4.1", 7 | "commands": [ 8 | "reportgenerator" 9 | ], 10 | "rollForward": false 11 | }, 12 | "docfx": { 13 | "version": "2.78.3", 14 | "commands": [ 15 | "docfx" 16 | ], 17 | "rollForward": false 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | [Oo]bj/ 2 | [Bb]in/ 3 | artifacts/ 4 | packages/ 5 | *.suo 6 | *.user 7 | /TestResults 8 | *.vspscc 9 | *.vssscc 10 | *.vsmdi 11 | *.vs* 12 | *.testsettings 13 | *.sln.docstates 14 | couchbase-net-client.userprefs 15 | .idea 16 | TestResult.xml 17 | project.lock.json 18 | *.exe 19 | local.config 20 | *.lock.json 21 | Couchbase.snk 22 | *.nupkg 23 | BenchmarkDotNet.Artifacts/ 24 | test-results/ 25 | TestResults/ 26 | .DS_Store 27 | /api/ -------------------------------------------------------------------------------- /Snappier.Benchmarks/Crc32CAlgorithm.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class Crc32CAlgorithm 4 | { 5 | private byte[] _buffer; 6 | 7 | [GlobalSetup] 8 | public void Setup() 9 | { 10 | _buffer = new byte[65536]; 11 | new Random().NextBytes(_buffer); 12 | } 13 | 14 | [Benchmark] 15 | public uint Default() 16 | { 17 | return Internal.Crc32CAlgorithm.Append(0, _buffer); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/GetHashTable.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class HashTable 4 | { 5 | private Snappier.Internal.HashTable _hashTable = new(); 6 | 7 | [GlobalSetup] 8 | public void GlobalSetup() 9 | { 10 | _hashTable = new Snappier.Internal.HashTable(); 11 | _hashTable.EnsureCapacity(65536); 12 | } 13 | 14 | [Benchmark] 15 | public Span GetHashTable() 16 | { 17 | return _hashTable.GetHashTable(65536); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/UnalignedCopy64.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using BenchmarkDotNet.Diagnostics.Windows.Configs; 3 | 4 | namespace Snappier.Benchmarks; 5 | 6 | [RyuJitX64Job] 7 | [InliningDiagnoser(false, ["Snappier.Benchmarks"])] 8 | public class UnalignedCopy64 9 | { 10 | private readonly byte[] _buffer = new byte[16]; 11 | 12 | [Benchmark] 13 | public void Default() 14 | { 15 | ref byte ptr = ref _buffer[0]; 16 | CopyHelpers.UnalignedCopy64(in ptr, ref Unsafe.Add(ref ptr, 8)); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/UnalignedCopy128.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using BenchmarkDotNet.Diagnostics.Windows.Configs; 3 | 4 | namespace Snappier.Benchmarks; 5 | 6 | [RyuJitX64Job] 7 | [InliningDiagnoser(false, ["Snappier.Benchmarks"])] 8 | public class UnalignedCopy128 9 | { 10 | private readonly byte[] _buffer = new byte[32]; 11 | 12 | [Benchmark] 13 | public void Default() 14 | { 15 | ref byte ptr = ref _buffer[0]; 16 | CopyHelpers.UnalignedCopy128(in ptr, ref Unsafe.Add(ref ptr,16)); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /nuget.config: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/VarIntEncodingRead.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class VarIntEncodingRead 4 | { 5 | [Params(0u, 256u, 65536u)] 6 | public uint Value { get; set; } 7 | 8 | readonly byte[] _source = new byte[16]; 9 | 10 | [GlobalSetup] 11 | public void GlobalSetup() 12 | { 13 | VarIntEncoding.TryWrite(_source, Value, out _); 14 | } 15 | 16 | [Benchmark] 17 | public (int, uint) TryRead() 18 | { 19 | _ = VarIntEncoding.TryRead(_source, out uint result, out int length); 20 | 21 | return (length, result); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Snappier.slnx.DotSettings: -------------------------------------------------------------------------------- 1 | 2 | True 3 | True 4 | True -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | | ------- | ------------------ | 7 | | 1.2.x | :white_check_mark: | 8 | | 1.1.x | :white_check_mark: | 9 | | 1.0.x | :x: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | Vulnerabilities may be reported to bburnett@centeredgesoftware.com. Please do not file vulnerabilities as issues, as doing so is public. 14 | The report will generally not be made public until it is confirmed and resolved. Once reported, you will receive an email response 15 | within a few days. The resolution, if any, will typically be available within a few days as well. 16 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/StandardConfig.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Columns; 2 | using BenchmarkDotNet.Configs; 3 | using BenchmarkDotNet.Exporters; 4 | using BenchmarkDotNet.Loggers; 5 | 6 | namespace Snappier.Benchmarks.Configuration; 7 | 8 | public class StandardConfig : ManualConfig 9 | { 10 | public StandardConfig() 11 | { 12 | AddColumnProvider(DefaultColumnProviders.Instance); 13 | AddColumn(RankColumn.Arabic); 14 | 15 | AddExporter(DefaultExporters.CsvMeasurements); 16 | AddExporter(DefaultExporters.Csv); 17 | AddExporter(DefaultExporters.Markdown); 18 | AddExporter(DefaultExporters.Html); 19 | 20 | AddLogger(ConsoleLogger.Default); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Snappier/Internal/CallerArgumentExpressionAttribute.cs: -------------------------------------------------------------------------------- 1 | #if !NET8_0_OR_GREATER 2 | // Licensed to the .NET Foundation under one or more agreements. 3 | // The .NET Foundation licenses this file to you under the MIT license. 4 | 5 | // ReSharper disable once CheckNamespace 6 | namespace System.Runtime.CompilerServices 7 | { 8 | [AttributeUsage(AttributeTargets.Parameter, AllowMultiple = false, Inherited = false)] 9 | internal sealed class CallerArgumentExpressionAttribute : Attribute 10 | { 11 | public CallerArgumentExpressionAttribute(string parameterName) 12 | { 13 | ParameterName = parameterName; 14 | } 15 | 16 | public string ParameterName { get; } 17 | } 18 | } 19 | #endif 20 | -------------------------------------------------------------------------------- /Snappier.Tests/Internal/Crc32CAlgorithmTests.cs: -------------------------------------------------------------------------------- 1 | using System.Text; 2 | 3 | namespace Snappier.Tests.Internal; 4 | 5 | public class Crc32CAlgorithmTests 6 | { 7 | [Theory] 8 | [InlineData("123456789", 0xe3069283)] 9 | [InlineData("1234567890123456", 0x9aa4287f)] 10 | [InlineData("123456789012345612345678901234", 0xecc74934)] 11 | [InlineData("12345678901234561234567890123456", 0xcd486b4b)] 12 | public void Compute(string asciiChars, uint expectedResult) 13 | { 14 | // Arrange 15 | 16 | byte[] bytes = Encoding.ASCII.GetBytes(asciiChars); 17 | 18 | // Act 19 | 20 | uint result = Crc32CAlgorithm.Compute(bytes); 21 | 22 | // Assert 23 | 24 | Assert.Equal(expectedResult, result); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/IncrementalCopy.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class IncrementalCopy 6 | { 7 | private readonly byte[] _buffer = new byte[128]; 8 | 9 | [Benchmark] 10 | public void Fast() 11 | { 12 | ref byte buffer = ref _buffer[0]; 13 | CopyHelpers.IncrementalCopy(in buffer, ref Unsafe.Add(ref buffer, 2), ref Unsafe.Add(ref buffer, 18), 14 | ref Unsafe.Add(ref buffer, _buffer.Length - 1)); 15 | } 16 | 17 | [Benchmark(Baseline = true)] 18 | public void SlowCopyMemory() 19 | { 20 | ref byte buffer = ref _buffer[0]; 21 | CopyHelpers.IncrementalCopySlow(in buffer, ref Unsafe.Add(ref buffer, 2), ref Unsafe.Add(ref buffer, 18)); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Snappier/Properties/AssemblyInfo.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | [assembly: InternalsVisibleTo("Snappier.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100855bc08ed6537cdde1801f2696977e9101c5129e1941bc2827c8ba835b0e66f22eb4ad5a7de447ac972c0452c0c49b1668d869562e7b37c3bf8db97326008c5e1cd797b55d9b05e8e61156fd5943c58c8d970c52073a7ba82329cd91485866e6d7ad56e019499789316a3b34ebb9e13cdf3efc01ebd76c4992b7bf5ab402abc8")] 4 | [assembly: InternalsVisibleTo("Snappier.Benchmarks, PublicKey=0024000004800000940000000602000000240000525341310004000001000100855bc08ed6537cdde1801f2696977e9101c5129e1941bc2827c8ba835b0e66f22eb4ad5a7de447ac972c0452c0c49b1668d869562e7b37c3bf8db97326008c5e1cd797b55d9b05e8e61156fd5943c58c8d970c52073a7ba82329cd91485866e6d7ad56e019499789316a3b34ebb9e13cdf3efc01ebd76c4992b7bf5ab402abc8")] -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [brantburnett] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: brantburnett 14 | thanks_dev: # Replace with a single thanks.dev username 15 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 16 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/BlockCompressHtml.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class BlockCompressHtml 4 | { 5 | private ReadOnlyMemory _input; 6 | private Memory _output; 7 | 8 | [GlobalSetup] 9 | public void LoadToMemory() 10 | { 11 | using Stream resource = 12 | typeof(BlockCompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html"); 13 | 14 | byte[] input = new byte[65536]; // Just test the first 64KB 15 | int inputLength = resource!.Read(input, 0, input.Length); 16 | _input = input.AsMemory(0, inputLength); 17 | 18 | _output = new byte[Snappy.GetMaxCompressedLength(inputLength)]; 19 | } 20 | 21 | [Benchmark] 22 | public int Compress() 23 | { 24 | using var compressor = new SnappyCompressor(); 25 | 26 | #pragma warning disable CS0618 // Type or member is obsolete 27 | return compressor.Compress(_input.Span, _output.Span); 28 | #pragma warning restore CS0618 // Type or member is obsolete 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/CompressHtml.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class CompressHtml 6 | { 7 | private MemoryStream _source; 8 | private MemoryStream _destination; 9 | 10 | [Params(16384)] 11 | public int ReadSize; 12 | 13 | [GlobalSetup] 14 | public void LoadToMemory() 15 | { 16 | _source = new MemoryStream(); 17 | _destination = new MemoryStream(); 18 | 19 | using Stream resource = 20 | typeof(DecompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html_x_4"); 21 | 22 | // ReSharper disable once PossibleNullReferenceException 23 | resource.CopyTo(_source); 24 | } 25 | 26 | [Benchmark] 27 | public void Compress() 28 | { 29 | _source.Position = 0; 30 | _destination.Position = 0; 31 | using var stream = new SnappyStream(_destination, CompressionMode.Compress, true); 32 | 33 | _source.CopyTo(stream, ReadSize); 34 | stream.Flush(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/DecompressHtml.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class DecompressHtml 6 | { 7 | private MemoryStream _memoryStream; 8 | private byte[] _buffer; 9 | 10 | [Params(16384)] 11 | public int ReadSize; 12 | 13 | [GlobalSetup] 14 | public void LoadToMemory() 15 | { 16 | _memoryStream = new MemoryStream(); 17 | 18 | using Stream resource = 19 | typeof(DecompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html_x_4.snappy"); 20 | 21 | // ReSharper disable once PossibleNullReferenceException 22 | resource.CopyTo(_memoryStream); 23 | 24 | _buffer = new byte[ReadSize]; 25 | } 26 | 27 | [Benchmark] 28 | public void Decompress() 29 | { 30 | _memoryStream.Position = 0; 31 | using var stream = new SnappyStream(_memoryStream, CompressionMode.Decompress, true); 32 | 33 | while (stream.Read(_buffer, 0, ReadSize) > 0) 34 | { 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/BlockDecompressHtml.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Benchmarks; 2 | 3 | public class BlockDecompressHtml 4 | { 5 | private ReadOnlyMemory _input; 6 | 7 | [GlobalSetup] 8 | public void LoadToMemory() 9 | { 10 | using Stream resource = 11 | typeof(DecompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html"); 12 | 13 | byte[] input = new byte[65536]; // Just test the first 64KB 14 | // ReSharper disable once PossibleNullReferenceException 15 | int inputLength = resource!.Read(input, 0, input.Length); 16 | 17 | byte[] compressed = new byte[Snappy.GetMaxCompressedLength(inputLength)]; 18 | int compressedLength = Snappy.Compress(input.AsSpan(0, inputLength), compressed); 19 | 20 | _input = compressed.AsMemory(0, compressedLength); 21 | } 22 | 23 | [Benchmark] 24 | public void Decompress() 25 | { 26 | using var decompressor = new SnappyDecompressor(); 27 | 28 | decompressor.Decompress(_input.Span); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/FrameworkCompareConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using BenchmarkDotNet.Columns; 3 | using BenchmarkDotNet.Configs; 4 | using BenchmarkDotNet.Environments; 5 | using BenchmarkDotNet.Jobs; 6 | 7 | namespace Snappier.Benchmarks.Configuration; 8 | 9 | public class FrameworkCompareConfig : StandardConfig 10 | { 11 | public FrameworkCompareConfig(Job baseJob) 12 | { 13 | #if NET6_0_OR_GREATER // OperatingSystem check is only available in .NET 6.0 or later, but the runner itself won't be .NET 4 anyway 14 | if (OperatingSystem.IsWindows()) 15 | { 16 | AddJob(baseJob 17 | .WithRuntime(ClrRuntime.Net48)); 18 | } 19 | #endif 20 | 21 | AddJob(baseJob 22 | .WithRuntime(CoreRuntime.Core80) 23 | .WithPgo(true)); 24 | AddJob(baseJob 25 | .WithRuntime(CoreRuntime.Core90) 26 | .WithPgo(true)); 27 | AddJob(baseJob 28 | .WithRuntime(CoreRuntime.Core10_0) 29 | .WithPgo(true)); 30 | 31 | AddLogicalGroupRules(BenchmarkLogicalGroupRule.ByJob); 32 | 33 | HideColumns(Column.EnvironmentVariables); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/CompressAll.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class CompressAll 6 | { 7 | private MemoryStream _source; 8 | private MemoryStream _destination; 9 | 10 | [Params("alice29.txt", "asyoulik.txt", "fireworks.jpeg", "geo.protodata", "html", "html_x_4", 11 | "kppkn.gtb", "lcet10.txt", "paper-100k.pdf", "plrabn12.txt", "urls.10K")] 12 | public string FileName; 13 | 14 | [GlobalSetup] 15 | public void LoadToMemory() 16 | { 17 | _source = new MemoryStream(); 18 | _destination = new MemoryStream(); 19 | 20 | using Stream resource = 21 | typeof(CompressAll).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData." + FileName); 22 | 23 | // ReSharper disable once PossibleNullReferenceException 24 | resource.CopyTo(_source); 25 | } 26 | 27 | [Benchmark] 28 | public void Compress() 29 | { 30 | _source.Position = 0; 31 | _destination.Position = 0; 32 | using var stream = new SnappyStream(_destination, CompressionMode.Compress, true); 33 | 34 | _source.CopyTo(stream, 65536); 35 | stream.Flush(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Snappier/Internal/DebugExtensions.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Diagnostics.CodeAnalysis; 3 | using System.Runtime.CompilerServices; 4 | 5 | namespace Snappier.Internal; 6 | 7 | internal static class DebugExtensions 8 | { 9 | // Variant of Debug.Assert that is marked with DoesNotReturnIf and CallerArgumentExpression on down-level runtimes 10 | [Conditional("DEBUG")] 11 | public static void Assert([DoesNotReturnIf(false)] bool condition, 12 | [CallerArgumentExpression(nameof(condition))] string? message = null) 13 | { 14 | #if NET8_0_OR_GREATER 15 | Debug.Assert(condition, message); 16 | #else 17 | if (!condition) 18 | { 19 | Debug.Fail(message); 20 | } 21 | #endif 22 | } 23 | 24 | #if !NET8_0_OR_GREATER 25 | 26 | // Variant of Debug.Fail that is marked with DoesNotReturn on down-level runtimes 27 | [Conditional("DEBUG")] 28 | [DoesNotReturn] 29 | private static void Fail(string message) 30 | { 31 | Debug.Fail(message); 32 | 33 | // Unreachable but prevents compiler warnings, on down-level runtimes Debug.Fail is not marked as DoesNotReturn 34 | ThrowHelper.ThrowInvalidOperationException(message); 35 | } 36 | 37 | #endif 38 | } 39 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Program.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Running; 2 | using Snappier.Benchmarks.Configuration; 3 | 4 | Console.WriteLine("Select configuration:"); 5 | Console.WriteLine(" #0 Frameworks Short"); 6 | Console.WriteLine(" #1 Frameworks Default"); 7 | Console.WriteLine(" #2 Version Comparison Short"); 8 | Console.WriteLine(" #3 Version Comparison Default"); 9 | Console.WriteLine(" #4 Basic Short"); 10 | Console.WriteLine(" #5 Basic Default"); 11 | Console.WriteLine(" #6 x86/x64 Short"); 12 | Console.WriteLine(" #7 x86/x64 Default"); 13 | 14 | Console.WriteLine(); 15 | Console.Write("Selection: "); 16 | 17 | string input = Console.ReadLine(); 18 | Console.WriteLine(); 19 | 20 | StandardConfig config = input switch 21 | { 22 | "0" => (StandardConfig) new FrameworkCompareConfig(Job.ShortRun), 23 | "1" => new FrameworkCompareConfig(Job.Default), 24 | "2" => new VersionComparisonConfig(Job.ShortRun), 25 | "3" => new VersionComparisonConfig(Job.Default), 26 | "4" => new BasicConfig(Job.ShortRun), 27 | "5" => new BasicConfig(Job.Default), 28 | "6" => new X86X64Config(Job.ShortRun), 29 | "7" => new X86X64Config(Job.Default), 30 | _ => null 31 | }; 32 | 33 | if (config is not null) 34 | { 35 | BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config); 36 | } 37 | -------------------------------------------------------------------------------- /docfx.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://raw.githubusercontent.com/dotnet/docfx/main/schemas/docfx.schema.json", 3 | "metadata": [ 4 | { 5 | "src": [ 6 | { 7 | "src": "./", 8 | "files": [ 9 | "artifacts/bin/Snappier/release_net8.0/Snappier.dll" 10 | ] 11 | } 12 | ], 13 | "output": "api", 14 | "properties": { 15 | "TargetFramework": "net8.0" 16 | } 17 | } 18 | ], 19 | "build": { 20 | "content": [ 21 | { 22 | "files": [ 23 | "**/*.{md,yml}" 24 | ], 25 | "exclude": [ 26 | "_site/**", 27 | "artifacts/**", 28 | "**/BenchmarkDotNet.Artifacts/**" 29 | ] 30 | } 31 | ], 32 | "resource": [ 33 | { 34 | "files": [ 35 | "images/**" 36 | ] 37 | } 38 | ], 39 | "output": "artifacts/_site", 40 | "template": [ 41 | "default", 42 | "material/material" 43 | ], 44 | "globalMetadata": { 45 | "_appName": "Snappier", 46 | "_appTitle": "Snappier", 47 | "_appLogoPath": "images/icon-48.png", 48 | "_disableContribution": true, 49 | "_enableSearch": true, 50 | "pdf": false 51 | }, 52 | "postProcessors": ["ExtractSearchIndex"] 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/X86X64Config.cs: -------------------------------------------------------------------------------- 1 | using BenchmarkDotNet.Diagnosers; 2 | using BenchmarkDotNet.Environments; 3 | using BenchmarkDotNet.Jobs; 4 | using BenchmarkDotNet.Toolchains.CsProj; 5 | using BenchmarkDotNet.Toolchains.DotNetCli; 6 | 7 | namespace Snappier.Benchmarks.Configuration; 8 | 9 | public class X86X64Config : StandardConfig 10 | { 11 | public X86X64Config(Job baseJob) 12 | { 13 | NetCoreAppSettings dotnetCli32Bit = NetCoreAppSettings 14 | .NetCoreApp50 15 | .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); 16 | 17 | NetCoreAppSettings dotnetCli64Bit = NetCoreAppSettings 18 | .NetCoreApp50 19 | .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); 20 | 21 | AddJob(baseJob 22 | .WithToolchain(CsProjCoreToolchain.From(dotnetCli32Bit)) 23 | .WithPlatform(Platform.X86) 24 | .WithId("x86")); 25 | AddJob(baseJob 26 | .WithToolchain(CsProjCoreToolchain.From(dotnetCli64Bit)) 27 | .WithPlatform(Platform.X64) 28 | .WithId("x64")); 29 | 30 | AddDiagnoser(new DisassemblyDiagnoser( 31 | new DisassemblyDiagnoserConfig(maxDepth: 2, printSource: true, exportDiff: true))); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/DecompressAll.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class DecompressAll 6 | { 7 | private MemoryStream _memoryStream; 8 | private byte[] _buffer; 9 | 10 | [Params("alice29.txt", "asyoulik.txt", "fireworks.jpeg", "geo.protodata", "html", "html_x_4", 11 | "kppkn.gtb", "lcet10.txt", "paper-100k.pdf", "plrabn12.txt", "urls.10K")] 12 | public string FileName; 13 | 14 | [GlobalSetup] 15 | public void LoadToMemory() 16 | { 17 | _memoryStream = new MemoryStream(); 18 | 19 | using Stream resource = 20 | typeof(DecompressAll).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData." + FileName); 21 | 22 | using var compressStream = new SnappyStream(_memoryStream, CompressionMode.Compress, true); 23 | 24 | // ReSharper disable once PossibleNullReferenceException 25 | resource.CopyTo(compressStream); 26 | compressStream.Flush(); 27 | 28 | _buffer = new byte[65536]; 29 | } 30 | 31 | 32 | [Benchmark] 33 | public void Decompress() 34 | { 35 | _memoryStream.Position = 0; 36 | using var stream = new SnappyStream(_memoryStream, CompressionMode.Decompress, true); 37 | 38 | while (stream.Read(_buffer, 0, _buffer.Length) > 0) 39 | { 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Snappier.Tests/SequenceHelpers.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | 3 | #nullable enable 4 | 5 | namespace Snappier.Tests; 6 | 7 | public static class SequenceHelpers 8 | { 9 | public static ReadOnlySequence CreateSequence(ReadOnlyMemory source, int maxSegmentSize) 10 | { 11 | ReadOnlySequenceSegment? lastSegment = null; 12 | ReadOnlySequenceSegment? currentSegment = null; 13 | 14 | while (source.Length > 0) 15 | { 16 | int index = Math.Max(source.Length - maxSegmentSize, 0); 17 | 18 | currentSegment = new Segment( 19 | source.Slice(index), 20 | currentSegment, 21 | index); 22 | 23 | lastSegment ??= currentSegment; 24 | source = source.Slice(0, index); 25 | } 26 | 27 | if (currentSegment is null) 28 | { 29 | return default; 30 | } 31 | 32 | return new ReadOnlySequence(currentSegment, 0, lastSegment!, lastSegment!.Memory.Length); 33 | } 34 | 35 | private sealed class Segment : ReadOnlySequenceSegment 36 | { 37 | public Segment(ReadOnlyMemory memory, ReadOnlySequenceSegment? next, long runningIndex) 38 | { 39 | Memory = memory; 40 | Next = next; 41 | RunningIndex = runningIndex; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/PgoColumn.cs: -------------------------------------------------------------------------------- 1 | using System.Linq; 2 | using BenchmarkDotNet.Columns; 3 | using BenchmarkDotNet.Reports; 4 | using BenchmarkDotNet.Running; 5 | 6 | namespace Snappier.Benchmarks.Configuration; 7 | 8 | public class PgoColumn : IColumn 9 | { 10 | public static readonly IColumn Default = new PgoColumn(); 11 | 12 | public const string PgoEnvironmentVariableName = "DOTNET_TieredPGO"; 13 | 14 | public string Id => "PGO"; 15 | public string ColumnName => "PGO"; 16 | 17 | public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => !IsPgo(benchmarkCase); 18 | public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => IsPgo(benchmarkCase) ? "Y" : "N"; 19 | 20 | public static bool IsPgo(BenchmarkCase benchmarkCase) => 21 | benchmarkCase.Job.Environment.EnvironmentVariables?.Any(p => p.Key == PgoEnvironmentVariableName && p.Value == "1") ?? false; 22 | 23 | public bool IsAvailable(Summary summary) => true; 24 | public bool AlwaysShow => true; 25 | public ColumnCategory Category => ColumnCategory.Job; 26 | public int PriorityInCategory => 0; 27 | public bool IsNumeric => false; 28 | public UnitType UnitType => UnitType.Dimensionless; 29 | public string Legend => $"Indicates state of Dynamic PGO"; 30 | public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); 31 | public override string ToString() => ColumnName; 32 | } 33 | -------------------------------------------------------------------------------- /Snappier.Tests/HelpersTests.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Tests; 2 | 3 | public class HelpersTests 4 | { 5 | #region LeftShiftOverflows 6 | 7 | [Theory] 8 | [InlineData(2, 31)] 9 | [InlineData(0xff, 25)] 10 | public void LeftShiftOverflows_True(byte value, int shift) 11 | { 12 | // Act 13 | 14 | bool result = Helpers.LeftShiftOverflows(value, shift); 15 | 16 | // Assert 17 | 18 | Assert.True(result); 19 | } 20 | 21 | [Theory] 22 | [InlineData(1, 31)] 23 | [InlineData(0xff, 24)] 24 | [InlineData(0, 31)] 25 | public void LeftShiftOverflows_False(byte value, int shift) 26 | { 27 | // Act 28 | 29 | bool result = Helpers.LeftShiftOverflows(value, shift); 30 | 31 | // Assert 32 | 33 | Assert.False(result); 34 | } 35 | 36 | public static TheoryData Log2FloorValues() => 37 | [ 38 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 39 | ]; 40 | 41 | [Theory] 42 | [MemberData(nameof(Log2FloorValues))] 43 | public void Log2Floor(uint value) 44 | { 45 | // Act 46 | 47 | int result = Helpers.Log2Floor(value); 48 | 49 | // Assert 50 | 51 | Assert.Equal((int) Math.Floor(Math.Log(value, 2)), result); 52 | } 53 | 54 | [Fact] 55 | public void Log2Floor_Zero() 56 | { 57 | // Act 58 | 59 | int result = Helpers.Log2Floor(0); 60 | 61 | // Assert 62 | 63 | Assert.Equal(0, result); 64 | } 65 | 66 | #endregion 67 | } 68 | -------------------------------------------------------------------------------- /Snappier.Tests/Internal/SnappyStreamCompressorTests.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Tests.Internal; 2 | 3 | 4 | public class SnappyStreamCompressorTests 5 | { 6 | [Theory] 7 | [InlineData("alice29.txt")] 8 | [InlineData("asyoulik.txt")] 9 | [InlineData("fireworks.jpeg")] 10 | [InlineData("geo.protodata")] 11 | [InlineData("html")] 12 | [InlineData("html_x_4")] 13 | [InlineData("kppkn.gtb")] 14 | [InlineData("lcet10.txt")] 15 | [InlineData("paper-100k.pdf")] 16 | [InlineData("plrabn12.txt")] 17 | [InlineData("urls.10K")] 18 | public void Write(string resourceName) 19 | { 20 | using Stream resource = 21 | typeof(SnappyStreamCompressorTests).Assembly.GetManifestResourceStream("Snappier.Tests.TestData." + resourceName); 22 | Assert.NotNull(resource); 23 | 24 | using var memStream = new MemoryStream(); 25 | resource.CopyTo(memStream); 26 | 27 | Span input = memStream.GetBuffer().AsSpan(0, (int) memStream.Length); 28 | 29 | using var output = new MemoryStream(); 30 | 31 | using var compressor = new SnappyStreamCompressor(); 32 | compressor.Write(input, output); 33 | compressor.Flush(output); 34 | 35 | using var decompressor = new SnappyStreamDecompressor(); 36 | decompressor.SetInput(output.GetBuffer().AsMemory(0, (int) output.Length)); 37 | 38 | byte[] decompressed = new byte[memStream.Length + 1]; // Add 1 to make sure decompress ends correctly 39 | int bytesDecompressed = decompressor.Decompress(decompressed); 40 | 41 | Assert.Equal(input.Length, bytesDecompressed); 42 | for (int i = 0; i < bytesDecompressed; i++) 43 | { 44 | Assert.Equal(input[i], decompressed[i]); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /.github/workflows/pages.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Pages 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: 7 | - main 8 | - release-* 9 | 10 | # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. 11 | # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. 12 | concurrency: 13 | group: pages 14 | cancel-in-progress: false 15 | 16 | jobs: 17 | build-docs: 18 | name: Build Documentation 19 | 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v5 24 | - name: Setup .NET 25 | uses: actions/setup-dotnet@v5 26 | with: 27 | dotnet-version: 10.0.x 28 | 29 | - name: Build 30 | # Target framework should match docfx.json 31 | run: dotnet build -f net8.0 --configuration Release Snappier/Snappier.csproj 32 | 33 | - name: Install docfX 34 | run: dotnet tool restore 35 | - name: Build documentation 36 | run: dotnet docfx docfx.json 37 | 38 | - name: Upload artifact 39 | uses: actions/upload-pages-artifact@v4 40 | with: 41 | path: 'artifacts/_site' 42 | 43 | publish-docs: 44 | name: Publish Documentation 45 | needs: build-docs 46 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} 47 | 48 | # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages 49 | permissions: 50 | actions: read 51 | pages: write 52 | id-token: write 53 | 54 | # Deploy to the github-pages environment 55 | environment: 56 | name: github-pages 57 | url: ${{ steps.deployment.outputs.page_url }} 58 | 59 | runs-on: ubuntu-latest 60 | steps: 61 | - name: Deploy to GitHub Pages 62 | id: deployment 63 | uses: actions/deploy-pages@v4 64 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thanks for taking the time to contribute! 4 | 5 | The following is a set of guidelines for contributing to Snappier. These are just guidelines, not rules, so use your best judgement and feel free to propose changes to this document in a pull request. 6 | 7 | ## Getting Started 8 | 9 | Snappier is built with C#. If you are new to C#, please head [here](https://docs.microsoft.com/en-us/dotnet/csharp/getting-started/)! 10 | 11 | ## Community 12 | 13 | * If you have any questions regarding Snappier, open an [issue](https://github.com/brantburnett/Snappier/issues/new/). 14 | 15 | ## Issues 16 | 17 | Ensure the bug was not already reported by searching on GitHub under [issues](https://github.com/brantburnett/Snappier/issues). If you're unable to find an open issue addressing the bug, open a [new issue](https://github.com/brantburnett/Snappier/issues/new). 18 | 19 | Please pay attention to the following points while opening an issue. 20 | 21 | ### Write detailed information 22 | 23 | Detailed information is very helpful to understand an issue. 24 | 25 | For example: 26 | 27 | * How to reproduce the issue, step-by-step. 28 | * The expected behavior (or what is wrong). 29 | * Error messages and stack traces, if applicable. 30 | * If possible, sample files that produce the error. 31 | 32 | ## Pull Requests 33 | 34 | Pull Requests are always welcome. 35 | 36 | 1. When you edit the code, please make sure everything is building and any and all tests succeed. 37 | 2. If new behaviors are added, tests should be added to prevent regressions in the future. 38 | 3. Ensure the PR description clearly describes the problem and solution. It should include: 39 | * An imperative first line. 40 | * Motivation for the change. 41 | * If warranted, and explanation of why the given solution was chosen. 42 | * The relevant issue number, if applicable. 43 | -------------------------------------------------------------------------------- /docs/stream.md: -------------------------------------------------------------------------------- 1 | # Stream Compression 2 | 3 | Stream compression reads or writes the [Snappy framing format](https://github.com/google/snappy/blob/master/framing_format.txt) designed for streaming. 4 | It is ideal for data being sent over a network stream, and includes additional framing data and CRC validation. 5 | It also recognizes when an individual block in the stream compresses poorly and will include it in uncompressed form. 6 | 7 | ## Stream compression/decompression 8 | 9 | Compressing or decompressing a stream follows the same paradigm as other compression streams in .NET. `SnappyStream` wraps an inner stream. If decompressing you read from the `SnappyStream`, if compressing you write to the `SnappyStream` 10 | 11 | ```cs 12 | using System.IO; 13 | using System.IO.Compression; 14 | using Snappier; 15 | 16 | public class Program 17 | { 18 | public static async Task Main() 19 | { 20 | using var fileStream = File.OpenRead("somefile.txt"); 21 | 22 | // First, compression 23 | using var compressed = new MemoryStream(); 24 | 25 | using (var compressor = new SnappyStream(compressed, CompressionMode.Compress, leaveOpen: true)) 26 | { 27 | await fileStream.CopyToAsync(compressor); 28 | 29 | // Disposing the compressor also flushes the buffers to the inner stream 30 | // We pass true to the constructor above so that it doesn't close/dispose the inner stream 31 | // Alternatively, we could call compressor.Flush() 32 | } 33 | 34 | // Then, decompression 35 | 36 | compressed.Position = 0; // Reset to beginning of the stream so we can read 37 | using var decompressor = new SnappyStream(compressed, CompressionMode.Decompress); 38 | 39 | var buffer = new byte[65536]; 40 | var bytesRead = decompressor.Read(buffer, 0, buffer.Length); 41 | while (bytesRead > 0) 42 | { 43 | // Do something with the data 44 | 45 | bytesRead = decompressor.Read(buffer, 0, buffer.Length) 46 | } 47 | } 48 | } 49 | ``` -------------------------------------------------------------------------------- /Snappier/Internal/ByteArrayPoolMemoryOwner.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | 3 | namespace Snappier.Internal; 4 | 5 | /// 6 | /// Wraps an inner byte array from "/> with a limited length. 7 | /// 8 | /// 9 | /// We use this instead of the built-in because we want to slice the array without 10 | /// allocating another wrapping class on the heap. 11 | /// 12 | internal sealed class ByteArrayPoolMemoryOwner : IMemoryOwner 13 | { 14 | private byte[]? _innerArray; 15 | 16 | /// 17 | public Memory Memory { get; private set; } 18 | 19 | /// 20 | /// Create an empty ByteArrayPoolMemoryOwner. 21 | /// 22 | public ByteArrayPoolMemoryOwner() 23 | { 24 | // _innerArray will be null and Memory will be a default empty Memory 25 | } 26 | 27 | /// 28 | /// Given a byte array from , create a ByteArrayPoolMemoryOwner 29 | /// which wraps it until disposed and slices it to . 30 | /// 31 | /// An array from the . 32 | /// The length of the array to return from . 33 | public ByteArrayPoolMemoryOwner(byte[] innerArray, int length) 34 | { 35 | ArgumentNullException.ThrowIfNull(innerArray); 36 | 37 | _innerArray = innerArray; 38 | Memory = innerArray.AsMemory(0, length); // Also validates length 39 | } 40 | 41 | /// 42 | public void Dispose() 43 | { 44 | byte[]? innerArray = _innerArray; 45 | if (innerArray is not null) 46 | { 47 | // Clear the used portion of the array before returning it to the pool 48 | Memory.Span.Clear(); 49 | 50 | _innerArray = null; 51 | Memory = default; 52 | ArrayPool.Shared.Return(innerArray); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Snappier.Tests/Snappier.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net8.0;net9.0;net10.0 5 | $(TargetFrameworks);net48 6 | Exe 7 | 8 | false 9 | enable 10 | true 11 | ..\Snappier.snk 12 | 13 | 14 | false 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | all 26 | runtime; build; native; contentfiles; analyzers; buildtransitive 27 | 28 | 29 | all 30 | runtime; build; native; contentfiles; analyzers; buildtransitive 31 | 32 | 33 | 34 | 35 | all 36 | runtime; build; native; contentfiles; analyzers; buildtransitive 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Snappier.Benchmarks.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Exe 5 | net48;net8.0;net9.0;net10.0 6 | AnyCPU 7 | 8 | false 9 | enable 10 | true 11 | ..\Snappier.snk 12 | $(NoWarn);8002 13 | 14 | 15 | 16 | 17 | TestData\%(FileName)%(Extension) 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | all 31 | runtime; build; native; contentfiles; analyzers; buildtransitive 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | workflow_dispatch: {} 10 | schedule: 11 | - cron: '0 19 * * 3' 12 | 13 | jobs: 14 | analyze: 15 | name: Analyze 16 | runs-on: ubuntu-latest 17 | 18 | permissions: 19 | security-events: write 20 | 21 | strategy: 22 | fail-fast: false 23 | matrix: 24 | # Override automatic language detection by changing the below list 25 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 26 | language: ['csharp'] 27 | # Learn more... 28 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 29 | 30 | steps: 31 | - name: Checkout repository 32 | uses: actions/checkout@v5 33 | - name: Setup .NET 6.0 34 | uses: actions/setup-dotnet@v5 35 | with: 36 | dotnet-version: 10.0.x 37 | 38 | # Initializes the CodeQL tools for scanning. 39 | - name: Initialize CodeQL 40 | uses: github/codeql-action/init@v2 41 | with: 42 | languages: ${{ matrix.language }} 43 | # If you wish to specify custom queries, you can do so here or in a config file. 44 | # By default, queries listed here will override any specified in a config file. 45 | # Prefix the list here with "+" to use these queries and those in the config file. 46 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 47 | 48 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 49 | # If this step fails, then you should remove it and run the build manually (see below) 50 | - name: Autobuild 51 | uses: github/codeql-action/autobuild@v2 52 | 53 | # ℹ️ Command-line programs to run using the OS shell. 54 | # 📚 https://git.io/JvXDl 55 | 56 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 57 | # and modify them (or add more) to build your code if your project 58 | # uses a compiled language 59 | 60 | #- run: | 61 | # make bootstrap 62 | # make release 63 | 64 | - name: Perform CodeQL Analysis 65 | uses: github/codeql-action/analyze@v2 66 | -------------------------------------------------------------------------------- /Snappier/Internal/VarIntEncoding.Write.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Internal; 2 | 3 | internal static partial class VarIntEncoding 4 | { 5 | private static bool TryWriteSlow(Span output, uint length, out int bytesWritten) 6 | { 7 | const int b = 0b1000_0000; 8 | 9 | unchecked 10 | { 11 | if (length < (1 << 7)) 12 | { 13 | if (output.Length < 1) 14 | { 15 | bytesWritten = 0; 16 | return false; 17 | } 18 | 19 | output[0] = (byte) length; 20 | bytesWritten = 1; 21 | } 22 | else if (length < (1 << 14)) 23 | { 24 | if (output.Length < 2) 25 | { 26 | bytesWritten = 0; 27 | return false; 28 | } 29 | 30 | output[0] = (byte) (length | b); 31 | output[1] = (byte) (length >> 7); 32 | bytesWritten = 2; 33 | } 34 | else if (length < (1 << 21)) 35 | { 36 | if (output.Length < 3) 37 | { 38 | bytesWritten = 0; 39 | return false; 40 | } 41 | 42 | output[0] = (byte) (length | b); 43 | output[1] = (byte) ((length >> 7) | b); 44 | output[2] = (byte) (length >> 14); 45 | bytesWritten = 3; 46 | } 47 | else if (length < (1 << 28)) 48 | { 49 | if (output.Length < 4) 50 | { 51 | bytesWritten = 0; 52 | return false; 53 | } 54 | 55 | output[0] = (byte) (length | b); 56 | output[1] = (byte) ((length >> 7) | b); 57 | output[2] = (byte) ((length >> 14) | b); 58 | output[3] = (byte) (length >> 21); 59 | bytesWritten = 4; 60 | } 61 | else 62 | { 63 | if (output.Length < 5) 64 | { 65 | bytesWritten = 0; 66 | return false; 67 | } 68 | 69 | output[0] = (byte) (length | b); 70 | output[1] = (byte) ((length >> 7) | b); 71 | output[2] = (byte) ((length >> 14) | b); 72 | output[3] = (byte) ((length >> 21) | b); 73 | output[4] = (byte) (length >> 28); 74 | bytesWritten = 5; 75 | } 76 | } 77 | 78 | return true; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Snappier/Internal/ThrowHelper.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics.CodeAnalysis; 2 | using System.Runtime.CompilerServices; 3 | 4 | namespace Snappier.Internal; 5 | 6 | internal static class ThrowHelper 7 | { 8 | [DoesNotReturn] 9 | public static void ThrowArgumentException(string? message, string? paramName) => 10 | throw new ArgumentException(message, paramName); 11 | 12 | [DoesNotReturn] 13 | public static void ThrowArgumentOutOfRangeException(string? paramName, string? message) => 14 | throw new ArgumentOutOfRangeException(paramName, message); 15 | 16 | [DoesNotReturn] 17 | [MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce code size for a cold path 18 | public static void ThrowArgumentExceptionInsufficientOutputBuffer(string? paramName) => 19 | ThrowArgumentException("Output buffer is too small.", paramName); 20 | 21 | [DoesNotReturn] 22 | public static void ThrowInvalidDataException(string? message) => 23 | throw new InvalidDataException(message); 24 | 25 | [DoesNotReturn] 26 | [MethodImpl(MethodImplOptions.NoInlining)] // Avoid inlining to reduce code size for a cold path 27 | public static void ThrowInvalidDataExceptionIncompleteSnappyBlock() => 28 | throw new InvalidDataException("Incomplete Snappy block."); 29 | 30 | [DoesNotReturn] 31 | public static void ThrowInvalidOperationException(string? message = null) => 32 | throw new InvalidOperationException(message); 33 | 34 | [DoesNotReturn] 35 | public static void ThrowNotSupportedException(string? message = null) => 36 | throw new NotSupportedException(message); 37 | 38 | #if !NET8_0_OR_GREATER 39 | [DoesNotReturn] 40 | private static void ThrowArgumentNullException(string? paramName) => 41 | throw new ArgumentNullException(paramName); 42 | 43 | [DoesNotReturn] 44 | private static void ThrowObjectDisposedException(object? instance) => 45 | throw new ObjectDisposedException(instance?.GetType().FullName); 46 | 47 | extension(ArgumentNullException) 48 | { 49 | public static void ThrowIfNull([NotNull] object? argument, [CallerArgumentExpression(nameof(argument))] string? paramName = null) 50 | { 51 | if (argument is null) 52 | { 53 | ThrowArgumentNullException(paramName); 54 | } 55 | } 56 | } 57 | 58 | extension(ObjectDisposedException) 59 | { 60 | public static void ThrowIf([DoesNotReturnIf(true)] bool condition, object instance) 61 | { 62 | if (condition) 63 | { 64 | ThrowObjectDisposedException(instance); 65 | } 66 | } 67 | } 68 | #endif 69 | } 70 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/FindMatchLength.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | namespace Snappier.Benchmarks; 4 | 5 | public class FindMatchLength 6 | { 7 | private static readonly byte[] s_fourByteMatch = 8 | [ 9 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 10 | 1, 2, 3, 4, 4, 6, 7, 8, 9, 10, 11, 13, 11 | // Padding so we ensure we're hitting the hot (and fast) path where there is plenty more data in the input buffer 12 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13 | ]; 14 | 15 | private static readonly byte[] s_sevenByteMatch = 16 | [ 17 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18 | 1, 2, 3, 4, 5, 6, 7, 7, 9, 10, 11, 13, 19 | // Padding so we ensure we're hitting the hot (and fast) path where there is plenty more data in the input buffer 20 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21 | ]; 22 | 23 | private static readonly byte[] s_elevenByteMatch = 24 | [ 25 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 26 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 27 | // Padding so we ensure we're hitting the hot (and fast) path where there is plenty more data in the input buffer 28 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29 | ]; 30 | 31 | private static readonly byte[] s_thirtyTwoByteMatch = 32 | [ 33 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 34 | 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 35 | // Padding so we ensure we're hitting the hot (and fast) path where there is plenty more data in the input buffer 36 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37 | ]; 38 | 39 | [Params(4, 7, 11, 32)] 40 | public int MatchLength { get; set; } 41 | 42 | private byte[] _array; 43 | 44 | [GlobalSetup] 45 | public void GlobalSetup() 46 | { 47 | _array = MatchLength switch 48 | { 49 | 4 => s_fourByteMatch, 50 | 7 => s_sevenByteMatch, 51 | 11 => s_elevenByteMatch, 52 | 32 => s_thirtyTwoByteMatch, 53 | _ => throw new InvalidOperationException() 54 | }; 55 | } 56 | 57 | [Benchmark(Baseline = true)] 58 | public (long, bool) Regular() 59 | { 60 | ulong data = 0; 61 | 62 | ref byte s1 = ref _array[0]; 63 | ref byte s2 = ref Unsafe.Add(ref s1, 12); 64 | ref byte s2Limit = ref Unsafe.Add(ref s1, _array.Length); 65 | 66 | return SnappyCompressor.FindMatchLength(ref s1, ref s2, ref s2Limit, ref data); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /COPYING.txt: -------------------------------------------------------------------------------- 1 | Copyright 2011-2024, Google, Inc. and Snappier Authors. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following disclaimer 12 | in the documentation and/or other materials provided with the 13 | distribution. 14 | * Neither the name of Google, Inc., any Snappier authors, nor the 15 | names of its contributors may be used to endorse or promote products 16 | derived from this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | 30 | === 31 | 32 | Some of the benchmark data in testdata/ is licensed differently: 33 | 34 | - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and 35 | is licensed under the Creative Commons Attribution 3.0 license 36 | (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/ 37 | for more information. 38 | 39 | - kppkn.gtb is taken from the Gaviota chess tablebase set, and 40 | is licensed under the MIT License. See 41 | https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1 42 | for more information. 43 | 44 | - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper 45 | “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA 46 | Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro, 47 | which is licensed under the CC-BY license. See 48 | http://www.ploscompbiol.org/static/license for more information. 49 | 50 | - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project 51 | Gutenberg. The first three have expired copyrights and are in the public 52 | domain; the latter does not have expired copyright, but is still in the 53 | public domain according to the license information 54 | (http://www.gutenberg.org/ebooks/53). 55 | -------------------------------------------------------------------------------- /Snappier.Tests/Internal/VarIntEncodingWriteTests.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Tests.Internal; 2 | 3 | public class VarIntEncodingWriteTests 4 | { 5 | public static TheoryData TestData() => 6 | new() { 7 | { 0x00, [ 0x00 ] }, 8 | { 0x01, [ 0x01 ] }, 9 | { 0x7F, [ 0x7F ] }, 10 | { 0x80, [ 0x80, 0x01 ] }, 11 | { 0x555, [ 0xD5, 0x0A ] }, 12 | { 0x7FFF, [ 0xFF, 0xFF, 0x01 ] }, 13 | { 0xBFFF, [ 0xFF, 0xFF, 0x02 ] }, 14 | { 0xFFFF, [ 0XFF, 0xFF, 0x03 ] }, 15 | { 0x8000, [ 0x80, 0x80, 0x02 ] }, 16 | { 0x5555, [ 0xD5, 0xAA, 0x01 ] }, 17 | { 0xCAFEF00, [ 0x80, 0xDE, 0xBF, 0x65 ] }, 18 | { 0xCAFEF00D, [ 0x8D, 0xE0, 0xFB, 0xD7, 0x0C ] }, 19 | { 0xFFFFFFFF, [ 0xFF, 0xFF, 0xFF, 0xFF, 0x0F ] }, 20 | }; 21 | 22 | [Theory] 23 | [MemberData(nameof(TestData))] 24 | public void Test_TryWrite(uint value, byte[] expected) 25 | { 26 | byte[] bytes = new byte[5]; 27 | 28 | bool success = VarIntEncoding.TryWrite(bytes, value, out int length); 29 | Assert.True(success); 30 | Assert.Equal(expected, bytes.Take(length)); 31 | } 32 | 33 | [Theory] 34 | [MemberData(nameof(TestData))] 35 | public void Test_TryWriteInsufficientBufferLength(uint value, byte[] expected) 36 | { 37 | byte[] bytes = new byte[expected.Length - 1]; 38 | 39 | bool success = VarIntEncoding.TryWrite(bytes, value, out int length); 40 | Assert.False(success); 41 | } 42 | 43 | [Theory] 44 | [MemberData(nameof(TestData))] 45 | public void Test_TryWriteWithPadding(uint value, byte[] expected) 46 | { 47 | // Test of the fast path where there are at least 8 bytes in the buffer 48 | 49 | byte[] bytes = new byte[sizeof(ulong)]; 50 | 51 | bool success = VarIntEncoding.TryWrite(bytes, value, out int length); 52 | Assert.True(success); 53 | Assert.Equal(expected, bytes.Take(length)); 54 | } 55 | } 56 | 57 | /* ************************************************************ 58 | * 59 | * @author Couchbase 60 | * @copyright 2021 Couchbase, Inc. 61 | * 62 | * Licensed under the Apache License, Version 2.0 (the "License"); 63 | * you may not use this file except in compliance with the License. 64 | * You may obtain a copy of the License at 65 | * 66 | * http://www.apache.org/licenses/LICENSE-2.0 67 | * 68 | * Unless required by applicable law or agreed to in writing, software 69 | * distributed under the License is distributed on an "AS IS" BASIS, 70 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 71 | * See the License for the specific language governing permissions and 72 | * limitations under the License. 73 | * 74 | * ************************************************************/ 75 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Overview.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | using Snappier.Internal; 3 | 4 | namespace Snappier.Benchmarks; 5 | 6 | public class Overview 7 | { 8 | private MemoryStream _htmlStream; 9 | private Memory _htmlMemory; 10 | 11 | private ReadOnlyMemory _compressed; 12 | private MemoryStream _compressedStream; 13 | 14 | private Memory _outputBuffer; 15 | private byte[] _streamOutputBuffer; 16 | private MemoryStream _streamOutput; 17 | 18 | [GlobalSetup] 19 | public void LoadToMemory() 20 | { 21 | _htmlStream = new MemoryStream(); 22 | using Stream resource = 23 | typeof(DecompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html"); 24 | resource!.CopyTo(_htmlStream); 25 | _htmlStream.Position = 0; 26 | 27 | byte[] input = new byte[65536]; // Just test the first 64KB 28 | // ReSharper disable once PossibleNullReferenceException 29 | int inputLength = _htmlStream.Read(input, 0, input.Length); 30 | _htmlMemory = input.AsMemory(0, inputLength); 31 | 32 | byte[] compressed = new byte[Snappy.GetMaxCompressedLength(inputLength)]; 33 | int compressedLength = Snappy.Compress(_htmlMemory.Span, compressed); 34 | 35 | _compressed = compressed.AsMemory(0, compressedLength); 36 | 37 | _compressedStream = new MemoryStream(); 38 | using Stream resource2 = 39 | typeof(DecompressHtml).Assembly.GetManifestResourceStream("Snappier.Benchmarks.TestData.html_x_4.snappy"); 40 | // ReSharper disable once PossibleNullReferenceException 41 | resource2.CopyTo(_compressedStream); 42 | 43 | _outputBuffer = new byte[Snappy.GetMaxCompressedLength(inputLength)]; 44 | _streamOutputBuffer = new byte[16384]; 45 | _streamOutput = new MemoryStream(); 46 | } 47 | 48 | [Benchmark] 49 | public int BlockCompress64KbHtml() 50 | { 51 | using var compressor = new SnappyCompressor(); 52 | 53 | #pragma warning disable CS0618 // Type or member is obsolete 54 | return compressor.Compress(_htmlMemory.Span, _outputBuffer.Span); 55 | #pragma warning restore CS0618 // Type or member is obsolete 56 | } 57 | 58 | [Benchmark] 59 | public void BlockDecompress64KbHtml() 60 | { 61 | using var decompressor = new SnappyDecompressor(); 62 | 63 | decompressor.Decompress(_compressed.Span); 64 | } 65 | 66 | [Benchmark] 67 | public void StreamCompressHtml() 68 | { 69 | _htmlStream.Position = 0; 70 | _streamOutput.Position = 0; 71 | using var stream = new SnappyStream(_streamOutput, CompressionMode.Compress, true); 72 | 73 | _htmlStream.CopyTo(stream, 16384); 74 | stream.Flush(); 75 | } 76 | 77 | [Benchmark] 78 | public void StreamDecompressHtml() 79 | { 80 | _compressedStream.Position = 0; 81 | using var stream = new SnappyStream(_compressedStream, CompressionMode.Decompress, true); 82 | 83 | while (stream.Read(_streamOutputBuffer, 0, _streamOutputBuffer.Length) > 0) 84 | { 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Snappier/Internal/UnsafeReadonly.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | namespace Snappier.Internal; 4 | 5 | // Helpers to perform Unsafe.Add operations on readonly refs. 6 | internal static class UnsafeReadonly 7 | { 8 | extension(Unsafe) 9 | { 10 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 11 | public static ref readonly T Add(ref readonly T source, int elementOffset) => 12 | ref Unsafe.Add(ref Unsafe.AsRef(in source), elementOffset); 13 | 14 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 15 | public static ref readonly T Add(ref readonly T source, nint elementOffset) => 16 | ref Unsafe.Add(ref Unsafe.AsRef(in source), elementOffset); 17 | 18 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 19 | public static ref readonly T Add(ref readonly T source, nuint elementOffset) => 20 | ref Unsafe.Add(ref Unsafe.AsRef(in source), elementOffset); 21 | 22 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 23 | public static ref readonly TTo As(ref readonly TFrom source) => 24 | ref Unsafe.As(ref Unsafe.AsRef(in source)); 25 | 26 | #if !NET8_0_OR_GREATER 27 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 28 | public static nint ByteOffset(ref readonly T origin, ref readonly T target) => 29 | Unsafe.ByteOffset(ref Unsafe.AsRef(in origin), ref Unsafe.AsRef(in target)); 30 | 31 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 32 | public static void CopyBlockUnaligned(ref byte destination, ref readonly byte source, uint byteCount) => 33 | Unsafe.CopyBlockUnaligned(ref destination, ref Unsafe.AsRef(in source), byteCount); 34 | 35 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 36 | public static bool IsAddressGreaterThan(ref readonly T left, ref readonly T right) => 37 | Unsafe.IsAddressGreaterThan(ref Unsafe.AsRef(in left), ref Unsafe.AsRef(in right)); 38 | 39 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 40 | public static bool IsAddressLessThan(ref readonly T left, ref readonly T right) => 41 | Unsafe.IsAddressLessThan(ref Unsafe.AsRef(in left), ref Unsafe.AsRef(in right)); 42 | 43 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 44 | public static T ReadUnaligned(ref readonly byte source) => 45 | Unsafe.ReadUnaligned(ref Unsafe.AsRef(in source)); 46 | #endif 47 | 48 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 49 | public static ref readonly T Subtract(ref readonly T source, int elementOffset) => 50 | ref Unsafe.Subtract(ref Unsafe.AsRef(in source), elementOffset); 51 | 52 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 53 | public static ref readonly T Subtract(ref readonly T source, nint elementOffset) => 54 | ref Unsafe.Subtract(ref Unsafe.AsRef(in source), elementOffset); 55 | 56 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 57 | public static ref readonly T Subtract(ref readonly T source, nuint elementOffset) => 58 | ref Unsafe.Subtract(ref Unsafe.AsRef(in source), elementOffset); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Snappier/Snappier.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | net472;netstandard2.0;net8.0 5 | 6 | enable 7 | enable 8 | true 9 | ..\Snappier.snk 10 | 11 | btburnett3 12 | snappy;compression;fast;io 13 | 14 | Copyright 2011-2020, Google, Inc. and Snappier Authors 15 | 16 | A near-C++ performance implementation of the Snappy compression algorithm for .NET. Snappier is ported to C# directly 17 | from the official C++ implementation, with the addition of support for the framed stream format. 18 | 19 | By avoiding P/Invoke, Snappier is fully cross-platform and works on both Linux and Windows and against any CPU supported 20 | by .NET. However, Snappier performs best in .NET 6 and later on little-endian x86/64 processors with the 21 | help of System.Runtime.Instrinsics. 22 | 23 | BSD-3-Clause 24 | README.md 25 | icon.png 26 | https://brantburnett.github.io/Snappier/ 27 | true 28 | true 29 | true 30 | true 31 | snupkg 32 | 33 | true 34 | 35 | 36 | 37 | 38 | true 39 | 40 | 41 | 42 | true 43 | 1.2.0 44 | true 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | --- 2 | _layout: landing 3 | --- 4 | 5 | # Snappier 6 | 7 | Snappier is a pure C# port of Google's [Snappy](https://github.com/google/snappy) compression algorithm. It is designed with speed as the primary goal, rather than compression ratio, and is ideal for compressing network traffic. Please see [the Snappy README file](https://github.com/google/snappy/blob/master/README.md) for more details on Snappy. 8 | 9 | ## Project Goals 10 | 11 | The Snappier project aims to meet the following needs of the .NET community. 12 | 13 | - Cross-platform C# implementation for Linux and Windows, without P/Invoke or special OS installation requirements 14 | - Compatible with .NET 4.6.1 and later and .NET 6 and later 15 | - Use .NET paradigms, including asynchronous stream support 16 | - Full compatibility with both block and stream formats 17 | - Near C++ level performance 18 | - Note: This is only possible on .NET 6 and later with the aid of [Span<T>](https://docs.microsoft.com/en-us/dotnet/api/system.span-1?view=net-9.0) and [System.Runtime.Intrinsics](https://devblogs.microsoft.com/dotnet/dotnet-8-hardware-intrinsics/) 19 | - .NET 4.6.1 is the slowest 20 | - Keep allocations and garbage collection to a minimum using buffer pools 21 | 22 | ## License 23 | 24 | Copyright © 2011-2024, Google, Inc. and Snappier Authors. 25 | All rights reserved. 26 | 27 | Redistribution and use in source and binary forms, with or without 28 | modification, are permitted provided that the following conditions are 29 | met: 30 | 31 | * Redistributions of source code must retain the above copyright 32 | notice, this list of conditions and the following disclaimer. 33 | * Redistributions in binary form must reproduce the above 34 | copyright notice, this list of conditions and the following disclaimer 35 | in the documentation and/or other materials provided with the 36 | distribution. 37 | * Neither the name of Google, Inc., any Snappier authors, nor the 38 | names of its contributors may be used to endorse or promote products 39 | derived from this software without specific prior written permission. 40 | 41 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 42 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 43 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 44 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 45 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 46 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 47 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 48 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 49 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 50 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 51 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 52 | 53 | ## Other Projects 54 | 55 | There are other projects available for C#/.NET which implement Snappy compression. 56 | 57 | - [Snappy.NET](https://snappy.machinezoo.com/) - Uses P/Invoke to C++ for great performance. However, it only works on Windows, and is a bit heap allocation heavy in some cases. It also hasn't been updated since 2014 (as of 10/2020). This project may still be the best choice if your project is on the legacy .NET Framework on Windows, where Snappier is much less performant. 58 | - [IronSnappy](https://github.com/aloneguid/IronSnappy) - Another pure C# port, based on the Golang implementation instead of the C++ implementation. 59 | -------------------------------------------------------------------------------- /Snappier.Benchmarks/Configuration/VersionComparisonConfig.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Collections.Immutable; 4 | using System.Linq; 5 | using BenchmarkDotNet.Columns; 6 | using BenchmarkDotNet.Configs; 7 | using BenchmarkDotNet.Environments; 8 | using BenchmarkDotNet.Jobs; 9 | using BenchmarkDotNet.Order; 10 | using BenchmarkDotNet.Reports; 11 | using BenchmarkDotNet.Running; 12 | 13 | namespace Snappier.Benchmarks.Configuration; 14 | 15 | public class VersionComparisonConfig : StandardConfig 16 | { 17 | public VersionComparisonConfig(Job baseJob) 18 | { 19 | Job jobBefore = baseJob.WithCustomBuildConfiguration("Previous"); 20 | 21 | Job jobBefore80 = jobBefore.WithRuntime(CoreRuntime.Core80).WithPgo().AsBaseline(); 22 | Job jobBefore90 = jobBefore.WithRuntime(CoreRuntime.Core90).WithPgo().AsBaseline(); 23 | Job jobBefore10_0 = jobBefore.WithRuntime(CoreRuntime.Core10_0).WithPgo().AsBaseline(); 24 | 25 | Job jobAfter80 = baseJob.WithRuntime(CoreRuntime.Core80).WithPgo(); 26 | Job jobAfter90 = baseJob.WithRuntime(CoreRuntime.Core90).WithPgo(); 27 | Job jobAfter10_0 = baseJob.WithRuntime(CoreRuntime.Core10_0).WithPgo(); 28 | 29 | AddJob(jobBefore80); 30 | AddJob(jobBefore90); 31 | AddJob(jobBefore10_0); 32 | AddJob(jobAfter80); 33 | AddJob(jobAfter90); 34 | AddJob(jobAfter10_0); 35 | 36 | #if NET6_0_OR_GREATER // OperatingSystem check is only available in .NET 6.0 or later, but the runner itself won't be .NET 4 anyway 37 | if (OperatingSystem.IsWindows()) 38 | { 39 | Job jobBefore48 = jobBefore.WithRuntime(ClrRuntime.Net48).AsBaseline(); 40 | Job jobAfter48 = baseJob.WithRuntime(ClrRuntime.Net48); 41 | 42 | AddJob(jobBefore48); 43 | AddJob(jobAfter48); 44 | } 45 | #endif 46 | 47 | WithOrderer(VersionComparisonOrderer.Default); 48 | 49 | HideColumns(Column.EnvironmentVariables, Column.Job); 50 | } 51 | 52 | private class VersionComparisonOrderer : IOrderer 53 | { 54 | public static readonly IOrderer Default = new VersionComparisonOrderer(); 55 | 56 | public IEnumerable GetExecutionOrder(ImmutableArray benchmarksCase, 57 | IEnumerable order = null) => 58 | benchmarksCase 59 | .OrderBy(p => p.Job.Environment.Runtime.MsBuildMoniker) 60 | .ThenBy(p => PgoColumn.IsPgo(p) ? 1 : 0) 61 | .ThenBy(p => !p.Descriptor.Baseline) 62 | .ThenBy(p => p.DisplayInfo); 63 | 64 | public IEnumerable GetSummaryOrder(ImmutableArray benchmarksCases, 65 | Summary summary) => 66 | GetExecutionOrder(benchmarksCases); 67 | 68 | public string GetHighlightGroupKey(BenchmarkCase benchmarkCase) => null; 69 | 70 | public string GetLogicalGroupKey(ImmutableArray allBenchmarksCases, 71 | BenchmarkCase benchmarkCase) => 72 | $"{benchmarkCase.Job.Environment.Runtime.MsBuildMoniker}-Pgo={(PgoColumn.IsPgo(benchmarkCase) ? "Y" : "N")}-{benchmarkCase.Descriptor.MethodIndex}"; 73 | 74 | public IEnumerable> GetLogicalGroupOrder( 75 | IEnumerable> logicalGroups, 76 | IEnumerable order = null) => 77 | logicalGroups.OrderBy(p => p.Key); 78 | 79 | public bool SeparateLogicalGroups => true; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Snappier/Internal/Constants.cs: -------------------------------------------------------------------------------- 1 | namespace Snappier.Internal; 2 | 3 | internal static class Constants 4 | { 5 | public enum ChunkType : byte 6 | { 7 | CompressedData = 0x00, 8 | UncompressedData = 0x01, 9 | SkippableChunk = 0x80, // If this bit is set, we can safely skip the chunk if unknown 10 | Padding = 0xfe, 11 | StreamIdentifier = 0xff, 12 | 13 | // This is not part of the spec, but having this extra value representing null avoids 14 | // the cost of wrapping in a Nullable 15 | Null = 0xfd, 16 | } 17 | 18 | public const byte Literal = 0b00; 19 | public const byte Copy1ByteOffset = 1; // 3 bit length + 3 bits of offset in opcode 20 | public const byte Copy2ByteOffset = 2; 21 | public const byte Copy4ByteOffset = 3; 22 | 23 | public const int MaximumTagLength = 5; 24 | 25 | public const int BlockLog = 16; 26 | public const long BlockSize = 1 << BlockLog; 27 | public const nint InputMarginBytes = 15; 28 | 29 | /// 30 | /// Data stored per entry in lookup table: 31 | /// Range Bits-used Description 32 | /// ------------------------------------ 33 | /// 1..64 0..7 Literal/copy length encoded in opcode byte 34 | /// 0..7 8..10 Copy offset encoded in opcode byte / 256 35 | /// 0..4 11..13 Extra bytes after opcode 36 | /// 37 | /// We use eight bits for the length even though 7 would have sufficed 38 | /// because of efficiency reasons: 39 | /// (1) Extracting a byte is faster than a bit-field 40 | /// (2) It properly aligns copy offset so we do not need a <<8 41 | /// 42 | public static ReadOnlySpan CharTable => 43 | [ 44 | 0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002, 45 | 0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004, 46 | 0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006, 47 | 0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008, 48 | 0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a, 49 | 0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c, 50 | 0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e, 51 | 0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010, 52 | 0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012, 53 | 0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014, 54 | 0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016, 55 | 0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018, 56 | 0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a, 57 | 0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c, 58 | 0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e, 59 | 0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020, 60 | 0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022, 61 | 0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024, 62 | 0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026, 63 | 0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028, 64 | 0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a, 65 | 0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c, 66 | 0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e, 67 | 0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030, 68 | 0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032, 69 | 0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034, 70 | 0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036, 71 | 0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038, 72 | 0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a, 73 | 0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c, 74 | 0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e, 75 | 0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040 76 | ]; 77 | } 78 | -------------------------------------------------------------------------------- /Snappier/Internal/VarIntEncoding.Read.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | 3 | #if NET8_0_OR_GREATER 4 | using System.Buffers.Binary; 5 | using System.Diagnostics; 6 | using System.Numerics; 7 | using System.Runtime.InteropServices; 8 | using System.Runtime.Intrinsics; 9 | using System.Runtime.Intrinsics.X86; 10 | #endif 11 | 12 | namespace Snappier.Internal; 13 | 14 | internal static partial class VarIntEncoding 15 | { 16 | public static uint Read(ReadOnlySpan input, out int bytesRead) 17 | { 18 | if (TryRead(input, out uint result, out bytesRead) != OperationStatus.Done) 19 | { 20 | ThrowHelper.ThrowInvalidDataException("Invalid stream length"); 21 | } 22 | 23 | return result; 24 | } 25 | 26 | public static OperationStatus TryRead(ReadOnlySpan input, out uint result, out int bytesRead) 27 | { 28 | #if NET8_0_OR_GREATER 29 | if (Sse2.IsSupported && Bmi2.IsSupported && BitConverter.IsLittleEndian && input.Length >= Vector128.Count) 30 | { 31 | return ReadFast(input, out result, out bytesRead); 32 | } 33 | #endif 34 | 35 | return TryReadSlow(input, out result, out bytesRead); 36 | } 37 | 38 | private static OperationStatus TryReadSlow(ReadOnlySpan input, out uint result, out int bytesRead) 39 | { 40 | result = 0; 41 | int shift = 0; 42 | bool foundEnd = false; 43 | 44 | bytesRead = 0; 45 | while (input.Length > bytesRead) 46 | { 47 | byte c = input[bytesRead++]; 48 | 49 | int val = c & 0x7f; 50 | if (Helpers.LeftShiftOverflows((byte) val, shift)) 51 | { 52 | bytesRead = 0; 53 | return OperationStatus.InvalidData; 54 | } 55 | 56 | result |= (uint)(val << shift); 57 | shift += 7; 58 | 59 | if (c < 128) 60 | { 61 | foundEnd = true; 62 | break; 63 | } 64 | 65 | if (shift >= 32) 66 | { 67 | bytesRead = 0; 68 | return OperationStatus.InvalidData; 69 | } 70 | } 71 | 72 | if (!foundEnd) 73 | { 74 | bytesRead = 0; 75 | return OperationStatus.NeedMoreData; 76 | } 77 | 78 | return OperationStatus.Done; 79 | } 80 | 81 | #if NET8_0_OR_GREATER 82 | 83 | private static ReadOnlySpan ReadMasks => 84 | [ 85 | 0x00000000, // Not used, present for padding 86 | 0x0000007f, 87 | 0x00003fff, 88 | 0x001fffff, 89 | 0x0fffffff, 90 | 0xffffffff 91 | ]; 92 | 93 | private static OperationStatus ReadFast(ReadOnlySpan input, out uint result, out int bytesRead) 94 | { 95 | DebugExtensions.Assert(Sse2.IsSupported); 96 | DebugExtensions.Assert(Bmi2.IsSupported); 97 | DebugExtensions.Assert(input.Length >= Vector128.Count); 98 | DebugExtensions.Assert(BitConverter.IsLittleEndian); 99 | 100 | int mask = ~Sse2.MoveMask(Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(input))); 101 | bytesRead = BitOperations.TrailingZeroCount(mask) + 1; 102 | 103 | uint shuffledBits = Bmi2.X64.IsSupported 104 | ? unchecked((uint)Bmi2.X64.ParallelBitExtract(BinaryPrimitives.ReadUInt64LittleEndian(input), 0x7F7F7F7F7Fu)) 105 | : Bmi2.ParallelBitExtract(BinaryPrimitives.ReadUInt32LittleEndian(input), 0x7F7F7F7Fu) | 106 | ((BinaryPrimitives.ReadUInt32LittleEndian(input.Slice(4)) & 0xf) << 28); 107 | 108 | if (bytesRead < ReadMasks.Length) 109 | { 110 | result = shuffledBits & ReadMasks[bytesRead]; 111 | } 112 | else 113 | { 114 | // Currently, JIT doesn't optimize the bounds check away in the branch above, 115 | // but we'll leave it written this way in case JIT improves in the future to avoid 116 | // checking the bounds twice. 117 | 118 | result = 0; 119 | bytesRead = 0; 120 | return OperationStatus.InvalidData; 121 | } 122 | 123 | return OperationStatus.Done; 124 | } 125 | 126 | #endif 127 | } 128 | -------------------------------------------------------------------------------- /Snappier.Tests/Internal/VarIntEncodingReadTests.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | 3 | namespace Snappier.Tests.Internal; 4 | 5 | public class VarIntEncodingReadTests 6 | { 7 | public static TheoryData TestData() => 8 | new() { 9 | { 0x00, [ 0x00 ] }, 10 | { 0x01, [ 0x01 ] }, 11 | { 0x7F, [ 0x7F ] }, 12 | { 0x80, [ 0x80, 0x01 ] }, 13 | { 0x555, [ 0xD5, 0x0A ] }, 14 | { 0x7FFF, [ 0xFF, 0xFF, 0x01 ] }, 15 | { 0xBFFF, [ 0xFF, 0xFF, 0x02 ] }, 16 | { 0xFFFF, [ 0XFF, 0xFF, 0x03 ] }, 17 | { 0x8000, [ 0x80, 0x80, 0x02 ] }, 18 | { 0x5555, [ 0xD5, 0xAA, 0x01 ] }, 19 | { 0xCAFEF00, [ 0x80, 0xDE, 0xBF, 0x65 ] }, 20 | { 0xCAFEF00D, [ 0x8D, 0xE0, 0xFB, 0xD7, 0x0C ] }, 21 | { 0xFFFFFFFF, [ 0xFF, 0xFF, 0xFF, 0xFF, 0x0F ] }, 22 | }; 23 | 24 | public static TheoryData IncompleteTestData() => 25 | new() { 26 | { [ 0x80 ] }, 27 | { [ 0xD5 ] }, 28 | { [ 0xFF, 0xFF ] }, 29 | { [ 0xFF, 0xFF ] }, 30 | { [ 0XFF, 0xFF ] }, 31 | { [ 0x80, 0x80 ] }, 32 | { [ 0xD5, 0xAA ] }, 33 | { [ 0x80, 0xDE, 0xBF ] }, 34 | { [ 0x8D, 0xE0, 0xFB, 0xD7 ] }, 35 | { [ 0xFF, 0xFF, 0xFF, 0xFF ] }, 36 | }; 37 | 38 | [Theory] 39 | [MemberData(nameof(TestData))] 40 | public void Test_TryRead(uint expected, byte[] input) 41 | { 42 | OperationStatus status = VarIntEncoding.TryRead(input, out uint result, out int bytesRead); 43 | Assert.Equal(OperationStatus.Done, status); 44 | Assert.Equal(input.Length, bytesRead); 45 | Assert.Equal(expected, result); 46 | } 47 | 48 | [Theory] 49 | [MemberData(nameof(TestData))] 50 | public void Test_TryRead_ZeroPadding(uint expected, byte[] input) 51 | { 52 | byte[] bytes = new byte[16]; 53 | input.AsSpan().CopyTo(bytes); 54 | 55 | OperationStatus status = VarIntEncoding.TryRead(bytes, out uint result, out int bytesRead); 56 | Assert.Equal(OperationStatus.Done, status); 57 | Assert.Equal(input.Length, bytesRead); 58 | Assert.Equal(expected, result); 59 | } 60 | 61 | [Theory] 62 | [MemberData(nameof(TestData))] 63 | public void Test_TryRead_OnePadding(uint expected, byte[] input) 64 | { 65 | byte[] bytes = new byte[16]; 66 | bytes.AsSpan().Fill(0xff); 67 | input.AsSpan().CopyTo(bytes); 68 | 69 | OperationStatus status = VarIntEncoding.TryRead(bytes, out uint result, out int bytesRead); 70 | Assert.Equal(OperationStatus.Done, status); 71 | Assert.Equal(input.Length, bytesRead); 72 | Assert.Equal(expected, result); 73 | } 74 | 75 | [Theory] 76 | [MemberData(nameof(IncompleteTestData))] 77 | public void Test_TryRead_Incomplete(byte[] input) 78 | { 79 | OperationStatus status = VarIntEncoding.TryRead(input, out _, out int bytesRead); 80 | Assert.Equal(OperationStatus.NeedMoreData, status); 81 | Assert.Equal(0, bytesRead); 82 | } 83 | 84 | [Fact] 85 | public void Test_TryRead_BadData() 86 | { 87 | OperationStatus status = VarIntEncoding.TryRead([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF], out _, out int bytesRead); 88 | Assert.Equal(OperationStatus.InvalidData, status); 89 | Assert.Equal(0, bytesRead); 90 | } 91 | } 92 | 93 | /* ************************************************************ 94 | * 95 | * @author Couchbase 96 | * @copyright 2021 Couchbase, Inc. 97 | * 98 | * Licensed under the Apache License, Version 2.0 (the "License"); 99 | * you may not use this file except in compliance with the License. 100 | * You may obtain a copy of the License at 101 | * 102 | * http://www.apache.org/licenses/LICENSE-2.0 103 | * 104 | * Unless required by applicable law or agreed to in writing, software 105 | * distributed under the License is distributed on an "AS IS" BASIS, 106 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 107 | * See the License for the specific language governing permissions and 108 | * limitations under the License. 109 | * 110 | * ************************************************************/ 111 | -------------------------------------------------------------------------------- /Snappier/Internal/HashTable.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using System.Diagnostics; 3 | using System.Runtime.CompilerServices; 4 | 5 | #if NET8_0_OR_GREATER 6 | using System.Runtime.Intrinsics.Arm; 7 | using System.Runtime.Intrinsics.X86; 8 | #endif 9 | 10 | namespace Snappier.Internal; 11 | 12 | internal class HashTable : IDisposable 13 | { 14 | private const int MinHashTableBits = 8; 15 | private const int MinHashTableSize = 1 << MinHashTableBits; 16 | 17 | private const int MaxHashTableBits = 14; 18 | private const int MaxHashTableSize = 1 << MaxHashTableBits; 19 | 20 | private ushort[]? _buffer; 21 | 22 | public void EnsureCapacity(long inputSize) 23 | { 24 | int maxFragmentSize = (int) Math.Min(inputSize, Constants.BlockSize); 25 | int tableSize = CalculateTableSize(maxFragmentSize); 26 | 27 | if (_buffer is null || tableSize > _buffer.Length) 28 | { 29 | if (_buffer is not null) 30 | { 31 | ArrayPool.Shared.Return(_buffer); 32 | } 33 | 34 | _buffer = ArrayPool.Shared.Rent(tableSize); 35 | } 36 | } 37 | 38 | public Span GetHashTable(int fragmentSize) 39 | { 40 | if (_buffer is null) 41 | { 42 | ThrowHelper.ThrowInvalidOperationException("Buffer not initialized"); 43 | } 44 | 45 | int hashTableSize = CalculateTableSize(fragmentSize); 46 | if (hashTableSize > _buffer.Length) 47 | { 48 | ThrowHelper.ThrowInvalidOperationException("Insufficient buffer size"); 49 | } 50 | 51 | Span hashTable = _buffer.AsSpan(0, hashTableSize); 52 | hashTable.Clear(); 53 | 54 | return hashTable; 55 | } 56 | 57 | private static int CalculateTableSize(int inputSize) 58 | { 59 | if (inputSize > MaxHashTableSize) 60 | { 61 | return MaxHashTableSize; 62 | } 63 | 64 | if (inputSize < MinHashTableSize) 65 | { 66 | return MinHashTableSize; 67 | } 68 | 69 | DebugExtensions.Assert(inputSize > 1); 70 | return 2 << Helpers.Log2Floor((uint)(inputSize - 1)); 71 | } 72 | 73 | public void Dispose() 74 | { 75 | if (_buffer is not null) 76 | { 77 | ArrayPool.Shared.Return(_buffer); 78 | _buffer = null; 79 | } 80 | } 81 | 82 | /// 83 | /// Given a table of uint16_t whose size is mask / 2 + 1, return a pointer to the 84 | /// relevant entry, if any, for the given bytes. Any hash function will do, 85 | /// but a good hash function reduces the number of collisions and thus yields 86 | /// better compression for compressible input. 87 | /// 88 | /// REQUIRES: mask is 2 * (table_size - 1), and table_size is a power of two. 89 | /// 90 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 91 | public static ref ushort TableEntry(ref ushort table, uint bytes, uint mask) 92 | { 93 | // Our choice is quicker-and-dirtier than the typical hash function; 94 | // empirically, that seems beneficial. The upper bits of kMagic * bytes are a 95 | // higher-quality hash than the lower bits, so when using kMagic * bytes we 96 | // also shift right to get a higher-quality end result. There's no similar 97 | // issue with a CRC because all of the output bits of a CRC are equally good 98 | // "hashes." So, a CPU instruction for CRC, if available, tends to be a good 99 | // choice. 100 | 101 | uint hash; 102 | 103 | #if NET8_0_OR_GREATER 104 | // We use mask as the second arg to the CRC function, as it's about to 105 | // be used anyway; it'd be equally correct to use 0 or some constant. 106 | // Mathematically, _mm_crc32_u32 (or similar) is a function of the 107 | // xor of its arguments. 108 | 109 | if (System.Runtime.Intrinsics.X86.Sse42.IsSupported) 110 | { 111 | hash = Sse42.Crc32(bytes, mask); 112 | 113 | } 114 | else if (System.Runtime.Intrinsics.Arm.Crc32.IsSupported) 115 | { 116 | hash = Crc32.ComputeCrc32C(bytes, mask); 117 | } 118 | else 119 | #endif 120 | { 121 | const uint kMagic = 0x1e35a7bd; 122 | hash = (kMagic * bytes) >> (31 - MaxHashTableBits); 123 | } 124 | 125 | return ref Unsafe.AddByteOffset(ref table, hash & mask); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /Snappier.Tests/Internal/SnappyCompressorTests.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | using System.Text; 3 | 4 | namespace Snappier.Tests.Internal; 5 | 6 | public class SnappyCompressorTests 7 | { 8 | #region FindMatchLength 9 | 10 | [Theory] 11 | [InlineData(6, "012345", "012345", 6)] 12 | [InlineData(11, "01234567abc", "01234567abc", 11)] 13 | 14 | // Hit s1_limit in 64-bit loop, find a non-match in single-character loop. 15 | [InlineData(9, "01234567abc", "01234567axc", 9)] 16 | 17 | // Same, but edge cases. 18 | [InlineData(11, "01234567abc!", "01234567abc!", 11)] 19 | [InlineData(11, "01234567abc!", "01234567abc?", 11)] 20 | 21 | // Find non-match at once in first loop. 22 | [InlineData(0, "01234567xxxxxxxx", "?1234567xxxxxxxx", 16)] 23 | [InlineData(1, "01234567xxxxxxxx", "0?234567xxxxxxxx", 16)] 24 | [InlineData(4, "01234567xxxxxxxx", "01237654xxxxxxxx", 16)] 25 | [InlineData(7, "01234567xxxxxxxx", "0123456?xxxxxxxx", 16)] 26 | 27 | // Find non-match in first loop after one block. 28 | [InlineData(8, "abcdefgh01234567xxxxxxxx", "abcdefgh?1234567xxxxxxxx", 24)] 29 | [InlineData(9, "abcdefgh01234567xxxxxxxx", "abcdefgh0?234567xxxxxxxx", 24)] 30 | [InlineData(12, "abcdefgh01234567xxxxxxxx", "abcdefgh01237654xxxxxxxx", 24)] 31 | [InlineData(15, "abcdefgh01234567xxxxxxxx", "abcdefgh0123456?xxxxxxxx", 24)] 32 | 33 | // 32-bit version: 34 | 35 | // Short matches. 36 | [InlineData(0, "01234567", "?1234567", 8)] 37 | [InlineData(1, "01234567", "0?234567", 8)] 38 | [InlineData(2, "01234567", "01?34567", 8)] 39 | [InlineData(3, "01234567", "012?4567", 8)] 40 | [InlineData(4, "01234567", "0123?567", 8)] 41 | [InlineData(5, "01234567", "01234?67", 8)] 42 | [InlineData(6, "01234567", "012345?7", 8)] 43 | [InlineData(7, "01234567", "0123456?", 8)] 44 | [InlineData(7, "01234567", "0123456?", 7)] 45 | [InlineData(7, "01234567!", "0123456??", 7)] 46 | 47 | // Hit s1_limit in 32-bit loop, hit s1_limit in single-character loop. 48 | [InlineData(10, "xxxxxxabcd", "xxxxxxabcd", 10)] 49 | [InlineData(10, "xxxxxxabcd?", "xxxxxxabcd?", 10)] 50 | [InlineData(13, "xxxxxxabcdef", "xxxxxxabcdefx", 13)] 51 | 52 | // Same, but edge cases. 53 | [InlineData(12, "xxxxxx0123abc!", "xxxxxx0123abc!", 12)] 54 | [InlineData(12, "xxxxxx0123abc!", "xxxxxx0123abc?", 12)] 55 | 56 | // Hit s1_limit in 32-bit loop, find a non-match in single-character loop. 57 | [InlineData(11, "xxxxxx0123abc", "xxxxxx0123axc", 13)] 58 | 59 | // Find non-match at once in first loop. 60 | [InlineData(6, "xxxxxx0123xxxxxxxx", "xxxxxx?123xxxxxxxx", 18)] 61 | [InlineData(7, "xxxxxx0123xxxxxxxx", "xxxxxx0?23xxxxxxxx", 18)] 62 | [InlineData(8, "xxxxxx0123xxxxxxxx", "xxxxxx0132xxxxxxxx", 18)] 63 | [InlineData(9, "xxxxxx0123xxxxxxxx", "xxxxxx012?xxxxxxxx", 18)] 64 | 65 | // Same, but edge cases. 66 | [InlineData(6, "xxxxxx0123", "xxxxxx?123", 10)] 67 | [InlineData(7, "xxxxxx0123", "xxxxxx0?23", 10)] 68 | [InlineData(8, "xxxxxx0123", "xxxxxx0132", 10)] 69 | [InlineData(9, "xxxxxx0123", "xxxxxx012?", 10)] 70 | 71 | // Find non-match in first loop after one block. 72 | [InlineData(10, "xxxxxxabcd0123xx", "xxxxxxabcd?123xx", 16)] 73 | [InlineData(11, "xxxxxxabcd0123xx", "xxxxxxabcd0?23xx", 16)] 74 | [InlineData(12, "xxxxxxabcd0123xx", "xxxxxxabcd0132xx", 16)] 75 | [InlineData(13, "xxxxxxabcd0123xx", "xxxxxxabcd012?xx", 16)] 76 | 77 | // Same, but edge cases. 78 | [InlineData(10, "xxxxxxabcd0123", "xxxxxxabcd?123", 14)] 79 | [InlineData(11, "xxxxxxabcd0123", "xxxxxxabcd0?23", 14)] 80 | [InlineData(12, "xxxxxxabcd0123", "xxxxxxabcd0132", 14)] 81 | [InlineData(13, "xxxxxxabcd0123", "xxxxxxabcd012?", 14)] 82 | public void FindMatchLength(int expectedResult, string s1String, string s2String, int length) 83 | { 84 | byte[] array = Encoding.ASCII.GetBytes(s1String + s2String 85 | + new string('\0', Math.Max(0, length - s2String.Length))); 86 | 87 | ulong data = 0; 88 | ref readonly byte s1 = ref array[0]; 89 | ref readonly byte s2 = ref Unsafe.Add(in s1, s1String.Length); 90 | 91 | (int matchLength, bool matchLengthLessThan8) = 92 | SnappyCompressor.FindMatchLength(in s1, in s2, in Unsafe.Add(in s2, length), ref data); 93 | 94 | Assert.Equal(matchLength < 8, matchLengthLessThan8); 95 | Assert.Equal(expectedResult, matchLength); 96 | } 97 | 98 | #endregion 99 | } 100 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Snappier 2 | 3 | ## Introduction 4 | 5 | Snappier is a pure C# port of Google's [Snappy](https://github.com/google/snappy) compression algorithm. It is designed with speed as the primary goal, rather than compression ratio, and is ideal for compressing network traffic. Please see [the Snappy README file](https://github.com/google/snappy/blob/master/README.md) for more details on Snappy. 6 | 7 | Complete documentation is available at https://brantburnett.github.io/Snappier/. 8 | 9 | ## Project Goals 10 | 11 | The Snappier project aims to meet the following needs of the .NET community. 12 | 13 | - Cross-platform C# implementation for Linux and Windows, without P/Invoke or special OS installation requirements 14 | - Compatible with .NET 4.6.1 and later and .NET 6 and later 15 | - Use .NET paradigms, including asynchronous stream support 16 | - Full compatibility with both block and stream formats 17 | - Near C++ level performance 18 | - Note: This is only possible on .NET 6 and later with the aid of [Span<T>](https://docs.microsoft.com/en-us/dotnet/api/system.span-1?view=net-9.0) and [System.Runtime.Intrinsics](https://devblogs.microsoft.com/dotnet/dotnet-8-hardware-intrinsics/) 19 | - .NET 4.6.1 is the slowest 20 | - Keep allocations and garbage collection to a minimum using buffer pools 21 | 22 | ## Installing 23 | 24 | Simply add a NuGet package reference to the latest version of Snappier. 25 | 26 | ```xml 27 | 28 | ``` 29 | 30 | or 31 | 32 | ```sh 33 | dotnet add package Snappier 34 | ``` 35 | 36 | ## Block compression/decompression using a memory pool buffer 37 | 38 | ```cs 39 | using Snappier; 40 | 41 | public class Program 42 | { 43 | private static byte[] Data = {0, 1, 2}; // Wherever you get the data from 44 | 45 | public static void Main() 46 | { 47 | // This option uses `MemoryPool.Shared`. However, if you fail to 48 | // dispose of the returned buffers correctly it can result in inefficient garbage collection. 49 | // It is important to either call .Dispose() or use a using statement. 50 | 51 | // Compression 52 | using (IMemoryOwner compressed = Snappy.CompressToMemory(Data)) 53 | { 54 | // Decompression 55 | using (IMemoryOwner decompressed = Snappy.DecompressToMemory(compressed.Memory.Span)) 56 | { 57 | // Do something with the data 58 | } 59 | } 60 | } 61 | } 62 | ``` 63 | 64 | ## Stream compression/decompression 65 | 66 | Compressing or decompressing a stream follows the same paradigm as other compression streams in .NET. `SnappyStream` wraps an inner stream. If decompressing you read from the `SnappyStream`, if compressing you write to the `SnappyStream` 67 | 68 | This approach reads or writes the [Snappy framing format](https://github.com/google/snappy/blob/master/framing_format.txt) designed for streaming. The input/output is not the same as the block method above. It includes additional headers and CRC32C checks. 69 | 70 | ```cs 71 | using System.IO; 72 | using System.IO.Compression; 73 | using Snappier; 74 | 75 | public class Program 76 | { 77 | public static async Task Main() 78 | { 79 | using var fileStream = File.OpenRead("somefile.txt"); 80 | 81 | // First, compression 82 | using var compressed = new MemoryStream(); 83 | 84 | using (var compressor = new SnappyStream(compressed, CompressionMode.Compress, leaveOpen: true)) 85 | { 86 | await fileStream.CopyToAsync(compressor); 87 | 88 | // Disposing the compressor also flushes the buffers to the inner stream 89 | // We pass true to the constructor above so that it doesn't close/dispose the inner stream 90 | // Alternatively, we could call compressor.Flush() 91 | } 92 | 93 | // Then, decompression 94 | 95 | compressed.Position = 0; // Reset to beginning of the stream so we can read 96 | using var decompressor = new SnappyStream(compressed, CompressionMode.Decompress); 97 | 98 | var buffer = new byte[65536]; 99 | var bytesRead = decompressor.Read(buffer, 0, buffer.Length); 100 | while (bytesRead > 0) 101 | { 102 | // Do something with the data 103 | 104 | bytesRead = decompressor.Read(buffer, 0, buffer.Length) 105 | } 106 | } 107 | } 108 | ``` 109 | 110 | ## Other Projects 111 | 112 | There are other projects available for C#/.NET which implement Snappy compression. 113 | 114 | - [Snappy.NET](https://snappy.machinezoo.com/) - Uses P/Invoke to C++ for great performance. However, it only works on Windows, is a bit heap allocation heavy in some cases, and is a deprecated project. This project may still be the best choice if your project is on the legacy .NET Framework on Windows, where Snappier is much less performant. 115 | - [IronSnappy](https://www.nuget.org/packages/IronSnappy) - Another pure C# port, based on the Golang implementation instead of the C++ implementation. 116 | -------------------------------------------------------------------------------- /docs/block.md: -------------------------------------------------------------------------------- 1 | # Block Compression 2 | 3 | Block compression is ideal for data up to 64KB, though it may be used for data of any size. It does not include any stream 4 | framing or CRC validation. It also doesn't automatically revert to uncompressed data in the event of data size growth. 5 | 6 | ## Block compression/decompression using a buffer you already own 7 | 8 | ```cs 9 | using Snappier; 10 | 11 | public class Program 12 | { 13 | private static byte[] Data = {0, 1, 2}; // Wherever you get the data from 14 | 15 | public static void Main() 16 | { 17 | // This option assumes that you are managing buffers yourself in an efficient way. 18 | // In this example, we're using heap allocated byte arrays, however in most cases 19 | // you would get these buffers from a buffer pool like ArrayPool or MemoryPool. 20 | 21 | // If the output buffer is too small, an ArgumentException is thrown. This will not 22 | // occur in this example because a sufficient buffer is always allocated via 23 | // Snappy.GetMaxCompressedLength or Snappy.GetUncompressedLength. There are TryCompress 24 | // and TryDecompress overloads that return false if the output buffer is too small 25 | // rather than throwing an exception. 26 | 27 | // Compression 28 | byte[] buffer = new byte[Snappy.GetMaxCompressedLength(Data)]; 29 | int compressedLength = Snappy.Compress(Data, buffer); 30 | Span compressed = buffer.AsSpan(0, compressedLength); 31 | 32 | // Decompression 33 | byte[] outputBuffer = new byte[Snappy.GetUncompressedLength(compressed)]; 34 | int decompressedLength = Snappy.Decompress(compressed, outputBuffer); 35 | 36 | for (var i = 0; i < decompressedLength; i++) 37 | { 38 | // Do something with the data 39 | } 40 | } 41 | } 42 | ``` 43 | 44 | ## Block compression/decompression using a memory pool buffer 45 | 46 | ```cs 47 | using Snappier; 48 | 49 | public class Program 50 | { 51 | private static byte[] Data = {0, 1, 2}; // Wherever you get the data from 52 | 53 | public static void Main() 54 | { 55 | // This option uses `MemoryPool.Shared`. However, if you fail to 56 | // dispose of the returned buffers correctly it can result in inefficient garbage collection. 57 | // It is important to either call .Dispose() or use a using statement. 58 | 59 | // Compression 60 | using (IMemoryOwner compressed = Snappy.CompressToMemory(Data)) 61 | { 62 | // Decompression 63 | using (IMemoryOwner decompressed = Snappy.DecompressToMemory(compressed.Memory.Span)) 64 | { 65 | // Do something with the data 66 | } 67 | } 68 | } 69 | } 70 | ``` 71 | 72 | ## Block compression/decompression using a buffer writter 73 | 74 | ```cs 75 | using Snappier; 76 | using System.Buffers; 77 | 78 | public class Program 79 | { 80 | private static byte[] Data = {0, 1, 2}; // Wherever you get the data from 81 | 82 | public static void Main() 83 | { 84 | // This option uses `IBufferWriter`. In .NET 6 you can get a simple 85 | // implementation such as `ArrayBufferWriter` but it may also be a `PipeWriter` 86 | // or any other more advanced implementation of `IBufferWriter`. 87 | 88 | // These overloads also accept a `ReadOnlySequence` which allows the source data 89 | // to be made up of buffer segments rather than one large buffer. However, segment size 90 | // may be a factor in performance. For compression, segments that are some multiple of 91 | // 64KB are recommended. For decompression, simply avoid small segments. 92 | 93 | // Compression 94 | var compressedBufferWriter = new ArrayBufferWriter(); 95 | Snappy.Compress(new ReadOnlySequence(Data), compressedBufferWriter); 96 | var compressedData = compressedBufferWriter.WrittenMemory; 97 | 98 | // Decompression 99 | var decompressedBufferWriter = new ArrayBufferWriter(); 100 | Snappy.Decompress(new ReadOnlySequence(compressedData), decompressedBufferWriter); 101 | var decompressedData = decompressedBufferWriter.WrittenMemory; 102 | 103 | // Do something with the data 104 | } 105 | } 106 | ``` 107 | 108 | ## Block compression/decompression using heap allocated byte[] 109 | 110 | ```cs 111 | using Snappier; 112 | 113 | public class Program 114 | { 115 | private static byte[] Data = {0, 1, 2}; // Wherever you get the data from 116 | 117 | public static void Main() 118 | { 119 | // This is generally the least efficient option, 120 | // but in some cases may be the simplest to implement. 121 | 122 | // Compression 123 | byte[] compressed = Snappy.CompressToArray(Data); 124 | 125 | // Decompression 126 | byte[] decompressed = Snappy.DecompressToArray(compressed); 127 | } 128 | } 129 | ``` -------------------------------------------------------------------------------- /Snappier.Tests/Internal/SnappyDecompressorTests.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | 3 | namespace Snappier.Tests.Internal; 4 | 5 | public class SnappyDecompressorTests 6 | { 7 | #region Decompress 8 | 9 | [Fact] 10 | public void Decompress_SplitLength_Succeeds() 11 | { 12 | // Arrange 13 | 14 | using var decompressor = new SnappyDecompressor(); 15 | 16 | // Requires 3 bytes to varint encode the length 17 | byte[] data = new byte[65536]; 18 | using IMemoryOwner compressed = Snappy.CompressToMemory(data); 19 | 20 | // Act 21 | 22 | decompressor.Decompress(compressed.Memory.Span.Slice(0, 1)); 23 | Assert.True(decompressor.NeedMoreData); 24 | decompressor.Decompress(compressed.Memory.Span.Slice(1, 1)); 25 | Assert.True(decompressor.NeedMoreData); 26 | decompressor.Decompress(compressed.Memory.Span.Slice(2)); 27 | Assert.False(decompressor.NeedMoreData); 28 | 29 | using IMemoryOwner result = decompressor.ExtractData(); 30 | 31 | // Assert 32 | 33 | Assert.Equal(65536, result.Memory.Length); 34 | Assert.True(result.Memory.Span.SequenceEqual(data)); 35 | } 36 | 37 | #endregion 38 | 39 | #region DecompressAllTags 40 | 41 | [Fact] 42 | public void DecompressAllTags_ShortInputBufferWhichCopiesToScratch_DoesNotReadPastEndOfScratch() 43 | { 44 | // Arrange 45 | 46 | var decompressor = new SnappyDecompressor(); 47 | decompressor.SetExpectedLengthForTest(1024); 48 | 49 | decompressor.WriteToBufferForTest(Enumerable.Range(0, 255).Select(p => (byte) p).ToArray()); 50 | 51 | // if in error, decompressor will read the 222, 0, 0 as the next tag and throw a copy offset exception 52 | decompressor.LoadScratchForTest([222, 222, 222, 222, 0, 0], 0); 53 | 54 | // Act 55 | 56 | decompressor.DecompressAllTags([150, 255, 0]); 57 | 58 | } 59 | 60 | #endregion 61 | 62 | #region ExtractData 63 | 64 | [Fact] 65 | public void ExtractData_NoLength_InvalidOperationException() 66 | { 67 | // Arrange 68 | 69 | using var decompressor = new SnappyDecompressor(); 70 | 71 | // Act/Assert 72 | 73 | InvalidOperationException ex = Assert.Throws(() => decompressor.ExtractData()); 74 | 75 | Assert.Equal("No data present.", ex.Message); 76 | } 77 | 78 | [Fact] 79 | public void ExtractData_NotFullDecompressed_InvalidOperationException() 80 | { 81 | // Arrange 82 | 83 | using var decompressor = new SnappyDecompressor(); 84 | 85 | using IMemoryOwner compressed = Snappy.CompressToMemory([1, 2, 3, 4]); 86 | 87 | // Only length is forwarded 88 | decompressor.Decompress(compressed.Memory.Span.Slice(0, 1)); 89 | 90 | // Act/Assert 91 | 92 | InvalidOperationException ex = Assert.Throws(() => decompressor.ExtractData()); 93 | Assert.Equal("Block is not fully decompressed.", ex.Message); 94 | } 95 | 96 | [Fact] 97 | public void ExtractData_ZeroLength_EmptyMemory() 98 | { 99 | // Arrange 100 | 101 | using var decompressor = new SnappyDecompressor(); 102 | 103 | using IMemoryOwner compressed = Snappy.CompressToMemory([]); 104 | 105 | decompressor.Decompress(compressed.Memory.Span); 106 | 107 | // Act 108 | 109 | using IMemoryOwner result = decompressor.ExtractData(); 110 | 111 | // Assert 112 | 113 | Assert.Equal(0, result.Memory.Length); 114 | } 115 | 116 | [Fact] 117 | public void ExtractData_SomeData_Memory() 118 | { 119 | // Arrange 120 | 121 | using var decompressor = new SnappyDecompressor(); 122 | 123 | using IMemoryOwner compressed = Snappy.CompressToMemory([1, 2, 3, 4]); 124 | 125 | decompressor.Decompress(compressed.Memory.Span); 126 | 127 | // Act 128 | 129 | using IMemoryOwner result = decompressor.ExtractData(); 130 | 131 | // Assert 132 | 133 | Assert.Equal(4, result.Memory.Length); 134 | } 135 | 136 | [Fact] 137 | public void ExtractData_SomeData_DoesResetForReuse() 138 | { 139 | // Arrange 140 | 141 | using var decompressor = new SnappyDecompressor(); 142 | 143 | using IMemoryOwner compressed = Snappy.CompressToMemory([1, 2, 3, 4]); 144 | using IMemoryOwner compressed2 = Snappy.CompressToMemory([4, 3, 2, 1]); 145 | 146 | decompressor.Decompress(compressed.Memory.Span); 147 | 148 | // Act 149 | 150 | using IMemoryOwner result = decompressor.ExtractData(); 151 | 152 | decompressor.Decompress(compressed2.Memory.Span); 153 | 154 | using IMemoryOwner result2 = decompressor.ExtractData(); 155 | 156 | // Assert 157 | 158 | Assert.Equal(4, result2.Memory.Length); 159 | Assert.Equal(new byte[] {4, 3, 2, 1}, result2.Memory.ToArray() ); 160 | } 161 | 162 | #endregion 163 | } 164 | -------------------------------------------------------------------------------- /Snappier/Internal/Crc32CAlgorithm.cs: -------------------------------------------------------------------------------- 1 | using System.Runtime.CompilerServices; 2 | 3 | #if NET8_0_OR_GREATER 4 | using System.Runtime.InteropServices; 5 | using System.Runtime.Intrinsics.Arm; 6 | using System.Runtime.Intrinsics.X86; 7 | #endif 8 | 9 | namespace Snappier.Internal; 10 | 11 | internal static class Crc32CAlgorithm 12 | { 13 | #region static 14 | 15 | private const uint Poly = 0x82F63B78u; 16 | 17 | #pragma warning disable IDE1006 18 | // ReSharper disable once InconsistentNaming 19 | private static readonly uint[] Table; 20 | #pragma warning restore IDE1006 21 | 22 | static Crc32CAlgorithm() 23 | { 24 | uint[] table = new uint[16 * 256]; 25 | for (uint i = 0; i < 256; i++) 26 | { 27 | uint res = i; 28 | for (int t = 0; t < 16; t++) 29 | { 30 | for (int k = 0; k < 8; k++) res = (res & 1) == 1 ? Poly ^ (res >> 1) : (res >> 1); 31 | table[(t * 256) + i] = res; 32 | } 33 | } 34 | 35 | Table = table; 36 | } 37 | 38 | #endregion 39 | 40 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 41 | public static uint Compute(ReadOnlySpan source) 42 | { 43 | return Append(0, source); 44 | } 45 | 46 | public static uint Append(uint crc, ReadOnlySpan source) 47 | { 48 | uint crcLocal = uint.MaxValue ^ crc; 49 | 50 | #if NET8_0_OR_GREATER 51 | // If available on the current CPU, use ARM CRC32C intrinsic operations. 52 | // The if Crc32 statements are optimized out by the JIT compiler based on CPU support. 53 | if (Crc32.IsSupported) 54 | { 55 | if (Crc32.Arm64.IsSupported) 56 | { 57 | while (source.Length >= 8) 58 | { 59 | crcLocal = Crc32.Arm64.ComputeCrc32C(crcLocal, MemoryMarshal.Read(source)); 60 | source = source.Slice(8); 61 | } 62 | } 63 | 64 | // Process in 4-byte chunks 65 | while (source.Length >= 4) 66 | { 67 | crcLocal = Crc32.ComputeCrc32C(crcLocal, MemoryMarshal.Read(source)); 68 | source = source.Slice(4); 69 | } 70 | 71 | // Process the remainder 72 | int j = 0; 73 | while (j < source.Length) 74 | { 75 | crcLocal = Crc32.ComputeCrc32C(crcLocal, source[j++]); 76 | } 77 | 78 | return crcLocal ^ uint.MaxValue; 79 | } 80 | 81 | // If available on the current CPU, use Intel CRC32C intrinsic operations. 82 | // The Sse42 if statements are optimized out by the JIT compiler based on CPU support. 83 | else if (Sse42.IsSupported) 84 | { 85 | // Process in 8-byte chunks first if 64-bit 86 | if (Sse42.X64.IsSupported) 87 | { 88 | if (source.Length >= 8) 89 | { 90 | // work with a ulong local during the loop to reduce typecasts 91 | ulong crcLocalLong = crcLocal; 92 | 93 | while (source.Length >= 8) 94 | { 95 | crcLocalLong = Sse42.X64.Crc32(crcLocalLong, MemoryMarshal.Read(source)); 96 | source = source.Slice(8); 97 | } 98 | 99 | crcLocal = (uint) crcLocalLong; 100 | } 101 | } 102 | 103 | // Process in 4-byte chunks 104 | while (source.Length >= 4) 105 | { 106 | crcLocal = Sse42.Crc32(crcLocal, MemoryMarshal.Read(source)); 107 | source = source.Slice(4); 108 | } 109 | 110 | // Process the remainder 111 | int j = 0; 112 | while (j < source.Length) 113 | { 114 | crcLocal = Sse42.Crc32(crcLocal, source[j++]); 115 | } 116 | 117 | return crcLocal ^ uint.MaxValue; 118 | } 119 | #endif 120 | 121 | uint[] table = Table; 122 | while (source.Length >= 16) 123 | { 124 | uint a = table[(3 * 256) + source[12]] 125 | ^ table[(2 * 256) + source[13]] 126 | ^ table[(1 * 256) + source[14]] 127 | ^ table[(0 * 256) + source[15]]; 128 | 129 | uint b = table[(7 * 256) + source[8]] 130 | ^ table[(6 * 256) + source[9]] 131 | ^ table[(5 * 256) + source[10]] 132 | ^ table[(4 * 256) + source[11]]; 133 | 134 | uint c = table[(11 * 256) + source[4]] 135 | ^ table[(10 * 256) + source[5]] 136 | ^ table[(9 * 256) + source[6]] 137 | ^ table[(8 * 256) + source[7]]; 138 | 139 | uint d = table[(15 * 256) + ((byte)crcLocal ^ source[0])] 140 | ^ table[(14 * 256) + ((byte)(crcLocal >> 8) ^ source[1])] 141 | ^ table[(13 * 256) + ((byte)(crcLocal >> 16) ^ source[2])] 142 | ^ table[(12 * 256) + ((crcLocal >> 24) ^ source[3])]; 143 | 144 | crcLocal = d ^ c ^ b ^ a; 145 | source = source.Slice(16); 146 | } 147 | 148 | for (int offset = 0; offset < source.Length; offset++) 149 | { 150 | crcLocal = table[(byte) (crcLocal ^ source[offset])] ^ crcLocal >> 8; 151 | } 152 | 153 | return crcLocal ^ uint.MaxValue; 154 | } 155 | 156 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 157 | public static uint ApplyMask(uint x) => 158 | unchecked(((x >> 15) | (x << 17)) + 0xa282ead8); 159 | } 160 | -------------------------------------------------------------------------------- /Snappier/Internal/VarIntEncoding.WriteFast.cs: -------------------------------------------------------------------------------- 1 | #if NET8_0_OR_GREATER 2 | using System.Numerics; 3 | using System.Runtime.CompilerServices; 4 | using System.Runtime.InteropServices; 5 | using System.Runtime.Intrinsics.X86; 6 | #endif 7 | 8 | /* 9 | * This file is ported from https://github.com/couchbase/couchbase-net-client/blob/c10fe9ef09beadb8512f696d764b7a770429e641/src/Couchbase/Core/Utils/Leb128.cs 10 | * and therefore retains a Couchbase copyright. 11 | **/ 12 | 13 | namespace Snappier.Internal; 14 | 15 | internal static partial class VarIntEncoding 16 | { 17 | /// 18 | /// Maximum length, in bytes, when encoding a 32-bit integer. 19 | /// 20 | public const int MaxLength = 5; 21 | 22 | /// 23 | /// Encodes a value onto a buffer using little-ending varint encoding. 24 | /// 25 | /// Buffer to receive the value. 26 | /// Value to encode. 27 | /// Number of bytes written to the buffer. 28 | /// if the value was written successfully. if the buffer is too small. 29 | public static bool TryWrite(Span buffer, uint value, out int bytesWritten) 30 | { 31 | // Note: This method is likely to be inlined into the caller, potentially 32 | // eliding the size check if JIT knows the size of the buffer. BitConverter.IsLittleEndian 33 | // will always be elided based on CPU architecture. 34 | 35 | #if NET8_0_OR_GREATER 36 | if (BitConverter.IsLittleEndian && buffer.Length >= sizeof(ulong)) 37 | { 38 | // Only use the fast path on little-endian CPUs and when there's enough padding in the 39 | // buffer to write an ulong. At most there will be 5 real bytes written, but for speed 40 | // up to 8 bytes are being copied to the buffer from a register. This guard prevents a 41 | // potential buffer overrun. 42 | 43 | bytesWritten = WriteFast(ref MemoryMarshal.GetReference(buffer), value); 44 | return true; 45 | } 46 | #endif 47 | 48 | return TryWriteSlow(buffer, value, out bytesWritten); 49 | } 50 | 51 | #if NET8_0_OR_GREATER 52 | 53 | private static int WriteFast(ref byte buffer, uint value) 54 | { 55 | // The use of unsafe writes below is made safe because this method is never 56 | // called without at least 8 bytes available in the buffer. 57 | 58 | if (value < 128) 59 | { 60 | // We need to special case 0 to ensure we write one byte, so go ahead and 61 | // special case 0-127, which all write only one byte with the continuation bit unset. 62 | 63 | buffer = (byte)value; 64 | return 1; 65 | } 66 | 67 | // First get the value spread onto an ulong with 7 bit groups 68 | 69 | ulong result = Spread7BitGroupsIntoBytes(value); 70 | 71 | // Next, calculate the size of the output in bytes 72 | 73 | int unusedBytes = BitOperations.LeadingZeroCount(result) >>> 3; // right shift is the equivalent of divide by 8 74 | 75 | // Build a mask to set the continuation bits 76 | 77 | const ulong allContinuationBits = 0x8080808080808080UL; 78 | ulong mask = allContinuationBits >>> ((unusedBytes + 1) << 3); // left shift is the equivalent of multiply by 8 79 | 80 | // Finally, write the result to the buffer 81 | 82 | Unsafe.WriteUnaligned(ref buffer, result | mask); 83 | 84 | return sizeof(ulong) - unusedBytes; 85 | } 86 | 87 | // This spreads the 4 bytes of an uint into the lower 5 bytes of an 8 byte ulong 88 | // as 7 bit blocks, with the high bit of each byte set to 0. This is the basis 89 | // of LEB128 encoding, but without the continuation bit set. 90 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 91 | private static ulong Spread7BitGroupsIntoBytes(uint value) 92 | { 93 | // Only one of the three branches below will be included in the JIT output 94 | // based on CPU support at runtime 95 | 96 | if (Bmi2.X64.IsSupported) 97 | { 98 | return Bmi2.X64.ParallelBitDeposit(value, 0xf7f7f7f7fUL); 99 | } 100 | 101 | if (Bmi2.IsSupported) 102 | { 103 | // Intel x86 branch, using 32-bit BMI2 instruction 104 | 105 | return Bmi2.ParallelBitDeposit(value, 0x7f7f7f7fU) | 106 | ((value & 0xf0000000UL) << 4); 107 | } 108 | 109 | // Fallback for unsupported CPUs (i.e. ARM) 110 | return value & 0x0000007fUL 111 | | ((value & 0x00003f80UL) << 1) 112 | | ((value & 0x001fc000UL) << 2) 113 | | ((value & 0x0fe00000UL) << 3) 114 | | ((value & 0xf0000000UL) << 4); 115 | } 116 | 117 | #endif 118 | } 119 | 120 | /* ************************************************************ 121 | * 122 | * @author Couchbase 123 | * @copyright 2021 Couchbase, Inc. 124 | * 125 | * Licensed under the Apache License, Version 2.0 (the "License"); 126 | * you may not use this file except in compliance with the License. 127 | * You may obtain a copy of the License at 128 | * 129 | * http://www.apache.org/licenses/LICENSE-2.0 130 | * 131 | * Unless required by applicable law or agreed to in writing, software 132 | * distributed under the License is distributed on an "AS IS" BASIS, 133 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 134 | * See the License for the specific language governing permissions and 135 | * limitations under the License. 136 | * 137 | * ************************************************************/ 138 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: 7 | - main 8 | - release-* 9 | 10 | jobs: 11 | 12 | test: 13 | 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | framework: ["net8.0", "net9.0", "net10.0"] 18 | disable: ["HWIntrinsics", "SSSE3", "BMI2", "Noop"] 19 | 20 | steps: 21 | - uses: actions/checkout@v5 22 | - name: Setup .NET 23 | uses: actions/setup-dotnet@v5 24 | with: 25 | dotnet-version: | 26 | 8.0.x 27 | 9.0.x 28 | 10.0.x 29 | # Cache packages for faster subsequent runs 30 | - uses: actions/cache@v4 31 | with: 32 | path: ~/.nuget/packages 33 | key: ${{ runner.os }}-${{ runner.arch }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets') }} 34 | restore-keys: | 35 | ${{ runner.os }}-${{ runner.arch }}-nuget- 36 | 37 | - name: Install dependencies 38 | run: dotnet restore 39 | - name: Build 40 | run: dotnet build --no-restore --configuration Release --verbosity normal 41 | - name: Test 42 | run: | 43 | export COMPlus_Enable${{ matrix.disable }}=0 && \ 44 | dotnet test --no-build -f ${{ matrix.framework }} --configuration Release --verbosity normal --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" 45 | - name: Collect Coverage 46 | uses: actions/upload-artifact@v4 47 | with: 48 | name: coverage-${{ matrix.framework }}-${{ runner.arch }}-${{ matrix.disable }} 49 | path: Snappier.Tests/TestResults/**/*.xml 50 | retention-days: 1 51 | 52 | test-arm: 53 | 54 | runs-on: ubuntu-24.04-arm 55 | strategy: 56 | matrix: 57 | framework: ["net8.0", "net9.0", "net10.0"] 58 | 59 | steps: 60 | - uses: actions/checkout@v5 61 | - name: Setup .NET 62 | uses: actions/setup-dotnet@v5 63 | with: 64 | dotnet-version: | 65 | 8.0.x 66 | 9.0.x 67 | 10.0.x 68 | # Cache packages for faster subsequent runs 69 | - uses: actions/cache@v4 70 | with: 71 | path: ~/.nuget/packages 72 | key: ${{ runner.os }}-${{ runner.arch }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets') }} 73 | restore-keys: | 74 | ${{ runner.os }}-${{ runner.arch }}-nuget- 75 | 76 | - name: Install dependencies 77 | run: dotnet restore 78 | - name: Build 79 | run: dotnet build --no-restore --configuration Release --verbosity normal 80 | - name: Test 81 | run: | 82 | dotnet test --no-build -f ${{ matrix.framework }} --configuration Release --verbosity normal --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" 83 | - name: Collect Coverage 84 | uses: actions/upload-artifact@v4 85 | with: 86 | name: coverage-${{ matrix.framework }}-${{ runner.arch }} 87 | path: Snappier.Tests/TestResults/**/*.xml 88 | retention-days: 1 89 | 90 | test-windows: 91 | 92 | runs-on: windows-latest 93 | strategy: 94 | matrix: 95 | arch: ["x64", "x86"] 96 | 97 | steps: 98 | - uses: actions/checkout@v5 99 | - name: Setup .NET 100 | uses: actions/setup-dotnet@v5 101 | with: 102 | dotnet-version: '10.0.x' 103 | # Cache packages for faster subsequent runs 104 | - uses: actions/cache@v4 105 | with: 106 | path: ~/.nuget/packages 107 | key: ${{ runner.os }}-${{ runner.arch }}-nuget-${{ hashFiles('**/*.csproj', '**/*.props', '**/*.targets') }} 108 | restore-keys: | 109 | ${{ runner.os }}-${{ runner.arch }}-nuget- 110 | 111 | - name: Install dependencies 112 | run: dotnet restore --runtime win-${{ matrix.arch }} 113 | - name: Test 114 | run: |- 115 | dotnet test --no-restore --runtime win-${{ matrix.arch }} -f net48 --configuration Release --verbosity normal --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" -- RunConfiguration.DisableAppDomain=true 116 | - name: Collect Coverage 117 | uses: actions/upload-artifact@v4 118 | with: 119 | name: coverage-net4-${{ matrix.arch }} 120 | path: Snappier.Tests/TestResults/**/*.xml 121 | retention-days: 1 122 | 123 | coverage-report: 124 | 125 | name: Coverage Report 126 | runs-on: ubuntu-latest 127 | needs: 128 | - test 129 | - test-windows 130 | 131 | steps: 132 | - uses: actions/checkout@v5 133 | - name: Setup .NET 134 | uses: actions/setup-dotnet@v5 135 | with: 136 | dotnet-version: '10.0.x' 137 | - name: Download Coverage 138 | uses: actions/download-artifact@v4 139 | with: 140 | pattern: coverage-* 141 | path: TestResults 142 | - name: Cleanup Windows Coverage 143 | run: | 144 | find ./TestResults/coverage-net4-x86 -type f -name '*.xml' | xargs sed -i 's|[A-Z]:.*\\Snappier\\Snappier\\Snappier\\|${{ github.workspace }}/Snappier/|g' 145 | find ./TestResults/coverage-net4-x64 -type f -name '*.xml' | xargs sed -i 's|[A-Z]:.*\\Snappier\\Snappier\\Snappier\\|${{ github.workspace }}/Snappier/|g' 146 | shell: bash 147 | - name: ReportGenerator 148 | uses: danielpalme/ReportGenerator-GitHub-Action@v5 149 | with: 150 | reports: 'TestResults/**/*.xml' 151 | targetdir: 'artifacts/coveragereport' 152 | reporttypes: 'Html;MarkdownSummaryGithub' 153 | classfilters: '-System.Diagnostics.*;-System.Runtime.*;-Snappier.Internal.ThrowHelper' 154 | license: ${{ secrets.REPORT_GENERATOR_LICENSE }} 155 | - name: Collect Report 156 | uses: actions/upload-artifact@v4 157 | with: 158 | name: coverage-report 159 | path: artifacts/coveragereport 160 | - name: Add to Build Summary 161 | run: cat artifacts/coveragereport/SummaryGithub.md >> $GITHUB_STEP_SUMMARY # Adjust path and filename if necessary 162 | shell: bash 163 | 164 | publish: 165 | 166 | runs-on: ubuntu-latest 167 | needs: 168 | - test 169 | - test-arm 170 | - test-windows 171 | 172 | permissions: 173 | contents: read 174 | id-token: write # enable GitHub OIDC token issuance for this job 175 | 176 | steps: 177 | - uses: actions/checkout@v5 178 | with: 179 | fetch-depth: 0 # required for GitVersion to work properly 180 | - name: Setup .NET 181 | uses: actions/setup-dotnet@v5 182 | with: 183 | dotnet-version: 10.0.x 184 | 185 | - name: Install GitVersion 186 | uses: gittools/actions/gitversion/setup@v4 187 | with: 188 | versionSpec: "6.4.0" 189 | - name: Determine Version 190 | id: gitversion 191 | uses: gittools/actions/gitversion/execute@v4 192 | with: 193 | configFilePath: "GitVersion.yml" 194 | 195 | - name: NuGet login (OIDC → temp API key) 196 | uses: NuGet/login@v1 197 | id: login 198 | with: 199 | user: ${{ secrets.NUGET_USER }} 200 | 201 | - name: Install dependencies 202 | run: dotnet restore 203 | - name: Pack 204 | run: dotnet pack --configuration Release -p:Version=${{ steps.gitversion.outputs.fullSemVer }} 205 | - name: Upload Package Artifact 206 | uses: actions/upload-artifact@v4 207 | with: 208 | name: package 209 | path: artifacts/package/** 210 | - name: Push to NuGet.org 211 | if: ${{ startsWith(github.ref, 'refs/tags/release/') }} 212 | run: | 213 | dotnet nuget push artifacts/package/**/*.nupkg --api-key ${{ steps.login.outputs.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json 214 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Default settings: 7 | # A newline ending every file 8 | # Use 4 spaces as indentation 9 | [*] 10 | insert_final_newline = true 11 | indent_style = space 12 | indent_size = 4 13 | trim_trailing_whitespace = true 14 | 15 | # C# files 16 | [*.cs] 17 | # New line preferences 18 | csharp_new_line_before_open_brace = all 19 | csharp_new_line_before_else = true 20 | csharp_new_line_before_catch = true 21 | csharp_new_line_before_finally = true 22 | csharp_new_line_before_members_in_object_initializers = true 23 | csharp_new_line_before_members_in_anonymous_types = true 24 | csharp_new_line_between_query_expression_clauses = true 25 | 26 | # Indentation preferences 27 | csharp_indent_block_contents = true 28 | csharp_indent_braces = false 29 | csharp_indent_case_contents = true 30 | csharp_indent_case_contents_when_block = true 31 | csharp_indent_switch_labels = true 32 | csharp_indent_labels = one_less_than_current 33 | 34 | # Modifier preferences 35 | csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:suggestion 36 | 37 | # avoid this. unless absolutely necessary 38 | dotnet_style_qualification_for_field = false:suggestion 39 | dotnet_style_qualification_for_property = false:suggestion 40 | dotnet_style_qualification_for_method = false:suggestion 41 | dotnet_style_qualification_for_event = false:suggestion 42 | 43 | # Types: use keywords instead of BCL types, and permit var only when the type is clear 44 | csharp_style_var_for_built_in_types = false:suggestion 45 | csharp_style_var_when_type_is_apparent = false:none 46 | csharp_style_var_elsewhere = false:suggestion 47 | dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion 48 | dotnet_style_predefined_type_for_member_access = true:suggestion 49 | 50 | # name all constant fields using PascalCase 51 | dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion 52 | dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields 53 | dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style 54 | dotnet_naming_symbols.constant_fields.applicable_kinds = field 55 | dotnet_naming_symbols.constant_fields.required_modifiers = const 56 | dotnet_naming_style.pascal_case_style.capitalization = pascal_case 57 | 58 | # static fields should have s_ prefix 59 | dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion 60 | dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields 61 | dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style 62 | dotnet_naming_symbols.static_fields.applicable_kinds = field 63 | dotnet_naming_symbols.static_fields.required_modifiers = static 64 | dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected 65 | dotnet_naming_style.static_prefix_style.required_prefix = s_ 66 | dotnet_naming_style.static_prefix_style.capitalization = camel_case 67 | 68 | # internal and private fields should be _camelCase 69 | dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion 70 | dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields 71 | dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style 72 | dotnet_naming_symbols.private_internal_fields.applicable_kinds = field 73 | dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal 74 | dotnet_naming_style.camel_case_underscore_style.required_prefix = _ 75 | dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case 76 | 77 | # Code style defaults 78 | csharp_using_directive_placement = outside_namespace:suggestion 79 | dotnet_sort_system_directives_first = true 80 | csharp_prefer_braces = true:silent 81 | csharp_preserve_single_line_blocks = true:none 82 | csharp_preserve_single_line_statements = false:none 83 | csharp_prefer_static_local_function = true:suggestion 84 | csharp_prefer_simple_using_statement = false:none 85 | csharp_style_prefer_switch_expression = true:suggestion 86 | 87 | # Code quality 88 | dotnet_style_readonly_field = true:suggestion 89 | dotnet_code_quality_unused_parameters = non_public:suggestion 90 | 91 | # Expression-level preferences 92 | dotnet_style_object_initializer = true:suggestion 93 | dotnet_style_collection_initializer = true:suggestion 94 | dotnet_style_explicit_tuple_names = true:suggestion 95 | dotnet_style_coalesce_expression = true:suggestion 96 | dotnet_style_null_propagation = true:suggestion 97 | dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion 98 | dotnet_style_prefer_inferred_tuple_names = true:suggestion 99 | dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion 100 | dotnet_style_prefer_auto_properties = true:suggestion 101 | dotnet_style_prefer_conditional_expression_over_assignment = true:silent 102 | dotnet_style_prefer_conditional_expression_over_return = true:silent 103 | csharp_prefer_simple_default_expression = true:suggestion 104 | 105 | # Expression-bodied members 106 | csharp_style_expression_bodied_methods = true:silent 107 | csharp_style_expression_bodied_constructors = true:silent 108 | csharp_style_expression_bodied_operators = true:silent 109 | csharp_style_expression_bodied_properties = true:silent 110 | csharp_style_expression_bodied_indexers = true:silent 111 | csharp_style_expression_bodied_accessors = true:silent 112 | csharp_style_expression_bodied_lambdas = true:silent 113 | csharp_style_expression_bodied_local_functions = true:silent 114 | 115 | # Pattern matching 116 | csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion 117 | csharp_style_pattern_matching_over_as_with_null_check = true:suggestion 118 | csharp_style_inlined_variable_declaration = true:suggestion 119 | 120 | # Null checking preferences 121 | csharp_style_throw_expression = true:suggestion 122 | csharp_style_conditional_delegate_call = true:suggestion 123 | 124 | # Other features 125 | csharp_style_prefer_index_operator = false:none 126 | csharp_style_prefer_range_operator = false:none 127 | csharp_style_pattern_local_over_anonymous_function = false:none 128 | 129 | # Space preferences 130 | csharp_space_after_cast = false 131 | csharp_space_after_colon_in_inheritance_clause = true 132 | csharp_space_after_comma = true 133 | csharp_space_after_dot = false 134 | csharp_space_after_keywords_in_control_flow_statements = true 135 | csharp_space_after_semicolon_in_for_statement = true 136 | csharp_space_around_binary_operators = before_and_after 137 | csharp_space_around_declaration_statements = do_not_ignore 138 | csharp_space_before_colon_in_inheritance_clause = true 139 | csharp_space_before_comma = false 140 | csharp_space_before_dot = false 141 | csharp_space_before_open_square_brackets = false 142 | csharp_space_before_semicolon_in_for_statement = false 143 | csharp_space_between_empty_square_brackets = false 144 | csharp_space_between_method_call_empty_parameter_list_parentheses = false 145 | csharp_space_between_method_call_name_and_opening_parenthesis = false 146 | csharp_space_between_method_call_parameter_list_parentheses = false 147 | csharp_space_between_method_declaration_empty_parameter_list_parentheses = false 148 | csharp_space_between_method_declaration_name_and_open_parenthesis = false 149 | csharp_space_between_method_declaration_parameter_list_parentheses = false 150 | csharp_space_between_parentheses = false 151 | csharp_space_between_square_brackets = false 152 | 153 | # Analyzers 154 | dotnet_code_quality.ca1802.api_surface = private, internal 155 | dotnet_code_quality.ca2208.api_surface = public 156 | 157 | # Xml project files 158 | [*.{csproj,vbproj,vcxproj,vcxproj.filters,proj,nativeproj,locproj}] 159 | indent_size = 2 160 | 161 | [*.{csproj,vbproj,proj,nativeproj,locproj}] 162 | charset = utf-8 163 | 164 | # Xml build files 165 | [*.builds] 166 | indent_size = 2 167 | 168 | # Xml files 169 | [*.{xml,stylecop,resx,ruleset}] 170 | indent_size = 2 171 | 172 | # Xml config files 173 | [*.{props,targets,config,nuspec}] 174 | indent_size = 2 175 | 176 | # YAML config files 177 | [*.{yml,yaml}] 178 | indent_size = 2 179 | 180 | # Shell scripts 181 | [*.sh] 182 | end_of_line = lf 183 | [*.{cmd, bat}] 184 | end_of_line = crlf 185 | -------------------------------------------------------------------------------- /Snappier/Internal/Helpers.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using System.Buffers.Binary; 3 | using System.Diagnostics; 4 | using System.Runtime.CompilerServices; 5 | using System.Runtime.InteropServices; 6 | 7 | #if NET8_0_OR_GREATER 8 | using System.Numerics; 9 | using System.Runtime.Intrinsics.X86; 10 | #endif 11 | 12 | namespace Snappier.Internal; 13 | 14 | internal static class Helpers 15 | { 16 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 17 | public static int MaxCompressedLength(int sourceBytes) 18 | { 19 | // Compressed data can be defined as: 20 | // compressed := item* literal* 21 | // item := literal* copy 22 | // 23 | // The trailing literal sequence has a space blowup of at most 62/60 24 | // since a literal of length 60 needs one tag byte + one extra byte 25 | // for length information. 26 | // 27 | // We also add one extra byte to the blowup to account for the use of 28 | // "ref byte" pointers. The output index will be pushed one byte past 29 | // the end of the output data, but for safety we need to ensure that 30 | // it still points to an element in the buffer array. 31 | // 32 | // Item blowup is trickier to measure. Suppose the "copy" op copies 33 | // 4 bytes of data. Because of a special check in the encoding code, 34 | // we produce a 4-byte copy only if the offset is < 65536. Therefore 35 | // the copy op takes 3 bytes to encode, and this type of item leads 36 | // to at most the 62/60 blowup for representing literals. 37 | // 38 | // Suppose the "copy" op copies 5 bytes of data. If the offset is big 39 | // enough, it will take 5 bytes to encode the copy op. Therefore the 40 | // worst case here is a one-byte literal followed by a five-byte copy. 41 | // I.e., 6 bytes of input turn into 7 bytes of "compressed" data. 42 | // 43 | // This last factor dominates the blowup, so the final estimate is: 44 | 45 | return 32 + sourceBytes + sourceBytes / 6 + 1; 46 | } 47 | 48 | // Constant for MaxCompressedLength when passed Constants.BlockSize, keep this in sync with the above method 49 | public const int MaxBlockCompressedLength = (int)(32 + Constants.BlockSize + Constants.BlockSize / 6 + 1); 50 | 51 | /// 52 | /// Clears the array and returns it to the pool, clearing only the used portion of the array. 53 | /// This is a minor performance optimization to avoid clearing the entire array. 54 | /// 55 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 56 | public static void ClearAndReturn(byte[] array, int usedLength) 57 | { 58 | DebugExtensions.Assert(array is not null); 59 | DebugExtensions.Assert(usedLength >= 0); 60 | 61 | array.AsSpan(0, usedLength).Clear(); 62 | ArrayPool.Shared.Return(array); 63 | } 64 | 65 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 66 | public static bool LeftShiftOverflows(byte value, int shift) 67 | { 68 | DebugExtensions.Assert(shift < 32); 69 | return (value & ~(0xffff_ffffu >>> shift)) != 0; 70 | } 71 | 72 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 73 | public static uint ExtractLowBytes(uint value, int numBytes) 74 | { 75 | DebugExtensions.Assert(numBytes >= 0); 76 | DebugExtensions.Assert(numBytes <= 4); 77 | 78 | #if NET8_0_OR_GREATER 79 | if (Bmi2.IsSupported) 80 | { 81 | return Bmi2.ZeroHighBits(value, (uint)(numBytes * 8)); 82 | } 83 | #endif 84 | 85 | return value & ~(0xffffffff << (8 * numBytes)); 86 | } 87 | 88 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 89 | public static uint UnsafeReadUInt32(ref readonly byte ptr) 90 | { 91 | uint result = Unsafe.ReadUnaligned(in ptr); 92 | 93 | if (!BitConverter.IsLittleEndian) 94 | { 95 | result = BinaryPrimitives.ReverseEndianness(result); 96 | } 97 | 98 | return result; 99 | } 100 | 101 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 102 | public static ulong UnsafeReadUInt64(ref readonly byte ptr) 103 | { 104 | ulong result = Unsafe.ReadUnaligned(in ptr); 105 | 106 | if (!BitConverter.IsLittleEndian) 107 | { 108 | result = BinaryPrimitives.ReverseEndianness(result); 109 | } 110 | 111 | return result; 112 | } 113 | 114 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 115 | public static void UnsafeWriteUInt32(ref byte ptr, uint value) 116 | { 117 | if (!BitConverter.IsLittleEndian) 118 | { 119 | value = BinaryPrimitives.ReverseEndianness(value); 120 | } 121 | 122 | Unsafe.WriteUnaligned(ref ptr, value); 123 | } 124 | 125 | #if !NET8_0_OR_GREATER 126 | 127 | // Port from .NET 7 BitOperations of a faster fallback algorithm for .NET Standard since we don't have intrinsics 128 | // or BitOperations. This is the same algorithm used by BitOperations.Log2 when hardware acceleration is unavailable. 129 | // https://github.com/dotnet/runtime/blob/bee217ffbdd6b3ad60b0e1e17c6370f4bb618779/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs#L404 130 | 131 | private static ReadOnlySpan Log2DeBruijn => 132 | [ 133 | 00, 09, 01, 10, 13, 21, 02, 29, 134 | 11, 14, 16, 18, 22, 25, 03, 30, 135 | 08, 12, 20, 28, 15, 17, 24, 07, 136 | 19, 27, 23, 06, 26, 05, 04, 31 137 | ]; 138 | 139 | /// 140 | /// Returns the integer (floor) log of the specified value, base 2. 141 | /// Note that by convention, input value 0 returns 0 since Log(0) is undefined. 142 | /// Does not directly use any hardware intrinsics, nor does it incur branching. 143 | /// 144 | /// The value. 145 | private static int Log2SoftwareFallback(uint value) 146 | { 147 | // No AggressiveInlining due to large method size 148 | // Has conventional contract 0->0 (Log(0) is undefined) 149 | 150 | // Fill trailing zeros with ones, eg 00010010 becomes 00011111 151 | value |= value >> 01; 152 | value |= value >> 02; 153 | value |= value >> 04; 154 | value |= value >> 08; 155 | value |= value >> 16; 156 | 157 | // uint.MaxValue >> 27 is always in range [0 - 31] so we use Unsafe.AddByteOffset to avoid bounds check 158 | return Unsafe.AddByteOffset( 159 | // Using deBruijn sequence, k=2, n=5 (2^5=32) : 0b_0000_0111_1100_0100_1010_1100_1101_1101u 160 | ref MemoryMarshal.GetReference(Log2DeBruijn), 161 | // uint|long -> IntPtr cast on 32-bit platforms does expensive overflow checks not needed here 162 | (IntPtr)(int)((value * 0x07C4ACDDu) >> 27)); 163 | } 164 | 165 | #endif 166 | 167 | /// 168 | /// Return floor(log2(n)) for positive integer n. Returns 0 for the special case n = 0. 169 | /// 170 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 171 | public static int Log2Floor(uint n) 172 | { 173 | #if NET8_0_OR_GREATER 174 | return BitOperations.Log2(n); 175 | #else 176 | return Log2SoftwareFallback(n); 177 | #endif 178 | } 179 | 180 | /// 181 | /// Finds the index of the least significant non-zero bit. 182 | /// 183 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 184 | public static int FindLsbSetNonZero(uint n) 185 | { 186 | DebugExtensions.Assert(n != 0); 187 | 188 | #if NET8_0_OR_GREATER 189 | return BitOperations.TrailingZeroCount(n); 190 | #else 191 | int rc = 31; 192 | int shift = 1 << 4; 193 | 194 | for (int i = 4; i >= 0; --i) 195 | { 196 | uint x = n << shift; 197 | if (x != 0) 198 | { 199 | n = x; 200 | rc -= shift; 201 | } 202 | 203 | shift >>= 1; 204 | } 205 | 206 | return rc; 207 | #endif 208 | } 209 | 210 | /// 211 | /// Finds the index of the least significant non-zero bit. 212 | /// 213 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 214 | public static int FindLsbSetNonZero(ulong n) 215 | { 216 | DebugExtensions.Assert(n != 0); 217 | 218 | #if NET8_0_OR_GREATER 219 | return BitOperations.TrailingZeroCount(n); 220 | #else 221 | uint bottomBits = unchecked((uint)n); 222 | if (bottomBits == 0) 223 | { 224 | return 32 + FindLsbSetNonZero(unchecked((uint)(n >> 32))); 225 | } 226 | else 227 | { 228 | return FindLsbSetNonZero(bottomBits); 229 | } 230 | #endif 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /Snappier.Tests/SnappyStreamTests.cs: -------------------------------------------------------------------------------- 1 | using System.IO.Compression; 2 | using System.Text; 3 | 4 | namespace Snappier.Tests; 5 | 6 | public class SnappyStreamTests(ITestOutputHelper outputHelper) 7 | { 8 | [Theory] 9 | [InlineData("alice29.txt")] 10 | [InlineData("asyoulik.txt")] 11 | [InlineData("fireworks.jpeg")] 12 | [InlineData("geo.protodata")] 13 | [InlineData("html")] 14 | [InlineData("html_x_4")] 15 | [InlineData("kppkn.gtb")] 16 | [InlineData("lcet10.txt")] 17 | [InlineData("paper-100k.pdf")] 18 | [InlineData("plrabn12.txt")] 19 | [InlineData("urls.10K")] 20 | public void CompressAndDecompress(string filename) 21 | { 22 | using Stream resource = 23 | typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); 24 | Assert.NotNull(resource); 25 | 26 | using var output = new MemoryStream(); 27 | 28 | using (var compressor = new SnappyStream(output, CompressionMode.Compress, true)) 29 | { 30 | resource.CopyTo(compressor); 31 | } 32 | 33 | output.Position = 0; 34 | 35 | using var decompressor = new SnappyStream(output, CompressionMode.Decompress, true); 36 | 37 | using var streamReader = new StreamReader(decompressor, Encoding.UTF8); 38 | string decompressedText = streamReader.ReadToEnd(); 39 | 40 | outputHelper.WriteLine(decompressedText); 41 | 42 | using Stream sourceResource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); 43 | Assert.NotNull(sourceResource); 44 | 45 | using var streamReader2 = new StreamReader(sourceResource, Encoding.UTF8); 46 | string sourceText = streamReader2.ReadToEnd(); 47 | 48 | Assert.Equal(sourceText, decompressedText); 49 | } 50 | 51 | [Fact] 52 | public void CompressAndDecompress_SingleByte() 53 | { 54 | using Stream resource = 55 | typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.alice29.txt"); 56 | Assert.NotNull(resource); 57 | 58 | byte[] inBuffer = new byte[128]; 59 | int readBytes = resource.Read(inBuffer, 0, inBuffer.Length); 60 | 61 | using var output = new MemoryStream(); 62 | 63 | using (var compressor = new SnappyStream(output, CompressionMode.Compress, true)) 64 | { 65 | for (int i = 0; i < readBytes; i++) 66 | { 67 | compressor.WriteByte(inBuffer[i]); 68 | } 69 | } 70 | 71 | output.Position = 0; 72 | 73 | using var decompressor = new SnappyStream(output, CompressionMode.Decompress, true); 74 | 75 | byte[] outBuffer = new byte[128]; 76 | for (int i = 0; i < readBytes; i++) 77 | { 78 | outBuffer[i] = (byte)decompressor.ReadByte(); 79 | } 80 | 81 | Assert.Equal(inBuffer, outBuffer); 82 | } 83 | 84 | [Theory] 85 | [InlineData("alice29.txt")] 86 | [InlineData("asyoulik.txt")] 87 | [InlineData("fireworks.jpeg")] 88 | [InlineData("geo.protodata")] 89 | [InlineData("html")] 90 | [InlineData("html_x_4")] 91 | [InlineData("kppkn.gtb")] 92 | [InlineData("lcet10.txt")] 93 | [InlineData("paper-100k.pdf")] 94 | [InlineData("plrabn12.txt")] 95 | [InlineData("urls.10K")] 96 | public async Task CompressAndDecompressAsync(string filename) 97 | { 98 | using Stream resource = 99 | typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); 100 | Assert.NotNull(resource); 101 | 102 | using var output = new MemoryStream(); 103 | 104 | #if NET6_0_OR_GREATER 105 | await using (var compressor = new SnappyStream(output, CompressionMode.Compress, true)) 106 | { 107 | await resource.CopyToAsync(compressor, TestContext.Current.CancellationToken); 108 | } 109 | #else 110 | using (var compressor = new SnappyStream(output, CompressionMode.Compress, true)) 111 | { 112 | await resource.CopyToAsync(compressor); 113 | } 114 | #endif 115 | 116 | output.Position = 0; 117 | 118 | #if NET6_0_OR_GREATER 119 | await 120 | #endif 121 | using var decompressor = new SnappyStream(output, CompressionMode.Decompress, true); 122 | 123 | using var streamReader = new StreamReader(decompressor, Encoding.UTF8); 124 | #if NET6_0_OR_GREATER 125 | string decompressedText = await streamReader.ReadToEndAsync(TestContext.Current.CancellationToken); 126 | #else 127 | string decompressedText = await streamReader.ReadToEndAsync(); 128 | #endif 129 | 130 | outputHelper.WriteLine(decompressedText); 131 | 132 | using Stream sourceResource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); 133 | Assert.NotNull(sourceResource); 134 | 135 | using var streamReader2 = new StreamReader(sourceResource, Encoding.UTF8); 136 | #if NET6_0_OR_GREATER 137 | string sourceText = await streamReader2.ReadToEndAsync(TestContext.Current.CancellationToken); 138 | #else 139 | string sourceText = await streamReader2.ReadToEndAsync(); 140 | #endif 141 | 142 | Assert.Equal(sourceText, decompressedText); 143 | } 144 | 145 | [Theory] 146 | [InlineData("alice29.txt")] 147 | [InlineData("asyoulik.txt")] 148 | [InlineData("fireworks.jpeg")] 149 | [InlineData("geo.protodata")] 150 | [InlineData("html")] 151 | [InlineData("html_x_4")] 152 | [InlineData("kppkn.gtb")] 153 | [InlineData("lcet10.txt")] 154 | [InlineData("paper-100k.pdf")] 155 | [InlineData("plrabn12.txt")] 156 | [InlineData("urls.10K")] 157 | // Test writing lots of small chunks to catch errors where reading needs to break mid-chunk. 158 | public void CompressAndDecompressChunkStressTest(string filename) 159 | { 160 | Stream resource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream($"Snappier.Tests.TestData.{filename}"); 161 | using var resourceMem = new MemoryStream(); 162 | resource.CopyTo(resourceMem); 163 | byte[] originalBytes = resourceMem.ToArray(); 164 | 165 | var rand = new Random(123); 166 | 167 | using var compresed = new MemoryStream(); 168 | using (var inputStream = new MemoryStream(originalBytes)) 169 | using (var compressor = new SnappyStream(compresed, CompressionMode.Compress, true)) 170 | { 171 | // Write lots of small randomly sized chunks to increase change of hitting error conditions. 172 | byte[] buffer = new byte[100]; 173 | int requestedSize = rand.Next(1, buffer.Length); 174 | int n; 175 | while ((n = inputStream.Read(buffer, 0, requestedSize)) != 0) 176 | { 177 | compressor.Write(buffer, 0, n); 178 | // Flush after every write so we get lots of small chunks in the compressed output. 179 | compressor.Flush(); 180 | } 181 | } 182 | compresed.Position = 0; 183 | 184 | using var decompressed = new MemoryStream(); 185 | using (var decompressor = new SnappyStream(compresed, CompressionMode.Decompress, true)) 186 | { 187 | decompressor.CopyTo(decompressed); 188 | } 189 | 190 | Assert.Equal(originalBytes.Length, decompressed.Length); 191 | Assert.Equal(originalBytes, decompressed.ToArray()); 192 | } 193 | 194 | #if NET6_0_OR_GREATER 195 | 196 | // Test case that we know was failing on decompress with the default 8192 byte chunk size 197 | [Fact] 198 | public void Known8192ByteChunkStressTest() 199 | { 200 | using Stream resource = typeof(SnappyStreamTests).Assembly.GetManifestResourceStream("Snappier.Tests.TestData.streamerrorsequence.txt")!; 201 | byte[] originalBytes = ConvertFromHexStream(resource); 202 | 203 | using var compressed = new MemoryStream(); 204 | using SnappyStream compressor = new(compressed, CompressionMode.Compress); 205 | 206 | compressor.Write(originalBytes, 0, originalBytes.Length); 207 | compressor.Flush(); 208 | 209 | compressed.Position = 0; 210 | 211 | using SnappyStream decompressor = new(compressed, CompressionMode.Decompress); 212 | using var decompressed = new MemoryStream(); 213 | decompressor.CopyTo(decompressed); 214 | 215 | Assert.True(decompressed.GetBuffer().AsSpan(0, (int) decompressed.Length).SequenceEqual(originalBytes)); 216 | } 217 | 218 | private static byte[] ConvertFromHexStream(Stream stream) 219 | { 220 | using var output = new MemoryStream(); 221 | 222 | using var textReader = new StreamReader(stream, Encoding.UTF8); 223 | 224 | char[] buffer = new char[1024]; 225 | 226 | int charsRead = textReader.Read(buffer.AsSpan()); 227 | while (charsRead > 0) 228 | { 229 | byte[] bytes = Convert.FromHexString(buffer.AsSpan(0, charsRead)); 230 | output.Write(bytes.AsSpan()); 231 | 232 | charsRead = textReader.Read(buffer, 0, buffer.Length); 233 | } 234 | 235 | return output.ToArray(); 236 | } 237 | 238 | #endif 239 | 240 | // Test case that we know was failing on decompress with the default 8192 byte chunk size 241 | [Fact] 242 | public void UncompressedBlock() 243 | { 244 | byte[] originalBytes = [..Enumerable.Range(0, 256).Select(p => (byte)p)]; 245 | 246 | using var compressed = new MemoryStream(); 247 | using SnappyStream compressor = new(compressed, CompressionMode.Compress); 248 | 249 | compressor.Write(originalBytes, 0, originalBytes.Length); 250 | compressor.Flush(); 251 | 252 | // Snappy header + block header + uncompressed data 253 | Assert.Equal(10 + 8 + originalBytes.Length, compressed.Length); 254 | 255 | compressed.Position = 0; 256 | 257 | using SnappyStream decompressor = new(compressed, CompressionMode.Decompress); 258 | using var decompressed = new MemoryStream(); 259 | decompressor.CopyTo(decompressed); 260 | 261 | Assert.True(decompressed.GetBuffer().AsSpan(0, (int)decompressed.Length).SequenceEqual(originalBytes)); 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /Snappier/Internal/NullableAttributes.cs: -------------------------------------------------------------------------------- 1 | #define INTERNAL_NULLABLE_ATTRIBUTES 2 | #if !NET8_0_OR_GREATER 3 | 4 | // https://github.com/dotnet/corefx/blob/48363ac826ccf66fbe31a5dcb1dc2aab9a7dd768/src/Common/src/CoreLib/System/Diagnostics/CodeAnalysis/NullableAttributes.cs 5 | 6 | // Licensed to the .NET Foundation under one or more agreements. 7 | // The .NET Foundation licenses this file to you under the MIT license. 8 | // See the LICENSE file in the project root for more information. 9 | 10 | // ReSharper disable once CheckNamespace 11 | namespace System.Diagnostics.CodeAnalysis 12 | { 13 | /// Specifies that null is allowed as an input even if the corresponding type disallows it. 14 | [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] 15 | #if INTERNAL_NULLABLE_ATTRIBUTES 16 | internal 17 | #else 18 | public 19 | #endif 20 | sealed class AllowNullAttribute : Attribute 21 | { } 22 | 23 | /// Specifies that null is disallowed as an input even if the corresponding type allows it. 24 | [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)] 25 | #if INTERNAL_NULLABLE_ATTRIBUTES 26 | internal 27 | #else 28 | public 29 | #endif 30 | sealed class DisallowNullAttribute : Attribute 31 | { } 32 | 33 | /// Specifies that an output may be null even if the corresponding type disallows it. 34 | [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] 35 | #if INTERNAL_NULLABLE_ATTRIBUTES 36 | internal 37 | #else 38 | public 39 | #endif 40 | sealed class MaybeNullAttribute : Attribute 41 | { } 42 | 43 | /// Specifies that an output will not be null even if the corresponding type allows it. 44 | [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, Inherited = false)] 45 | #if INTERNAL_NULLABLE_ATTRIBUTES 46 | internal 47 | #else 48 | public 49 | #endif 50 | sealed class NotNullAttribute : Attribute 51 | { } 52 | 53 | /// Specifies that when a method returns , the parameter may be null even if the corresponding type disallows it. 54 | [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] 55 | #if INTERNAL_NULLABLE_ATTRIBUTES 56 | internal 57 | #else 58 | public 59 | #endif 60 | sealed class MaybeNullWhenAttribute : Attribute 61 | { 62 | /// Initializes the attribute with the specified return value condition. 63 | /// 64 | /// The return value condition. If the method returns this value, the associated parameter may be null. 65 | /// 66 | public MaybeNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; 67 | 68 | /// Gets the return value condition. 69 | public bool ReturnValue { get; } 70 | } 71 | 72 | /// Specifies that when a method returns , the parameter will not be null even if the corresponding type allows it. 73 | [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] 74 | #if INTERNAL_NULLABLE_ATTRIBUTES 75 | internal 76 | #else 77 | public 78 | #endif 79 | sealed class NotNullWhenAttribute : Attribute 80 | { 81 | /// Initializes the attribute with the specified return value condition. 82 | /// 83 | /// The return value condition. If the method returns this value, the associated parameter will not be null. 84 | /// 85 | public NotNullWhenAttribute(bool returnValue) => ReturnValue = returnValue; 86 | 87 | /// Gets the return value condition. 88 | public bool ReturnValue { get; } 89 | } 90 | 91 | /// Specifies that the output will be non-null if the named parameter is non-null. 92 | [AttributeUsage(AttributeTargets.Parameter | AttributeTargets.Property | AttributeTargets.ReturnValue, AllowMultiple = true, Inherited = false)] 93 | #if INTERNAL_NULLABLE_ATTRIBUTES 94 | internal 95 | #else 96 | public 97 | #endif 98 | sealed class NotNullIfNotNullAttribute : Attribute 99 | { 100 | /// Initializes the attribute with the associated parameter name. 101 | /// 102 | /// The associated parameter name. The output will be non-null if the argument to the parameter specified is non-null. 103 | /// 104 | public NotNullIfNotNullAttribute(string parameterName) => ParameterName = parameterName; 105 | 106 | /// Gets the associated parameter name. 107 | public string ParameterName { get; } 108 | } 109 | 110 | /// Applied to a method that will never return under any circumstance. 111 | [AttributeUsage(AttributeTargets.Method, Inherited = false)] 112 | #if INTERNAL_NULLABLE_ATTRIBUTES 113 | internal 114 | #else 115 | public 116 | #endif 117 | sealed class DoesNotReturnAttribute : Attribute 118 | { } 119 | 120 | /// Specifies that the method will not return if the associated Boolean parameter is passed the specified value. 121 | [AttributeUsage(AttributeTargets.Parameter, Inherited = false)] 122 | #if INTERNAL_NULLABLE_ATTRIBUTES 123 | internal 124 | #else 125 | public 126 | #endif 127 | sealed class DoesNotReturnIfAttribute : Attribute 128 | { 129 | /// Initializes the attribute with the specified parameter value. 130 | /// 131 | /// The condition parameter value. Code after the method will be considered unreachable by diagnostics if the argument to 132 | /// the associated parameter matches this value. 133 | /// 134 | public DoesNotReturnIfAttribute(bool parameterValue) => ParameterValue = parameterValue; 135 | 136 | /// Gets the condition parameter value. 137 | public bool ParameterValue { get; } 138 | } 139 | 140 | /// Specifies that the method or property will ensure that the listed field and property members have not-null values. 141 | [AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)] 142 | #if INTERNAL_NULLABLE_ATTRIBUTES 143 | internal 144 | #else 145 | public 146 | #endif 147 | sealed class MemberNotNullAttribute : Attribute 148 | { 149 | /// Initializes the attribute with a field or property member. 150 | /// 151 | /// The field or property member that is promised to be not-null. 152 | /// 153 | public MemberNotNullAttribute(string member) => Members = new[] { member }; 154 | 155 | /// Initializes the attribute with the list of field and property members. 156 | /// 157 | /// The list of field and property members that are promised to be not-null. 158 | /// 159 | public MemberNotNullAttribute(params string[] members) => Members = members; 160 | 161 | /// Gets field or property member names. 162 | public string[] Members { get; } 163 | } 164 | 165 | /// Specifies that the method or property will ensure that the listed field and property members have not-null values when returning with the specified return value condition. 166 | [AttributeUsage(AttributeTargets.Method | AttributeTargets.Property, Inherited = false, AllowMultiple = true)] 167 | #if INTERNAL_NULLABLE_ATTRIBUTES 168 | internal 169 | #else 170 | public 171 | #endif 172 | sealed class MemberNotNullWhenAttribute : Attribute 173 | { 174 | /// Initializes the attribute with the specified return value condition and a field or property member. 175 | /// 176 | /// The return value condition. If the method returns this value, the associated parameter will not be null. 177 | /// 178 | /// 179 | /// The field or property member that is promised to be not-null. 180 | /// 181 | public MemberNotNullWhenAttribute(bool returnValue, string member) 182 | { 183 | ReturnValue = returnValue; 184 | Members = new[] { member }; 185 | } 186 | 187 | /// Initializes the attribute with the specified return value condition and list of field and property members. 188 | /// 189 | /// The return value condition. If the method returns this value, the associated parameter will not be null. 190 | /// 191 | /// 192 | /// The list of field and property members that are promised to be not-null. 193 | /// 194 | public MemberNotNullWhenAttribute(bool returnValue, params string[] members) 195 | { 196 | ReturnValue = returnValue; 197 | Members = members; 198 | } 199 | 200 | /// Gets the return value condition. 201 | public bool ReturnValue { get; } 202 | 203 | /// Gets field or property member names. 204 | public string[] Members { get; } 205 | } 206 | } 207 | #endif 208 | 209 | 210 | /* ************************************************************ 211 | * 212 | * @author Couchbase 213 | * @copyright 2021 Couchbase, Inc. 214 | * 215 | * Licensed under the Apache License, Version 2.0 (the "License"); 216 | * you may not use this file except in compliance with the License. 217 | * You may obtain a copy of the License at 218 | * 219 | * http://www.apache.org/licenses/LICENSE-2.0 220 | * 221 | * Unless required by applicable law or agreed to in writing, software 222 | * distributed under the License is distributed on an "AS IS" BASIS, 223 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 224 | * See the License for the specific language governing permissions and 225 | * limitations under the License. 226 | * 227 | * ************************************************************/ 228 | -------------------------------------------------------------------------------- /Snappier/Internal/SnappyStreamCompressor.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using System.Buffers.Binary; 3 | using System.Diagnostics; 4 | using System.Diagnostics.CodeAnalysis; 5 | using System.Runtime.CompilerServices; 6 | 7 | #if !NET8_0_OR_GREATER 8 | using System.Runtime.InteropServices; 9 | #endif 10 | 11 | namespace Snappier.Internal; 12 | 13 | /// 14 | /// Emits the stream format used for Snappy streams. 15 | /// 16 | internal class SnappyStreamCompressor : IDisposable 17 | { 18 | private static ReadOnlySpan SnappyHeader => 19 | [ 20 | 0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59 21 | ]; 22 | 23 | private SnappyCompressor? _compressor = new(); 24 | 25 | private byte[]? _inputBuffer; 26 | private int _inputBufferSize; 27 | 28 | private byte[]? _outputBuffer; 29 | private int _outputBufferSize; 30 | 31 | private bool _streamHeaderWritten; 32 | 33 | /// 34 | /// Processes some input, potentially returning compressed data. Flush must be called when input is complete 35 | /// to get any remaining compressed data. 36 | /// 37 | /// Uncompressed data to emit. 38 | /// Output stream. 39 | /// A block of memory with compressed data (if any). Must be used before any subsequent call to Write. 40 | public void Write(ReadOnlySpan input, Stream stream) 41 | { 42 | ArgumentNullException.ThrowIfNull(stream); 43 | ObjectDisposedException.ThrowIf(_compressor is null, this); 44 | 45 | EnsureBuffer(); 46 | EnsureStreamHeaderWritten(); 47 | 48 | while (input.Length > 0) 49 | { 50 | int bytesRead = CompressInput(input); 51 | input = input.Slice(bytesRead); 52 | 53 | WriteOutputBuffer(stream); 54 | } 55 | } 56 | 57 | /// 58 | /// Processes some input, potentially returning compressed data. Flush must be called when input is complete 59 | /// to get any remaining compressed data. 60 | /// 61 | /// Uncompressed data to emit. 62 | /// Output stream. 63 | /// Cancellation token. 64 | /// A block of memory with compressed data (if any). Must be used before any subsequent call to Write. 65 | public async Task WriteAsync(ReadOnlyMemory input, Stream stream, CancellationToken cancellationToken = default) 66 | { 67 | ArgumentNullException.ThrowIfNull(stream); 68 | ObjectDisposedException.ThrowIf(_compressor is null, this); 69 | 70 | EnsureBuffer(); 71 | EnsureStreamHeaderWritten(); 72 | 73 | while (input.Length > 0) 74 | { 75 | int bytesRead = CompressInput(input.Span); 76 | input = input.Slice(bytesRead); 77 | 78 | await WriteOutputBufferAsync(stream, cancellationToken).ConfigureAwait(false); 79 | } 80 | } 81 | 82 | public void Flush(Stream stream) 83 | { 84 | ArgumentNullException.ThrowIfNull(stream); 85 | ObjectDisposedException.ThrowIf(_compressor is null, this); 86 | 87 | EnsureBuffer(); 88 | EnsureStreamHeaderWritten(); 89 | 90 | if (_inputBufferSize > 0) 91 | { 92 | CompressBlock(_inputBuffer.AsSpan(0, _inputBufferSize)); 93 | _inputBufferSize = 0; 94 | } 95 | 96 | WriteOutputBuffer(stream); 97 | } 98 | 99 | public async Task FlushAsync(Stream stream, CancellationToken cancellationToken = default) 100 | { 101 | ArgumentNullException.ThrowIfNull(stream); 102 | ObjectDisposedException.ThrowIf(_compressor is null, this); 103 | 104 | EnsureBuffer(); 105 | EnsureStreamHeaderWritten(); 106 | 107 | if (_inputBufferSize > 0) 108 | { 109 | CompressBlock(_inputBuffer.AsSpan(0, _inputBufferSize)); 110 | _inputBufferSize = 0; 111 | } 112 | 113 | await WriteOutputBufferAsync(stream, cancellationToken).ConfigureAwait(false); 114 | } 115 | 116 | private void WriteOutputBuffer(Stream stream) 117 | { 118 | DebugExtensions.Assert(_outputBuffer is not null); 119 | 120 | if (_outputBufferSize <= 0) 121 | { 122 | return; 123 | } 124 | 125 | stream.Write(_outputBuffer, 0, _outputBufferSize); 126 | 127 | _outputBufferSize = 0; 128 | } 129 | 130 | private async Task WriteOutputBufferAsync(Stream stream, CancellationToken cancellationToken = default) 131 | { 132 | DebugExtensions.Assert(_outputBuffer is not null); 133 | 134 | if (_outputBufferSize <= 0) 135 | { 136 | return; 137 | } 138 | 139 | #if NET8_0_OR_GREATER 140 | await stream.WriteAsync(_outputBuffer.AsMemory(0, _outputBufferSize), cancellationToken).ConfigureAwait(false); 141 | #else 142 | await stream.WriteAsync(_outputBuffer, 0, _outputBufferSize, cancellationToken).ConfigureAwait(false); 143 | #endif 144 | 145 | _outputBufferSize = 0; 146 | } 147 | 148 | private void EnsureStreamHeaderWritten() 149 | { 150 | if (!_streamHeaderWritten) 151 | { 152 | SnappyHeader.CopyTo(_outputBuffer.AsSpan()); 153 | _outputBufferSize += SnappyHeader.Length; 154 | 155 | _streamHeaderWritten = true; 156 | } 157 | } 158 | 159 | /// 160 | /// Processes up to one entire block from the input, potentially combining with previous input blocks. 161 | /// Fills the compressed data to the output buffer. Will not process more than one output block at a time 162 | /// to avoid overflowing the output buffer. 163 | /// 164 | /// Input to compress. 165 | /// Number of bytes consumed. 166 | private int CompressInput(ReadOnlySpan input) 167 | { 168 | DebugExtensions.Assert(input.Length > 0); 169 | 170 | if (_inputBufferSize == 0 && input.Length >= Constants.BlockSize) 171 | { 172 | // Optimize to avoid copying 173 | 174 | input = input.Slice(0, (int) Constants.BlockSize); 175 | CompressBlock(input); 176 | return input.Length; 177 | } 178 | 179 | // Append what we can to the input buffer 180 | 181 | int appendLength = Math.Min(input.Length, (int) Constants.BlockSize - _inputBufferSize); 182 | input.Slice(0, appendLength).CopyTo(_inputBuffer.AsSpan(_inputBufferSize)); 183 | _inputBufferSize += appendLength; 184 | 185 | if (_inputBufferSize >= Constants.BlockSize) 186 | { 187 | CompressBlock(_inputBuffer.AsSpan(0, _inputBufferSize)); 188 | _inputBufferSize = 0; 189 | } 190 | 191 | return appendLength; 192 | } 193 | 194 | private void CompressBlock(ReadOnlySpan input) 195 | { 196 | DebugExtensions.Assert(_compressor != null); 197 | DebugExtensions.Assert(input.Length <= Constants.BlockSize); 198 | 199 | const int headerSize = 8; // 1 byte for chunk type, 3 bytes for block size, 4 bytes for CRC 200 | 201 | Span output = _outputBuffer.AsSpan(_outputBufferSize); 202 | 203 | // Make room for the header and CRC 204 | Span blockBody = output.Slice(headerSize); 205 | 206 | if (!_compressor.TryCompress(input, blockBody, out int bytesWritten)) 207 | { 208 | // Should be unreachable since we're allocating a buffer of the correct size. 209 | ThrowHelper.ThrowInvalidOperationException(); 210 | } 211 | 212 | if (bytesWritten < input.Length) 213 | { 214 | // Compression resulted in a smaller size, write the header 215 | 216 | WriteCompressedBlockHeader(input, output, bytesWritten); 217 | 218 | _outputBufferSize += bytesWritten + headerSize; 219 | } 220 | else 221 | { 222 | // Compression resulted in growth, switch to an uncompressed block, 223 | // overwriting the compressed data in the output buffer 224 | 225 | WriteUncompressedBlockHeader(input, output); 226 | input.CopyTo(blockBody); 227 | 228 | _outputBufferSize += input.Length + headerSize; 229 | } 230 | } 231 | 232 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 233 | private static void WriteCompressedBlockHeader(ReadOnlySpan input, Span output, int compressedSize) 234 | { 235 | // Write block size to bytes 1-3 236 | int blockSize = compressedSize + 4; // CRC 237 | BinaryPrimitives.WriteInt32LittleEndian(output, blockSize << 8); 238 | 239 | // Overwrite byte 1 with the chunk type 240 | output[0] = (byte) Constants.ChunkType.CompressedData; 241 | 242 | // Write masked CRC to bytes 5-8 243 | uint crc = Crc32CAlgorithm.Compute(input); 244 | crc = Crc32CAlgorithm.ApplyMask(crc); 245 | BinaryPrimitives.WriteUInt32LittleEndian(output.Slice(4), crc); 246 | } 247 | 248 | private static void WriteUncompressedBlockHeader(ReadOnlySpan input, Span output) 249 | { 250 | // Write block size to bytes 1-3 251 | int blockSize = input.Length + 4; // CRC 252 | BinaryPrimitives.WriteInt32LittleEndian(output, blockSize << 8); 253 | 254 | // Overwrite byte 1 with the chunk type 255 | output[0] = (byte)Constants.ChunkType.UncompressedData; 256 | 257 | // Write masked CRC to bytes 5-8 258 | uint crc = Crc32CAlgorithm.Compute(input); 259 | crc = Crc32CAlgorithm.ApplyMask(crc); 260 | BinaryPrimitives.WriteUInt32LittleEndian(output.Slice(4), crc); 261 | } 262 | 263 | [MemberNotNull(nameof(_outputBuffer), nameof(_inputBuffer))] 264 | private void EnsureBuffer() 265 | { 266 | // Allocate enough room for the stream header and block headers 267 | _outputBuffer ??= 268 | ArrayPool.Shared.Rent(Helpers.MaxBlockCompressedLength + 8 + SnappyHeader.Length); 269 | 270 | // Allocate enough room for the stream header and block headers 271 | _inputBuffer ??= ArrayPool.Shared.Rent((int) Constants.BlockSize); 272 | } 273 | 274 | public void Dispose() 275 | { 276 | _compressor?.Dispose(); 277 | _compressor = null; 278 | 279 | if (_outputBuffer is not null) 280 | { 281 | ArrayPool.Shared.Return(_outputBuffer, clearArray: true); 282 | _outputBuffer = null; 283 | } 284 | if (_inputBuffer is not null) 285 | { 286 | ArrayPool.Shared.Return(_inputBuffer, clearArray: true); 287 | _inputBuffer = null; 288 | } 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /Snappier/Internal/SnappyStreamDecompressor.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers.Binary; 2 | using System.Diagnostics; 3 | 4 | namespace Snappier.Internal; 5 | 6 | /// 7 | /// Parses the stream format used for Snappy streams. 8 | /// 9 | internal sealed class SnappyStreamDecompressor : IDisposable 10 | { 11 | private const int ScratchBufferSize = 4; 12 | 13 | #if !NET8_0_OR_GREATER 14 | private readonly byte[] _scratch = new byte[ScratchBufferSize]; 15 | #else 16 | [System.Runtime.CompilerServices.InlineArray(ScratchBufferSize)] 17 | private struct ScratchBuffer 18 | { 19 | private byte _element0; 20 | } 21 | 22 | private ScratchBuffer _scratch; 23 | #endif 24 | 25 | private Span Scratch => _scratch; 26 | 27 | private SnappyDecompressor? _decompressor = new(); 28 | 29 | private ReadOnlyMemory _input; 30 | 31 | private int _scratchLength; 32 | private Constants.ChunkType _chunkType = Constants.ChunkType.Null; 33 | private int _chunkSize; 34 | private int _chunkBytesProcessed; 35 | private uint _expectedChunkCrc; 36 | private uint _chunkCrc; 37 | 38 | public int Decompress(Span buffer) 39 | { 40 | DebugExtensions.Assert(_decompressor != null); 41 | 42 | ReadOnlySpan input = _input.Span; 43 | 44 | // Cache this to use later to calculate the total bytes written 45 | int originalBufferLength = buffer.Length; 46 | 47 | while (buffer.Length > 0 48 | && (input.Length > 0 || (_chunkType == Constants.ChunkType.CompressedData && _decompressor.AllDataDecompressed))) 49 | { 50 | // ReSharper disable once SwitchStatementHandlesSomeKnownEnumValuesWithDefault 51 | switch (_chunkType) 52 | { 53 | case Constants.ChunkType.Null: 54 | // Not in a chunk, read the chunk type and size 55 | 56 | uint rawChunkHeader = ReadChunkHeader(ref input); 57 | 58 | if (rawChunkHeader == 0) 59 | { 60 | // Not enough data, get some more 61 | goto exit; 62 | } 63 | 64 | _chunkType = (Constants.ChunkType) (rawChunkHeader & 0xff); 65 | _chunkSize = unchecked((int)(rawChunkHeader >> 8)); 66 | _chunkBytesProcessed = 0; 67 | _scratchLength = 0; 68 | _chunkCrc = 0; 69 | break; 70 | 71 | case Constants.ChunkType.CompressedData: 72 | { 73 | if (_chunkBytesProcessed < 4) 74 | { 75 | _decompressor.Reset(); 76 | 77 | if (!ReadChunkCrc(ref input)) 78 | { 79 | // Incomplete CRC 80 | goto exit; 81 | } 82 | 83 | if (input.Length == 0) 84 | { 85 | // No more data 86 | goto exit; 87 | } 88 | } 89 | 90 | while (buffer.Length > 0 && !_decompressor.EndOfFile) 91 | { 92 | if (_decompressor.NeedMoreData) 93 | { 94 | if (input.Length == 0) 95 | { 96 | // No more data to give 97 | goto exit; 98 | } 99 | 100 | int availableChunkBytes = _chunkSize - _chunkBytesProcessed; 101 | if (availableChunkBytes > input.Length) 102 | { 103 | _decompressor.Decompress(input); 104 | _chunkBytesProcessed += input.Length; 105 | input = default; 106 | } 107 | else 108 | { 109 | _decompressor.Decompress(input.Slice(0, availableChunkBytes)); 110 | _chunkBytesProcessed += availableChunkBytes; 111 | input = input.Slice(availableChunkBytes); 112 | } 113 | } 114 | 115 | int decompressedBytes = _decompressor.Read(buffer); 116 | 117 | _chunkCrc = Crc32CAlgorithm.Append(_chunkCrc, buffer.Slice(0, decompressedBytes)); 118 | 119 | buffer = buffer.Slice(decompressedBytes); 120 | } 121 | 122 | if (_decompressor.EndOfFile) 123 | { 124 | // Completed reading the chunk 125 | _chunkType = Constants.ChunkType.Null; 126 | 127 | uint crc = Crc32CAlgorithm.ApplyMask(_chunkCrc); 128 | if (_expectedChunkCrc != crc) 129 | { 130 | ThrowHelper.ThrowInvalidDataException("Chunk CRC mismatch."); 131 | } 132 | } 133 | 134 | break; 135 | } 136 | 137 | case Constants.ChunkType.UncompressedData: 138 | { 139 | if (_chunkBytesProcessed < 4) 140 | { 141 | if (!ReadChunkCrc(ref input)) 142 | { 143 | // Incomplete CRC 144 | goto exit; 145 | } 146 | 147 | if (input.Length == 0) 148 | { 149 | // No more data 150 | goto exit; 151 | } 152 | } 153 | 154 | int chunkBytes = unchecked(Math.Min(Math.Min(buffer.Length, input.Length), 155 | _chunkSize - _chunkBytesProcessed)); 156 | 157 | input.Slice(0, chunkBytes).CopyTo(buffer); 158 | 159 | _chunkCrc = Crc32CAlgorithm.Append(_chunkCrc, buffer.Slice(0, chunkBytes)); 160 | 161 | buffer = buffer.Slice(chunkBytes); 162 | input = input.Slice(chunkBytes); 163 | _chunkBytesProcessed += chunkBytes; 164 | 165 | if (_chunkBytesProcessed >= _chunkSize) 166 | { 167 | // Completed reading the chunk 168 | _chunkType = Constants.ChunkType.Null; 169 | 170 | uint crc = Crc32CAlgorithm.ApplyMask(_chunkCrc); 171 | if (_expectedChunkCrc != crc) 172 | { 173 | ThrowHelper.ThrowInvalidDataException("Chunk CRC mismatch."); 174 | } 175 | } 176 | 177 | break; 178 | } 179 | 180 | default: 181 | { 182 | if (_chunkType < Constants.ChunkType.SkippableChunk) 183 | { 184 | ThrowHelper.ThrowInvalidDataException($"Unknown chunk type {(int) _chunkType:x}"); 185 | } 186 | 187 | int chunkBytes = Math.Min(input.Length, _chunkSize - _chunkBytesProcessed); 188 | 189 | input = input.Slice(chunkBytes); 190 | _chunkBytesProcessed += chunkBytes; 191 | 192 | if (_chunkBytesProcessed >= _chunkSize) 193 | { 194 | // Completed reading the chunk 195 | _chunkType = Constants.ChunkType.Null; 196 | } 197 | 198 | break; 199 | } 200 | } 201 | } 202 | 203 | // We use a label and goto exit to avoid an unnecessary comparison on the while loop clause before 204 | // exiting the loop in cases where we know we're done processing data. 205 | exit: 206 | _input = _input.Slice(_input.Length - input.Length); 207 | return originalBufferLength - buffer.Length; 208 | } 209 | 210 | public void SetInput(ReadOnlyMemory input) 211 | { 212 | _input = input; 213 | } 214 | 215 | private uint ReadChunkHeader(ref ReadOnlySpan buffer) 216 | { 217 | if (_scratchLength > 0) 218 | { 219 | int bytesToCopyToScratch = 4 - _scratchLength; 220 | 221 | Span scratch = Scratch; 222 | buffer.Slice(0, bytesToCopyToScratch).CopyTo(scratch.Slice(_scratchLength)); 223 | 224 | buffer = buffer.Slice(bytesToCopyToScratch); 225 | _scratchLength += bytesToCopyToScratch; 226 | 227 | if (_scratchLength < 4) 228 | { 229 | // Insufficient data 230 | return 0; 231 | } 232 | 233 | _scratchLength = 0; 234 | return BinaryPrimitives.ReadUInt32LittleEndian(scratch); 235 | } 236 | 237 | if (buffer.Length < 4) 238 | { 239 | // Insufficient data 240 | 241 | buffer.CopyTo(Scratch); 242 | 243 | _scratchLength = buffer.Length; 244 | buffer = default; 245 | 246 | return 0; 247 | } 248 | else 249 | { 250 | uint result = BinaryPrimitives.ReadUInt32LittleEndian(buffer); 251 | buffer = buffer.Slice(4); 252 | return result; 253 | } 254 | } 255 | 256 | /// 257 | /// Assuming that we're at the beginning of a chunk, reads the CRC. If partially read, stores the value in 258 | /// Scratch for subsequent reads. Should not be called if chunkByteProcessed >= 4. 259 | /// 260 | private bool ReadChunkCrc(ref ReadOnlySpan input) 261 | { 262 | DebugExtensions.Assert(_chunkBytesProcessed < 4); 263 | 264 | if (_chunkBytesProcessed == 0 && input.Length >= 4) 265 | { 266 | // Common fast path 267 | 268 | _expectedChunkCrc = BinaryPrimitives.ReadUInt32LittleEndian(input); 269 | input = input.Slice(4); 270 | _chunkBytesProcessed += 4; 271 | return true; 272 | } 273 | 274 | // Copy to scratch 275 | int crcBytesAvailable = Math.Min(input.Length, 4 - _chunkBytesProcessed); 276 | input.Slice(0, crcBytesAvailable).CopyTo(Scratch.Slice(_scratchLength)); 277 | _scratchLength += crcBytesAvailable; 278 | input = input.Slice(crcBytesAvailable); 279 | _chunkBytesProcessed += crcBytesAvailable; 280 | 281 | if (_scratchLength >= 4) 282 | { 283 | _expectedChunkCrc = BinaryPrimitives.ReadUInt32LittleEndian(Scratch); 284 | _scratchLength = 0; 285 | return true; 286 | } 287 | 288 | return false; 289 | } 290 | 291 | public void Dispose() 292 | { 293 | _decompressor?.Dispose(); 294 | _decompressor = null; 295 | } 296 | } 297 | -------------------------------------------------------------------------------- /Snappier/Internal/CopyHelpers.cs: -------------------------------------------------------------------------------- 1 | using System.Diagnostics; 2 | using System.Runtime.CompilerServices; 3 | 4 | #if NET8_0_OR_GREATER 5 | using System.Runtime.InteropServices; 6 | using System.Runtime.Intrinsics; 7 | using System.Runtime.Intrinsics.X86; 8 | using static System.Runtime.Intrinsics.X86.Ssse3; 9 | #endif 10 | 11 | namespace Snappier.Internal; 12 | 13 | internal class CopyHelpers 14 | { 15 | #if NET8_0_OR_GREATER 16 | 17 | // Raw bytes for PshufbFillPatterns. This syntax returns a ReadOnlySpan that references 18 | // directly to the static data within the DLL. This is only supported with bytes due to things 19 | // like byte-ordering on various architectures, so we can reference Vector128 directly. 20 | // It is however safe to convert to Vector128 so we'll do that below with some casts 21 | // that are elided by JIT. 22 | private static ReadOnlySpan PshufbFillPatternsAsBytes => 23 | [ 24 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Never referenced, here for padding 25 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26 | 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 27 | 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 28 | 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 29 | 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 30 | 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 31 | 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1 32 | ]; 33 | 34 | /// 35 | /// This is a table of shuffle control masks that can be used as the source 36 | /// operand for PSHUFB to permute the contents of the destination XMM register 37 | /// into a repeating byte pattern. 38 | /// 39 | private static ReadOnlySpan> PshufbFillPatterns 40 | { 41 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 42 | get => MemoryMarshal.CreateReadOnlySpan( 43 | reference: ref Unsafe.As>(ref MemoryMarshal.GetReference(PshufbFillPatternsAsBytes)), 44 | length: 8); 45 | } 46 | 47 | /// 48 | /// j * (16 / j) for all j from 0 to 7. 0 is not actually used. 49 | /// 50 | private static ReadOnlySpan PatternSizeTable => [0, 16, 16, 15, 16, 15, 12, 14]; 51 | 52 | #endif 53 | 54 | /// 55 | /// Copy [src, src+(opEnd-op)) to [op, (opEnd-op)) but faster than 56 | /// IncrementalCopySlow. buf_limit is the address past the end of the writable 57 | /// region of the buffer. May write past opEnd, but won't write past bufferEnd. 58 | /// 59 | /// Pointer to the source point in the buffer. 60 | /// Pointer to the destination point in the buffer. 61 | /// Pointer to the end of the area to write in the buffer. 62 | /// Pointer past the end of the buffer. 63 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 64 | public static void IncrementalCopy(ref readonly byte source, ref byte op, ref byte opEnd, ref byte bufferEnd) 65 | { 66 | DebugExtensions.Assert(Unsafe.IsAddressLessThan(in source, in op)); 67 | DebugExtensions.Assert(!Unsafe.IsAddressGreaterThan(ref op, ref opEnd)); 68 | DebugExtensions.Assert(!Unsafe.IsAddressGreaterThan(ref opEnd, ref bufferEnd)); 69 | // NOTE: The copy tags use 3 or 6 bits to store the copy length, so len <= 64. 70 | DebugExtensions.Assert(Unsafe.ByteOffset(ref op, ref opEnd) <= (nint) 64); 71 | // NOTE: In practice the compressor always emits len >= 4, so it is ok to 72 | // assume that to optimize this function, but this is not guaranteed by the 73 | // compression format, so we have to also handle len < 4 in case the input 74 | // does not satisfy these conditions. 75 | 76 | int patternSize = (int) Unsafe.ByteOffset(in source, in op); 77 | 78 | if (patternSize < 8) 79 | { 80 | #if NET8_0_OR_GREATER 81 | if (Ssse3.IsSupported) // SSSE3 82 | { 83 | // Load the first eight bytes into an 128-bit XMM register, then use PSHUFB 84 | // to permute the register's contents in-place into a repeating sequence of 85 | // the first "pattern_size" bytes. 86 | // For example, suppose: 87 | // src == "abc" 88 | // op == op + 3 89 | // After _mm_shuffle_epi8(), "pattern" will have five copies of "abc" 90 | // followed by one byte of slop: abcabcabcabcabca. 91 | // 92 | // The non-SSE fallback implementation suffers from store-forwarding stalls 93 | // because its loads and stores partly overlap. By expanding the pattern 94 | // in-place, we avoid the penalty. 95 | 96 | if (!Unsafe.IsAddressGreaterThan(ref op, ref Unsafe.Subtract(ref bufferEnd, 16))) 97 | { 98 | Vector128 shuffleMask = PshufbFillPatterns[patternSize]; 99 | Vector128 srcPattern = Vector128.LoadUnsafe(in source); 100 | Vector128 pattern = Shuffle(srcPattern, shuffleMask); 101 | 102 | // Get the new pattern size now that we've repeated it 103 | patternSize = PatternSizeTable[patternSize]; 104 | 105 | // If we're getting to the very end of the buffer, don't overrun 106 | ref byte loopEnd = ref Unsafe.Subtract(ref bufferEnd, 15); 107 | if (Unsafe.IsAddressGreaterThan(ref loopEnd, ref opEnd)) 108 | { 109 | loopEnd = ref opEnd; 110 | } 111 | 112 | while (Unsafe.IsAddressLessThan(ref op, ref loopEnd)) 113 | { 114 | pattern.StoreUnsafe(ref op); 115 | op = ref Unsafe.Add(ref op, patternSize); 116 | } 117 | 118 | if (!Unsafe.IsAddressLessThan(ref op, ref opEnd)) 119 | { 120 | return; 121 | } 122 | } 123 | 124 | IncrementalCopySlow(in source, ref op, ref opEnd); 125 | return; 126 | } 127 | else 128 | { 129 | #endif 130 | // No SSSE3 Fallback 131 | 132 | // If plenty of buffer space remains, expand the pattern to at least 8 133 | // bytes. The way the following loop is written, we need 8 bytes of buffer 134 | // space if pattern_size >= 4, 11 bytes if pattern_size is 1 or 3, and 10 135 | // bytes if pattern_size is 2. Precisely encoding that is probably not 136 | // worthwhile; instead, invoke the slow path if we cannot write 11 bytes 137 | // (because 11 are required in the worst case). 138 | if (!Unsafe.IsAddressGreaterThan(ref op, ref Unsafe.Subtract(ref bufferEnd, 11))) 139 | { 140 | while (patternSize < 8) 141 | { 142 | UnalignedCopy64(in source, ref op); 143 | op = ref Unsafe.Add(ref op, patternSize); 144 | patternSize *= 2; 145 | } 146 | 147 | if (!Unsafe.IsAddressLessThan(ref op, ref opEnd)) 148 | { 149 | return; 150 | } 151 | } 152 | else 153 | { 154 | IncrementalCopySlow(in source, ref op, ref opEnd); 155 | return; 156 | } 157 | #if NET8_0_OR_GREATER 158 | } 159 | #endif 160 | } 161 | 162 | // ReSharper disable once ConditionIsAlwaysTrueOrFalse 163 | DebugExtensions.Assert(patternSize >= 8); 164 | 165 | // Copy 2x 8 bytes at a time. Because op - src can be < 16, a single 166 | // UnalignedCopy128 might overwrite data in op. UnalignedCopy64 is safe 167 | // because expanding the pattern to at least 8 bytes guarantees that 168 | // op - src >= 8. 169 | // 170 | // Typically, the op_limit is the gating factor so try to simplify the loop 171 | // based on that. 172 | if (!Unsafe.IsAddressGreaterThan(ref opEnd, ref Unsafe.Subtract(ref bufferEnd, 16))) 173 | { 174 | UnalignedCopy64(in source, ref op); 175 | UnalignedCopy64(in Unsafe.Add(in source, 8), ref Unsafe.Add(ref op, 8)); 176 | 177 | if (Unsafe.IsAddressLessThan(ref op, ref Unsafe.Subtract(ref opEnd, 16))) { 178 | UnalignedCopy64(in Unsafe.Add(in source, 16), ref Unsafe.Add(ref op, 16)); 179 | UnalignedCopy64(in Unsafe.Add(in source, 24), ref Unsafe.Add(ref op, 24)); 180 | } 181 | if (Unsafe.IsAddressLessThan(ref op, ref Unsafe.Subtract(ref opEnd, 32))) { 182 | UnalignedCopy64(in Unsafe.Add(in source, 32), ref Unsafe.Add(ref op, 32)); 183 | UnalignedCopy64(in Unsafe.Add(in source, 40), ref Unsafe.Add(ref op, 40)); 184 | } 185 | if (Unsafe.IsAddressLessThan(ref op, ref Unsafe.Subtract(ref opEnd, 48))) { 186 | UnalignedCopy64(in Unsafe.Add(in source, 48), ref Unsafe.Add(ref op, 48)); 187 | UnalignedCopy64(in Unsafe.Add(in source, 56), ref Unsafe.Add(ref op, 56)); 188 | } 189 | 190 | return; 191 | } 192 | 193 | // Fall back to doing as much as we can with the available slop in the 194 | // buffer. 195 | 196 | for (ref byte loopEnd = ref Unsafe.Subtract(ref bufferEnd, 16); 197 | Unsafe.IsAddressLessThan(ref op, ref loopEnd); 198 | op = ref Unsafe.Add(ref op, 16), source = ref Unsafe.Add(in source, 16)) 199 | { 200 | UnalignedCopy64(in source, ref op); 201 | UnalignedCopy64(in Unsafe.Add(in source, 8), ref Unsafe.Add(ref op, 8)); 202 | } 203 | 204 | if (!Unsafe.IsAddressLessThan(ref op, ref opEnd)) 205 | { 206 | return; 207 | } 208 | 209 | // We only take this branch if we didn't have enough slop and we can do a 210 | // single 8 byte copy. 211 | if (!Unsafe.IsAddressGreaterThan(ref op, ref Unsafe.Subtract(ref bufferEnd, 8))) 212 | { 213 | UnalignedCopy64(in source, ref op); 214 | source = ref Unsafe.Add(in source, 8); 215 | op = ref Unsafe.Add(ref op, 8); 216 | } 217 | 218 | IncrementalCopySlow(in source, ref op, ref opEnd); 219 | } 220 | 221 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 222 | public static void IncrementalCopySlow(ref readonly byte source, ref byte op, ref byte opEnd) 223 | { 224 | while (Unsafe.IsAddressLessThan(ref op, ref opEnd)) 225 | { 226 | op = source; 227 | op = ref Unsafe.Add(ref op, 1); 228 | source = ref Unsafe.Add(in source, 1); 229 | } 230 | } 231 | 232 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 233 | public static void UnalignedCopy64(ref readonly byte source, ref byte destination) 234 | { 235 | long tempStackVar = Unsafe.As(in source); 236 | Unsafe.As(ref destination) = tempStackVar; 237 | } 238 | 239 | [MethodImpl(MethodImplOptions.AggressiveInlining)] 240 | public static void UnalignedCopy128(ref readonly byte source, ref byte destination) 241 | { 242 | Guid tempStackVar = Unsafe.As(in source); 243 | Unsafe.As(ref destination) = tempStackVar; 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /Snappier/Snappy.cs: -------------------------------------------------------------------------------- 1 | using System.Buffers; 2 | using Snappier.Internal; 3 | 4 | namespace Snappier; 5 | 6 | /// 7 | /// Routines for performing Snappy compression and decompression on raw data blocks using . 8 | /// These routines do not read or write any Snappy framing. 9 | /// 10 | public static class Snappy 11 | { 12 | /// 13 | /// For a given amount of input data, calculate the maximum potential size of the compressed output. 14 | /// 15 | /// Length of the input data, in bytes. 16 | /// The maximum potential size of the compressed output. 17 | /// 18 | /// This is useful for allocating a sufficient output buffer before calling . 19 | /// 20 | public static int GetMaxCompressedLength(int inputLength) => 21 | // When used to allocate a precise buffer for compression, we need to also pad for the length encoding. 22 | // Failure to do so will cause the compression process to think the buffer may not be large enough after the 23 | // length is encoded and use a temporary buffer for compression which must then be copied. 24 | Helpers.MaxCompressedLength(inputLength) + VarIntEncoding.MaxLength; 25 | 26 | /// 27 | /// Compress a block of Snappy data. 28 | /// 29 | /// Data to compress. 30 | /// Buffer to receive the compressed data. 31 | /// Number of bytes written to . 32 | /// Output buffer is too small. 33 | /// Input and output spans must not overlap. 34 | /// 35 | /// The output buffer must be large enough to contain the compressed output. 36 | /// 37 | public static int Compress(ReadOnlySpan input, Span output) 38 | { 39 | if (!TryCompress(input, output, out int bytesWritten)) 40 | { 41 | ThrowHelper.ThrowArgumentExceptionInsufficientOutputBuffer(nameof(output)); 42 | } 43 | 44 | return bytesWritten; 45 | } 46 | 47 | /// 48 | /// Attempt to compress the input data into the output buffer. 49 | /// 50 | /// Data to compress. 51 | /// Buffer to receive the compressed data. 52 | /// Number of bytes written to the . 53 | /// Input and output spans must not overlap. 54 | /// true if the compression was successful, false if the output buffer is too small. 55 | public static bool TryCompress(ReadOnlySpan input, Span output, out int bytesWritten) 56 | { 57 | if (output.IsEmpty) 58 | { 59 | // Minimum of 1 byte is required to store a zero-length block, short circuit. 60 | bytesWritten = 0; 61 | return false; 62 | } 63 | 64 | using var compressor = new SnappyCompressor(); 65 | 66 | return compressor.TryCompress(input, output, out bytesWritten); 67 | } 68 | 69 | /// 70 | /// Compress a block of Snappy data. 71 | /// 72 | /// Data to compress. 73 | /// Buffer writer to receive the compressed data. 74 | /// is null. 75 | /// is larger than the maximum of 4,294,967,295 bytes. 76 | /// 77 | /// 78 | /// For the best performance, sequences with more than one segement should be comprised of segments some multiple of 64KB 79 | /// in size (i.e. 64KB or 128KB or 256KB each) with only the final segment varying. 80 | /// 81 | /// 82 | public static void Compress(ReadOnlySequence input, IBufferWriter output) 83 | { 84 | ArgumentNullException.ThrowIfNull(output); 85 | 86 | using var compressor = new SnappyCompressor(); 87 | 88 | compressor.Compress(input, output); 89 | } 90 | 91 | /// 92 | /// Compress a block of Snappy data. 93 | /// 94 | /// Data to compress. 95 | /// An with the compressed data. The caller is responsible for disposing this object. 96 | /// 97 | /// Failing to dispose of the returned may result in performance loss. 98 | /// 99 | public static IMemoryOwner CompressToMemory(ReadOnlySpan input) 100 | { 101 | byte[] buffer = ArrayPool.Shared.Rent(GetMaxCompressedLength(input.Length)); 102 | 103 | if (!TryCompress(input, buffer, out int length)) 104 | { 105 | // The amount of data written is unknown, so clear the entire buffer when returning 106 | ArrayPool.Shared.Return(buffer, clearArray: true); 107 | 108 | // Should be unreachable since we're allocating a buffer of the correct size. 109 | ThrowHelper.ThrowInvalidOperationException(); 110 | } 111 | 112 | return new ByteArrayPoolMemoryOwner(buffer, length); 113 | } 114 | 115 | /// 116 | /// Compress a block of Snappy data. 117 | /// 118 | /// Data to compress. 119 | /// 120 | /// The resulting byte array is allocated on the heap. If possible, should 121 | /// be used instead since it uses a shared buffer pool. 122 | /// 123 | public static byte[] CompressToArray(ReadOnlySpan input) 124 | { 125 | using IMemoryOwner buffer = CompressToMemory(input); 126 | Span bufferSpan = buffer.Memory.Span; 127 | 128 | byte[] result = new byte[bufferSpan.Length]; 129 | bufferSpan.CopyTo(result); 130 | return result; 131 | } 132 | 133 | /// 134 | /// Get the uncompressed data length from a compressed Snappy block. 135 | /// 136 | /// Compressed snappy block. 137 | /// The length of the uncompressed data in the block. 138 | /// The data in has an invalid length. 139 | /// 140 | /// This is useful for allocating a sufficient output buffer before calling . 141 | /// 142 | public static int GetUncompressedLength(ReadOnlySpan input) => 143 | SnappyDecompressor.ReadUncompressedLength(input); 144 | 145 | /// 146 | /// Decompress a block of Snappy data. This must be an entire block. 147 | /// 148 | /// Data to decompress. 149 | /// Buffer to receive the decompressed data. 150 | /// Number of bytes written to . 151 | /// Invalid Snappy block. 152 | /// Output buffer is too small. 153 | public static int Decompress(ReadOnlySpan input, Span output) 154 | { 155 | bool result = TryDecompress(input, output, out int bytesWritten); 156 | if (!result) 157 | { 158 | ThrowHelper.ThrowArgumentExceptionInsufficientOutputBuffer(nameof(output)); 159 | } 160 | 161 | return bytesWritten; 162 | } 163 | 164 | /// 165 | /// Decompress a block of Snappy data. This must be an entire block. 166 | /// 167 | /// Data to decompress. 168 | /// Buffer to receive the decompressed data. 169 | /// Number of bytes written to the . 170 | /// true if the compression was successful, false if the output buffer is too small. 171 | /// Invalid Snappy block. 172 | public static bool TryDecompress(ReadOnlySpan input, Span output, out int bytesWritten) 173 | { 174 | using var decompressor = new SnappyDecompressor(); 175 | 176 | decompressor.Decompress(input); 177 | 178 | if (!decompressor.AllDataDecompressed) 179 | { 180 | ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock(); 181 | } 182 | 183 | bytesWritten = decompressor.Read(output); 184 | 185 | return decompressor.EndOfFile; 186 | } 187 | 188 | /// 189 | /// Decompress a block of Snappy data. This must be an entire block. 190 | /// 191 | /// Data to decompress. 192 | /// Buffer writer to receive the decompressed data. 193 | /// Invalid Snappy block. 194 | public static void Decompress(ReadOnlySequence input, IBufferWriter output) 195 | { 196 | ArgumentNullException.ThrowIfNull(output); 197 | 198 | using var decompressor = new SnappyDecompressor() 199 | { 200 | BufferWriter = output 201 | }; 202 | 203 | foreach (ReadOnlyMemory segment in input) 204 | { 205 | decompressor.Decompress(segment.Span); 206 | } 207 | 208 | if (!decompressor.AllDataDecompressed) 209 | { 210 | ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock(); 211 | } 212 | } 213 | 214 | /// 215 | /// Decompress a block of Snappy data to a new memory buffer. This must be an entire block. 216 | /// 217 | /// Data to decompress. 218 | /// An with the decompressed data. The caller is responsible for disposing this object. 219 | /// Incomplete Snappy block. 220 | /// 221 | /// Failing to dispose of the returned may result in performance loss. 222 | /// 223 | public static IMemoryOwner DecompressToMemory(ReadOnlySpan input) 224 | { 225 | using var decompressor = new SnappyDecompressor(); 226 | 227 | decompressor.Decompress(input); 228 | 229 | if (!decompressor.AllDataDecompressed) 230 | { 231 | ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock(); 232 | } 233 | 234 | return decompressor.ExtractData(); 235 | } 236 | 237 | /// 238 | /// Decompress a block of Snappy data to a new memory buffer. This must be an entire block. 239 | /// 240 | /// Data to decompress. 241 | /// An with the decompressed data. The caller is responsible for disposing this object. 242 | /// Incomplete Snappy block. 243 | /// 244 | /// Failing to dispose of the returned may result in performance loss. 245 | /// 246 | public static IMemoryOwner DecompressToMemory(ReadOnlySequence input) 247 | { 248 | using var decompressor = new SnappyDecompressor(); 249 | 250 | foreach (ReadOnlyMemory segment in input) 251 | { 252 | decompressor.Decompress(segment.Span); 253 | } 254 | 255 | if (!decompressor.AllDataDecompressed) 256 | { 257 | ThrowHelper.ThrowInvalidDataExceptionIncompleteSnappyBlock(); 258 | } 259 | 260 | return decompressor.ExtractData(); 261 | } 262 | 263 | /// 264 | /// Decompress a block of Snappy to a new byte array. This must be an entire block. 265 | /// 266 | /// Data to decompress. 267 | /// The decompressed data. 268 | /// Invalid Snappy block. 269 | /// 270 | /// The resulting byte array is allocated on the heap. If possible, should 271 | /// be used instead since it uses a shared buffer pool. 272 | /// 273 | public static byte[] DecompressToArray(ReadOnlySpan input) 274 | { 275 | int length = GetUncompressedLength(input); 276 | 277 | byte[] result = new byte[length]; 278 | 279 | Decompress(input, result); 280 | 281 | return result; 282 | } 283 | } 284 | --------------------------------------------------------------------------------