├── .gitignore
├── LICENSE
├── README.md
└── src
    ├── .gitignore
    ├── Utf16StringFastCompression.Benchmark
        ├── .gitignore
        ├── Program.cs
        ├── SimdTest.cs
        ├── Test.cs
        └── Utf16StringFastCompression.Benchmark.csproj
    ├── Utf16StringFastCompression.Test
        ├── Tests.cs
        ├── Usings.cs
        └── Utf16StringFastCompression.Test.csproj
    ├── Utf16StringFastCompression.sln
    └── Utf16StringFastCompression
        ├── DetectTextKind.cs
        ├── GetByteCount.cs
        ├── GetBytes.cs
        ├── GetCharCount.cs
        ├── GetChars.cs
        ├── ToCharState.cs
        ├── Utf16CompressionEncoding.cs
        └── Utf16StringFastCompression.csproj


/.gitignore:
--------------------------------------------------------------------------------
1 | .vscode/**
2 | .vs/**
3 | **/bin/**
4 | **/obj/**
5 | **/*.bat
6 | **/*.ps1


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT LICENSE
 2 | 
 3 | CopyRight (c) 2022 pCYSl5EDgo
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Utf16StringFastCompression
  2 | 
  3 | Very Fast UTF16 compression library for .NET7
  4 | 
  5 | # Usage
  6 | 
  7 | ```
  8 | dotnet add package Utf16StringFastCompression
  9 | ```
 10 | 
 11 | All API is under Utf16StringFastCompression.Utf16CompressionEncoding static class.
 12 | 
 13 | ## Full API List
 14 | 
 15 | ### From char to byte
 16 | 
 17 | ```csharp
 18 | int GetBytes(ReadOnlySpan<char> source, Span<byte> destination)
 19 | int GetBytes(char[] source, int sourceLength, byte[] destination)
 20 | nint GetBytes(scoped ref char source, nint sourceLength, scoped ref byte destination)
 21 | 
 22 | int GetByteCount(ReadOnlySpan<char> source)
 23 | int GetByteCount(char[] source, int sourceLength)
 24 | nint GetByteCount(scoped ref char source, nint sourceLength)
 25 | 
 26 | T GetMaxByteCount<T>(T charCount) where T : IShiftOperators<T, int, T> => charCount << 1;
 27 | ```
 28 | 
 29 | ### From byte to char
 30 | 
 31 | ```csharp
 32 | int GetChars(ReadOnlySpan<byte> source, Span<char> destination)
 33 | int GetChars(byte[] source, int sourceLength, char[] destination)
 34 | nint GetChars(scoped ref byte source, nint sourceLength, scoped ref char destination)
 35 | nint GetCharsStateful(scoped ref byte source, nint sourceLength, scoped ref char destination, scoped ref ToCharState state)
 36 | 
 37 | int GetCharCount(ReadOnlySpan<byte> source)
 38 | int GetCharCount(byte[] source, int sourceLength)
 39 | long GetCharCount(in ReadOnlySequence<byte> source)
 40 | nint GetCharCount(scoped ref byte source, nint sourceLength)
 41 | nint GetCharCountStateful(scoped ref byte source, nint sourceLength, scoped ref ToCharState state)
 42 | 
 43 | string GetString(ReadOnlySpan<byte> source)
 44 | string GetString(ref byte source, nint sourceLength)
 45 | 
 46 | T GetMaxCharCount<T>(T byteCount) => byteCount;
 47 | ```
 48 | 
 49 | If you want to process byte stream, you can use GetChar(Count|s)Stateful apis.
 50 | 
 51 | ```csharp
 52 | public struct ToCharState
 53 | {
 54 |     public bool IsAsciiMode;
 55 |     public bool HasRemainingByte;
 56 |     public byte RemainingByte;
 57 | }
 58 | ```
 59 | 
 60 | ## Detect input text kind
 61 | 
 62 | ```csharp
 63 | TextKind DetectTextKind(ReadOnlySpan<char> source)
 64 | TextKind DetectTextKind(char[] source, int sourceLength)
 65 | TextKind DetectTextKind(ref char source, nint sourceLength)
 66 | 
 67 | public enum TextKind
 68 | {
 69 |     Mixed,
 70 |     AllNotInAsciiRange,
 71 |     AllInAsciiRange,
 72 | }
 73 | ```
 74 | 
 75 | # Compression Ratio
 76 | 
 77 | If all input text characters are in ASCII range then output bytes length is (text.Length + 2).
 78 | If all input text characters are not in ASCII range then output bytes length is (text.Length << 1), which is the very same length of original text.
 79 | The output bytes length will never be larger than that of the original.
 80 | 
 81 | # Benchmark
 82 | 
 83 | ## Serialize Performance char → byte
 84 | 
 85 | BenchmarkDotNet=v0.13.2, OS=Windows 10 (10.0.19044.2251/21H2/November2021Update)
 86 | Intel Core i7-8750H CPU 2.20GHz (Coffee Lake), 1 CPU, 12 logical and 6 physical cores
 87 | .NET SDK=7.0.100
 88 |   [Host]    : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2
 89 |   MediumRun : .NET 7.0.0 (7.0.22.51805), X64 RyuJIT AVX2
 90 | 
 91 | Job=MediumRun  IterationCount=15  LaunchCount=2  
 92 | WarmupCount=10  
 93 | 
 94 | |          Method |                 Text |       Mean |      Error |     StdDev |     Median |
 95 | |---------------- |--------------------- |-----------:|-----------:|-----------:|-----------:|
 96 | | SerializeFast | |       5.912 ns |     0.0850 ns |     0.1191 ns |       5.885 ns |
 97 | | SerializeUtf8 | |      13.139 ns |     0.1501 ns |     0.2153 ns |      13.064 ns |
 98 | | SerializeFast | P(...)iki [174742] | 143,324.921 ns | 4,975.5960 ns | 6,810.6519 ns | 139,827.893 ns |
 99 | | SerializeUtf8 | P(...)iki [174742] | 133,992.946 ns |   772.3141 ns | 1,107.6298 ns | 133,662.439 ns |
100 | | SerializeFast | very (...) text [21] |      19.239 ns |     0.7261 ns |     1.0868 ns |      19.482 ns |
101 | | SerializeUtf8 | very (...) text [21] |      19.612 ns |     0.4811 ns |     0.6256 ns |      19.253 ns |
102 | | SerializeFast |  走れメ(...)カード [10610] |   2,837.155 ns |    57.6724 ns |    76.9910 ns |   2,803.810 ns |
103 | | SerializeUtf8 |  走れメ(...)カード [10610] |  21,684.962 ns |   247.6680 ns |   330.6296 ns |  21,599.664 ns |
104 | 
105 | ## ByteCount Performance
106 | 
107 | |          Method |                 Text |       Mean |      Error |     StdDev |     Median |
108 | |---------------- |--------------------- |-----------:|-----------:|-----------:|-----------:|
109 | | ByteCountFast | |       1.898 ns |     0.0257 ns |     0.0368 ns |       1.881 ns |
110 | | ByteCountUtf8 | |       6.887 ns |     0.0798 ns |     0.1169 ns |       6.847 ns |
111 | | ByteCountFast | P(...)iki [174742] |  24,466.736 ns |   338.5406 ns |   463.3982 ns |  24,240.948 ns |
112 | | ByteCountUtf8 | P(...)iki [174742] |  28,463.019 ns |   361.9417 ns |   507.3917 ns |  28,173.419 ns |
113 | | ByteCountFast | very (...) text [21] |      10.327 ns |     0.1098 ns |     0.1540 ns |      10.275 ns |
114 | | ByteCountUtf8 | very (...) text [21] |       9.887 ns |     0.0260 ns |     0.0365 ns |       9.886 ns |
115 | | ByteCountFast |  走れメ(...)カード [10610] |   1,330.227 ns |    16.6922 ns |    24.9841 ns |   1,336.935 ns |
116 | | ByteCountUtf8 |  走れメ(...)カード [10610] |   1,394.888 ns |    24.4221 ns |    35.7976 ns |   1,388.348 ns |
117 | 
118 | ## Deserialize Performance byte → char
119 | 
120 | |          Method |                 Text |       Mean |      Error |     StdDev |     Median |
121 | |---------------- |--------------------- |-----------:|-----------:|-----------:|-----------:|
122 | | DeserializeFast | |   1.114 μs |  0.0821 μs |  0.1177 μs |   1.100 μs |
123 | | DeserializeUtf8 | |   2.593 μs |  0.8732 μs |  1.3070 μs |   1.750 μs |
124 | | DeserializeFast | P(...)iki [174742] | 155.986 μs | 18.8278 μs | 27.5975 μs | 157.600 μs |
125 | | DeserializeUtf8 | P(...)iki [174742] | 157.247 μs | 17.2575 μs | 25.8301 μs | 158.200 μs |
126 | | DeserializeFast | very (...) text [21] |   2.743 μs |  0.9677 μs |  1.3879 μs |   2.750 μs |
127 | | DeserializeUtf8 | very (...) text [21] |   3.524 μs |  1.0384 μs |  1.5221 μs |   3.500 μs |
128 | | DeserializeFast |  走れメ(...)カード [10610] |   6.092 μs |  0.3951 μs |  0.5275 μs |   5.900 μs |
129 | | DeserializeUtf8 |  走れメ(...)カード [10610] |  33.415 μs |  0.5359 μs |  0.7513 μs |  33.300 μs |
130 | 
131 | # Format Specification
132 | 
133 | ## Shorter than 4 chars
134 | 
135 | All inputs whose lengths are less than 4 chars are not serialized, they are just copied to the destination.
136 | "abc" => [0x61, 0x00, 0x62, 0xff, 0x63, 0xff]
137 | "あ" => [0x39, 0x30]
138 | 
139 | ## Longer than 3 chars
140 | 
141 | This format has 2 modes, ASCII mode and non-ASCII mode. The default mode is non-ASCII mode.
142 | 
143 | There are byte markers where the mode transitions occur.
144 | 
145 | * From ASCII mode to non-ASCII mode: 0xff
146 | * From non-ASCII mode to ASCII mode: 0xff, 0xff
147 | 
148 | **Note:** ASCII mode consists of more than 3 chars.


--------------------------------------------------------------------------------
/src/.gitignore:
--------------------------------------------------------------------------------
1 | **/.vs/**
2 | **/.vscode/**


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Benchmark/.gitignore:
--------------------------------------------------------------------------------
1 | BenchmarkDotNet.Artifacts/**


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Benchmark/Program.cs:
--------------------------------------------------------------------------------
1 | using BenchmarkDotNet.Running;
2 | 
3 | // BenchmarkRunner.Run<SerializeTest>();
4 | // BenchmarkRunner.Run<DeserializeTest>();
5 | // BenchmarkRunner.Run<PextTest>();
6 | // BenchmarkRunner.Run<ContinuousTest>();
7 | BenchmarkRunner.Run<NarrowTest>();


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Benchmark/SimdTest.cs:
--------------------------------------------------------------------------------
  1 | using System;
  2 | using System.Text;
  3 | using System.Text.Unicode;
  4 | using System.Runtime.CompilerServices;
  5 | using System.Runtime.InteropServices;
  6 | using System.Runtime.Intrinsics;
  7 | using System.Runtime.Intrinsics.X86;
  8 | using Utf16StringFastCompression;
  9 | using BenchmarkDotNet.Attributes;
 10 | 
 11 | [MediumRunJob]
 12 | public class ContinuousTest
 13 | {
 14 |     private uint[] array = new uint[4096];
 15 |     [IterationSetup]
 16 |     public void Setup()
 17 |     {
 18 |         System.Random.Shared.NextBytes(MemoryMarshal.Cast<uint, byte>(array.AsSpan()));
 19 |     }
 20 | 
 21 |     [Benchmark]
 22 |     public uint Continuous4_2stepLoop()
 23 |     {
 24 |         uint answer = 0; for (nint i = 0; i < array.Length; ++i) answer += Continuous4_2step(array[i]); return answer;
 25 |     }
 26 |     [Benchmark]
 27 |     public uint Continuous4_4stepLoop()
 28 |     {
 29 |         uint answer = 0; for (nint i = 0; i < array.Length; ++i) answer += Continuous4_4step(array[i]); return answer;
 30 |     }
 31 |     [Benchmark]
 32 |     public uint Continuous4_Pext_2stepLoop()
 33 |     {
 34 |         uint answer = 0; for (nint i = 0; i < array.Length; ++i) answer += Continuous4_Pext_2step(array[i]); return answer;
 35 |     }
 36 |     public static uint Continuous4_Pext_2step(uint bits)
 37 |     {
 38 |         if (!Bmi2.IsSupported) { return 0; }
 39 |         bits = Bmi2.ParallelBitExtract(bits, 0x55555555U);
 40 |         bits |= bits >>> 2;
 41 |         return bits | (bits >>> 4);
 42 |     }
 43 |     public static uint Continuous4_4step(uint bits) => bits | (bits >>> 2) | (bits >>> 4) | (bits >>> 6);
 44 |     public static uint Continuous4_2step(uint bits)
 45 |     {
 46 |         bits |= (bits >>> 2);
 47 |         return bits | (bits >>> 4);
 48 |     }
 49 | }
 50 | 
 51 | [MediumRunJob]
 52 | public class NarrowTest
 53 | {
 54 |     private byte[] array = new byte[4096 * 32];
 55 |     [IterationSetup]
 56 |     public void Setup()
 57 |     {
 58 |         System.Random.Shared.NextBytes(array.AsSpan());
 59 |     }
 60 | 
 61 |     [Benchmark]
 62 |     public void Loop128()
 63 |     {
 64 |         var destination = new byte[array.Length >>> 1];
 65 |         ref var destItr = ref MemoryMarshal.GetArrayDataReference(destination);
 66 |         ref var itr = ref MemoryMarshal.GetArrayDataReference(array);
 67 |         ref var itrEnd = ref Unsafe.Add(ref itr, array.Length);
 68 |         do
 69 |         {
 70 |             Vector256<ushort> vec = Vector256.LoadUnsafe(ref itr).AsUInt16();
 71 |             Vector128<byte> narrow = Vector128.Narrow(vec.AsUInt16().GetLower(), vec.AsUInt16().GetUpper());
 72 |             narrow.StoreUnsafe(ref destItr);
 73 |             itr = ref Unsafe.Add(ref itr, 32);
 74 |             destItr = ref Unsafe.Add(ref destItr, 16);
 75 |         } while (!Unsafe.AreSame(ref itr, ref itrEnd));
 76 |     }
 77 | 
 78 |     [Benchmark]
 79 |     public void Loop256Self()
 80 |     {
 81 |         var destination = new byte[array.Length >>> 1];
 82 |         ref var destItr = ref MemoryMarshal.GetArrayDataReference(destination);
 83 |         ref var itr = ref MemoryMarshal.GetArrayDataReference(array);
 84 |         ref var itrEnd = ref Unsafe.Add(ref itr, array.Length);
 85 |         do
 86 |         {
 87 |             Vector256<ushort> vec = Vector256.LoadUnsafe(ref itr).AsUInt16();
 88 |             var lower = Vector256.Narrow(vec, vec).GetLower();
 89 |             lower.StoreUnsafe(ref destItr);
 90 |             itr = ref Unsafe.Add(ref itr, 32);
 91 |             destItr = ref Unsafe.Add(ref destItr, 16);
 92 |         } while (!Unsafe.AreSame(ref itr, ref itrEnd));
 93 |     }
 94 | 
 95 |     [Benchmark]
 96 |     public void Loop256()
 97 |     {
 98 |         var destination = new byte[array.Length >>> 1];
 99 |         ref var destItr = ref MemoryMarshal.GetArrayDataReference(destination);
100 |         ref var itr = ref MemoryMarshal.GetArrayDataReference(array);
101 |         ref var itrEnd = ref Unsafe.Add(ref itr, array.Length);
102 |         do
103 |         {
104 |             Vector256<byte> narrow = Vector256.Narrow(Vector256.LoadUnsafe(ref itr).AsUInt16(), Vector256.LoadUnsafe(ref Unsafe.Add(ref itr, 32)).AsUInt16());
105 |             narrow.StoreUnsafe(ref destItr);
106 |             itr = ref Unsafe.Add(ref itr, 64);
107 |             destItr = ref Unsafe.Add(ref destItr, 32);
108 |         } while (!Unsafe.AreSame(ref itr, ref itrEnd));
109 |     }
110 | }
111 | 
112 | [MediumRunJob]
113 | public class PextTest
114 | {
115 |     private Vector256<ushort>[] array = new Vector256<ushort>[4096];
116 |     [IterationSetup]
117 |     public void Setup()
118 |     {
119 |         System.Random.Shared.NextBytes(MemoryMarshal.Cast<Vector256<ushort>, byte>(array.AsSpan()));
120 |     }
121 | 
122 |     [Benchmark]
123 |     public ulong MSBLoop()
124 |     {
125 |         ulong answer = 0;
126 |         for (nint i = 0; i < array.Length; ++i)
127 |         {
128 |             ref var vector = ref array[i];
129 |             answer += vector.ExtractMostSignificantBits();
130 |         }
131 |         return answer;
132 |     }
133 |     [Benchmark]
134 |     public ulong PextLoop()
135 |     {
136 |         ulong answer = 0;
137 |         for (nint i = 0; i < array.Length; ++i)
138 |         {
139 |             ref var vector = ref array[i];
140 |             answer += Pext(vector.AsByte().ExtractMostSignificantBits());
141 |         }
142 |         return answer;
143 |     }
144 |     [Benchmark]
145 |     public ulong NarrowLoop()
146 |     {
147 |         ulong answer = 0;
148 |         for (nint i = 0; i < array.Length; ++i)
149 |         {
150 |             ref var vector = ref array[i];
151 |             answer += Narrow(vector.AsByte().ExtractMostSignificantBits());
152 |         }
153 |         return answer;
154 |     }
155 | 
156 |     public static ulong Pext(ulong value)
157 |     {
158 |         if (Bmi2.X64.IsSupported)
159 |         {
160 |             return Bmi2.X64.ParallelBitExtract(value, 0x55555555_55555555UL);
161 |         }
162 |         return 0;
163 |     }
164 | 
165 |     public static ulong Narrow(ulong value)
166 |     {
167 |         return
168 |            (value & 0x0000_0000_0000_0001UL)
169 |         | ((value & 0x0000_0000_0000_0004UL) >>> 1)
170 |         | ((value & 0x0000_0000_0000_0010UL) >>> 2)
171 |         | ((value & 0x0000_0000_0000_0040UL) >>> 3)
172 |         | ((value & 0x0000_0000_0000_0100UL) >>> 4)
173 |         | ((value & 0x0000_0000_0000_0400UL) >>> 5)
174 |         | ((value & 0x0000_0000_0000_1000UL) >>> 6)
175 |         | ((value & 0x0000_0000_0000_4000UL) >>> 7)
176 |         | ((value & 0x0000_0000_0001_0000UL) >>> 8)
177 |         | ((value & 0x0000_0000_0004_0000UL) >>> 9)
178 |         | ((value & 0x0000_0000_0010_0000UL) >>> 10)
179 |         | ((value & 0x0000_0000_0040_0000UL) >>> 11)
180 |         | ((value & 0x0000_0000_0100_0000UL) >>> 12)
181 |         | ((value & 0x0000_0000_0400_0000UL) >>> 13)
182 |         | ((value & 0x0000_0000_1000_0000UL) >>> 14)
183 |         | ((value & 0x0000_0000_4000_0000UL) >>> 15)
184 |         | ((value & 0x0000_0001_0000_0000UL) >>> 0x10)
185 |         | ((value & 0x0000_0004_0000_0000UL) >>> 0x11)
186 |         | ((value & 0x0000_0010_0000_0000UL) >>> 0x12)
187 |         | ((value & 0x0000_0040_0000_0000UL) >>> 0x13)
188 |         | ((value & 0x0000_0100_0000_0000UL) >>> 0x14)
189 |         | ((value & 0x0000_0400_0000_0000UL) >>> 0x15)
190 |         | ((value & 0x0000_1000_0000_0000UL) >>> 0x16)
191 |         | ((value & 0x0000_4000_0000_0000UL) >>> 0x17)
192 |         | ((value & 0x0001_0000_0000_0000UL) >>> 0x18)
193 |         | ((value & 0x0004_0000_0000_0000UL) >>> 0x19)
194 |         | ((value & 0x0010_0000_0000_0000UL) >>> 0x1b)
195 |         | ((value & 0x0040_0000_0000_0000UL) >>> 0x1a)
196 |         | ((value & 0x0100_0000_0000_0000UL) >>> 0x1c)
197 |         | ((value & 0x0400_0000_0000_0000UL) >>> 0x1d)
198 |         | ((value & 0x1000_0000_0000_0000UL) >>> 0x1e)
199 |         | ((value & 0x4000_0000_0000_0000UL) >>> 0x1f);
200 |     }
201 | }
202 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Benchmark/Utf16StringFastCompression.Benchmark.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk">
 2 | 
 3 |   <ItemGroup>
 4 |     <ProjectReference Include="..\Utf16StringFastCompression\Utf16StringFastCompression.csproj" />
 5 |   </ItemGroup>
 6 | 
 7 |   <ItemGroup>
 8 |     <PackageReference Include="BenchmarkDotNet" Version="0.13.2" />
 9 |   </ItemGroup>
10 | 
11 |   <PropertyGroup>
12 |     <OutputType>Exe</OutputType>
13 |     <TargetFramework>net7.0</TargetFramework>
14 |     <ImplicitUsings>enable</ImplicitUsings>
15 |     <Nullable>enable</Nullable>
16 |   </PropertyGroup>
17 | 
18 | </Project>
19 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Test/Usings.cs:
--------------------------------------------------------------------------------
1 | global using Xunit;


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.Test/Utf16StringFastCompression.Test.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk">
 2 | 
 3 |   <PropertyGroup>
 4 |     <TargetFramework>net7.0</TargetFramework>
 5 |     <ImplicitUsings>enable</ImplicitUsings>
 6 |     <Nullable>enable</Nullable>
 7 | 
 8 |     <IsPackable>false</IsPackable>
 9 |   </PropertyGroup>
10 | 
11 |   <ItemGroup>
12 |     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.3.2" />
13 |     <PackageReference Include="xunit" Version="2.4.2" />
14 |     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
15 |       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
16 |       <PrivateAssets>all</PrivateAssets>
17 |     </PackageReference>
18 |     <PackageReference Include="coverlet.collector" Version="3.1.2">
19 |       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
20 |       <PrivateAssets>all</PrivateAssets>
21 |     </PackageReference>
22 |   </ItemGroup>
23 | 
24 |   <ItemGroup>
25 |     <ProjectReference Include="..\Utf16StringFastCompression\Utf16StringFastCompression.csproj" />
26 |   </ItemGroup>
27 | 
28 | </Project>
29 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression.sln:
--------------------------------------------------------------------------------
 1 | 
 2 | Microsoft Visual Studio Solution File, Format Version 12.00
 3 | # Visual Studio Version 17
 4 | VisualStudioVersion = 17.0.31903.59
 5 | MinimumVisualStudioVersion = 10.0.40219.1
 6 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utf16StringFastCompression", "Utf16StringFastCompression\Utf16StringFastCompression.csproj", "{BBE99010-3FC2-4983-8887-7696F0981CBF}"
 7 | EndProject
 8 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utf16StringFastCompression.Benchmark", "Utf16StringFastCompression.Benchmark\Utf16StringFastCompression.Benchmark.csproj", "{2C84800C-06D8-408B-977D-2BE99F9C9A56}"
 9 | EndProject
10 | Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utf16StringFastCompression.Test", "Utf16StringFastCompression.Test\Utf16StringFastCompression.Test.csproj", "{C0863788-5C40-402C-A1B7-D0092D6DE2EF}"
11 | EndProject
12 | Global
13 | 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
14 | 		Debug|Any CPU = Debug|Any CPU
15 | 		Release|Any CPU = Release|Any CPU
16 | 	EndGlobalSection
17 | 	GlobalSection(SolutionProperties) = preSolution
18 | 		HideSolutionNode = FALSE
19 | 	EndGlobalSection
20 | 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
21 | 		{BBE99010-3FC2-4983-8887-7696F0981CBF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
22 | 		{BBE99010-3FC2-4983-8887-7696F0981CBF}.Debug|Any CPU.Build.0 = Debug|Any CPU
23 | 		{BBE99010-3FC2-4983-8887-7696F0981CBF}.Release|Any CPU.ActiveCfg = Release|Any CPU
24 | 		{BBE99010-3FC2-4983-8887-7696F0981CBF}.Release|Any CPU.Build.0 = Release|Any CPU
25 | 		{2C84800C-06D8-408B-977D-2BE99F9C9A56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
26 | 		{2C84800C-06D8-408B-977D-2BE99F9C9A56}.Debug|Any CPU.Build.0 = Debug|Any CPU
27 | 		{2C84800C-06D8-408B-977D-2BE99F9C9A56}.Release|Any CPU.ActiveCfg = Release|Any CPU
28 | 		{2C84800C-06D8-408B-977D-2BE99F9C9A56}.Release|Any CPU.Build.0 = Release|Any CPU
29 | 		{C0863788-5C40-402C-A1B7-D0092D6DE2EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
30 | 		{C0863788-5C40-402C-A1B7-D0092D6DE2EF}.Debug|Any CPU.Build.0 = Debug|Any CPU
31 | 		{C0863788-5C40-402C-A1B7-D0092D6DE2EF}.Release|Any CPU.ActiveCfg = Release|Any CPU
32 | 		{C0863788-5C40-402C-A1B7-D0092D6DE2EF}.Release|Any CPU.Build.0 = Release|Any CPU
33 | 	EndGlobalSection
34 | EndGlobal
35 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/DetectTextKind.cs:
--------------------------------------------------------------------------------
 1 | namespace Utf16StringFastCompression;
 2 | 
 3 | partial class Utf16CompressionEncoding
 4 | {
 5 |     public static TextKind DetectTextKind(ReadOnlySpan<char> source) => DetectTextKind(ref MemoryMarshal.GetReference(source), source.Length);
 6 |     
 7 |     public static TextKind DetectTextKind(char[] source, int sourceLength) => DetectTextKind(ref MemoryMarshal.GetArrayDataReference(source), sourceLength);
 8 | 
 9 |     public static TextKind DetectTextKind(ref char source, nint sourceLength)
10 |     {
11 |         if (Unsafe.IsNullRef(ref source) || sourceLength <= 0)
12 |         {
13 |             return default;
14 |         }
15 |         
16 |         var isAscii = ((ushort)source) < 0x80;
17 |         ref var sourceEnd = ref Unsafe.Add(ref source, sourceLength);
18 |         source = ref Unsafe.Add(ref source, 1);
19 |         if (Vector256.IsHardwareAccelerated && Unsafe.ByteOffset(ref source, ref sourceEnd) >= Unsafe.SizeOf<Vector256<ushort>>())
20 |         {
21 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
22 |             var filter = Vector256.Create<ushort>(0xff80);
23 |             if (isAscii)
24 |             {
25 |                 while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
26 |                 {
27 |                     var vector = Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
28 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
29 |                     if ((vector & filter) != Vector256<ushort>.Zero)
30 |                     {
31 |                         return default;
32 |                     }
33 |                 }
34 |             }
35 |             else
36 |             {
37 |                 while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
38 |                 {
39 |                     var vector = Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
40 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
41 |                     if (Vector256.EqualsAny(vector & filter, Vector256<ushort>.Zero))
42 |                     {
43 |                         return default;
44 |                     }
45 |                 }
46 |             }
47 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
48 |         }
49 | 
50 |         if (Vector128.IsHardwareAccelerated && Unsafe.ByteOffset(ref source, ref sourceEnd) >= Unsafe.SizeOf<Vector128<ushort>>())
51 |         {
52 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
53 |             var filter = Vector128.Create<ushort>(0xff80);
54 |             if (isAscii)
55 |             {
56 |                 while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
57 |                 {
58 |                     var vector = Vector128.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
59 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
60 |                     if ((vector & filter) != Vector128<ushort>.Zero)
61 |                     {
62 |                         return default;
63 |                     }
64 |                 }
65 |             }
66 |             else
67 |             {
68 |                 while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
69 |                 {
70 |                     var vector = Vector128.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
71 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
72 |                     if (Vector128.EqualsAny(vector & filter, Vector128<ushort>.Zero))
73 |                     {
74 |                         return default;
75 |                     }
76 |                 }
77 |             }
78 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
79 |         }
80 | 
81 |         while (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
82 |         {
83 |             if (((ushort)source < 0x80) ^ isAscii)
84 |             {
85 |                 return default;
86 |             }
87 |         }
88 | 
89 |         return (TextKind)(1 + (isAscii ? 1 : 0));
90 |     }
91 | }
92 | 
93 | public enum TextKind
94 | {
95 |     Mixed,
96 |     AllNotInAsciiRange,
97 |     AllInAsciiRange,
98 | }
99 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/GetByteCount.cs:
--------------------------------------------------------------------------------
  1 | namespace Utf16StringFastCompression;
  2 | 
  3 | partial class Utf16CompressionEncoding
  4 | {
  5 |     public static int GetByteCount(ReadOnlySpan<char> source) => GetByteCount(ref MemoryMarshal.GetReference(source), source.Length).ToInt32();
  6 | 
  7 |     public static int GetByteCount(char[] source, int sourceLength) => GetByteCount(ref MemoryMarshal.GetArrayDataReference(source), sourceLength).ToInt32();
  8 | 
  9 | /*
 10 | ec4mask = uint.MaxValue >>> 16;
 11 | ec4Accum += popcnt(ec4 & ec4mask);
 12 | difmask = uint.MaxValue >>> 13;
 13 | difAccum += popcnt(dif & dif)
 14 | ByteCount = (length << 1) - ec4Accum + (1 + difAccum * 3 >>> 1)
 15 | 
 16 | 一番最初に連続ASCIIがある場合
 17 |         | ←     Lower         Upper     →
 18 |   LAST  | bits
 19 | 0 0 0 0 | 1 1 1 1 0 0 0 0 0 0 0 0 1 1 1 1 _c
 20 |         | c4 = bits & (bits >>> 1) & (bits >>> 2) & (bits >>> 3)
 21 | 0 0 0 0 | 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 _c
 22 |         | ec4 = c4 | (c4 << 1) | (c4 << 2) | (c4 << 3)
 23 | 0 0 0 0 | 1 1 1 1 0 0 0 0 0 0 0 0 1 1 1 1 _c
 24 |         | tec4 = (uint)((int)(~ec4) >> 1)
 25 | 1 1 1 0 | 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 1 !c
 26 |         | dif = ~(ec4 ^ tex4)
 27 | 0 0 0 1 | 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 _c
 28 | 
 29 | 
 30 | */
 31 | 
 32 |     public static nint GetByteCount(scoped ref char source, nint sourceLength)
 33 |     {
 34 |         if (sourceLength < 4)
 35 |         {
 36 |             return sourceLength << 1;
 37 |         }
 38 | 
 39 |         scoped ref char sourceEnd = ref Unsafe.Add(ref source, sourceLength);
 40 |         if (Vector256.IsHardwareAccelerated)
 41 |         {
 42 |             var filter = Vector256.Create((ushort)0xff80);
 43 |             nint asciiAccum = 0, changeAccum = 0;
 44 |             uint bitCarrier = 0;
 45 |             nint loopCount = (sourceLength - 1) >>> 4;
 46 |             var restLength = (int)(sourceLength - (loopCount << 4));
 47 |             if (Bmi2.IsSupported)
 48 |             {
 49 |                 while (loopCount >= 3)
 50 |                 {
 51 |                     ulong bits = (ulong)bitCarrier | ((ulong)(Bmi2.ParallelBitExtract(Vector256.Equals(Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source)) & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits(), 0x55555555U)) << 4)
 52 |                         | ((ulong)(Bmi2.ParallelBitExtract(Vector256.Equals(Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref Unsafe.AddByteOffset(ref source, 32))) & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits(), 0x55555555U)) << 20)
 53 |                         | ((ulong)(Bmi2.ParallelBitExtract(Vector256.Equals(Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref Unsafe.AddByteOffset(ref source, 64))) & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits(), 0x55555555U)) << 36);
 54 |                     bitCarrier = (uint)((bits >>> 48) & 0xFUL);
 55 |                     var c4 = bits & (bits >>> 1) & (bits >>> 2) & (bits >>> 3);
 56 |                     c4  |= (c4 << 1) | (c4 << 2) | (c4 << 3);
 57 |                     asciiAccum += BitOperations.PopCount(c4) - ((bitCarrier == 0xFU ? 1 : 0) << 2);
 58 |                     changeAccum += BitOperations.PopCount((c4 ^ (c4 >>> 1)) & ((1UL << 51) - 1UL));
 59 |                     source = ref Unsafe.AddByteOffset(ref source, 96);
 60 |                     loopCount -= 3;
 61 |                 }
 62 |                 bitCarrier = GetBitCarrierConversionTable()[(int)bitCarrier];
 63 |             }
 64 |             while (--loopCount >= 0)
 65 |             {
 66 |                 ulong bits = (((ulong)Vector256.Equals(Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source)) & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits()) << 8) | (ulong)bitCarrier;
 67 |                 bitCarrier = ((uint)(bits >>> 32)) & 0xFFU;
 68 |                 var c4 = bits & (bits >>> 2) & (bits >>> 4) & (bits >>> 6);
 69 |                 c4 |= (c4 << 2) | (c4 << 4) | (c4 << 6);
 70 |                 asciiAccum += (BitOperations.PopCount(c4) >>> 1) - ((bitCarrier == 0xFFU ? 1 : 0) << 2);
 71 |                 changeAccum += BitOperations.PopCount((c4 ^ (c4 >>> 2)) & 0x3F_FFFFFFFFUL) >>> 1;
 72 |                 source = ref Unsafe.AddByteOffset(ref source, 32);
 73 |             }
 74 |             {
 75 |                 ulong bitsMask = (1UL << ((restLength + 4) << 1)) - 1UL;
 76 |                 ulong bits = ((((ulong)Vector256.Equals(Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source)) & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits()) << 8) | bitCarrier) & bitsMask;
 77 |                 if (bits != 0UL)
 78 |                 {
 79 |                     bits = bits & (bits >>> 2) & (bits >>> 4) & (bits >>> 6);
 80 |                     bits = bits | (bits << 2) | (bits << 4) | (bits << 6);
 81 |                     asciiAccum += BitOperations.PopCount(bits) >>> 1;
 82 |                     changeAccum += BitOperations.PopCount((bits ^ (bits >>> 2)) & (bitsMask >>> 2)) >>> 1;
 83 |                 }
 84 |                 return (sourceLength << 1) - asciiAccum + ((changeAccum * 3 + 1) >>> 1);
 85 |             }
 86 |         }
 87 | 
 88 |         var status = 0U;
 89 |         nint answer = 0;
 90 |         while (!Unsafe.AreSame(ref source, ref sourceEnd))
 91 |         {
 92 |             var value = (ushort)source;
 93 |             source = ref Unsafe.Add(ref source, 1);
 94 |             if (value >= 0x80)
 95 |             {
 96 |                 answer += (status >= 4 ? 1 : 0) + 2;
 97 |                 status = 0;
 98 |                 continue;
 99 |             }
100 |             if (++status < 4)
101 |             {
102 |                 answer += 2;
103 |                 continue;
104 |             }
105 |             answer += status != 4 ? 1 : 0;
106 |             status = 4;
107 |         }
108 | 
109 |         return answer;
110 |     }
111 | 
112 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
113 |     private static nint ByteCountTransitFromAsciiToUnicode(scoped ref uint status)
114 |     {
115 |         var temp = status;
116 |         status = 0;
117 |         return temp >= 4 ? 1 : 0;
118 |     }
119 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
120 |     private static nint ByteCountTransitFromUnicodeToAscii(scoped ref uint status)
121 |     {
122 |         var temp = status;
123 |         status = 4;
124 |         return temp < 4 ? (2 - (int)temp) : 0;
125 |     }
126 |     private static ReadOnlySpan<byte> GetBitCarrierConversionTable() => new byte[] { 0, 0b11, 0b1100, 0b1111, 0b11_0000, 0b11_0011, 0b11_1100, 0b11_1111, 0b1100_0000, 0b1100_0011, 0b1100_1100, 0b1100_1111, 0b1111_0000, 0b1111_0011, 0b1111_1100, 0b1111_1111, };
127 | }
128 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/GetBytes.cs:
--------------------------------------------------------------------------------
  1 | namespace Utf16StringFastCompression;
  2 | 
  3 | partial class Utf16CompressionEncoding
  4 | {
  5 |     /// <summary>
  6 |     /// Serialize to span.
  7 |     /// </summary>
  8 |     /// <param name="source">Source Char Span</param>
  9 |     /// <param name="destination">Destination Byte Span</param>
 10 |     /// <returns>The number of encoded bytes.</returns>
 11 |     public static int GetBytes(ReadOnlySpan<char> source, Span<byte> destination) => checked((int)GetBytes(ref MemoryMarshal.GetReference(source), source.Length, ref MemoryMarshal.GetReference(destination)));
 12 | 
 13 |     public static int GetBytes(char[] source, int sourceLength, byte[] destination) => checked((int)GetBytes(ref MemoryMarshal.GetArrayDataReference(source), sourceLength, ref MemoryMarshal.GetArrayDataReference(destination)));
 14 | 
 15 |     /// <summary>
 16 |     /// Serialize to reference.
 17 |     /// </summary>
 18 |     /// <param name="source">Source Char Reference</param>
 19 |     /// <param name="sourceLength">Source Char Length</param>
 20 |     /// <param name="destination">Destination Byte Reference</param>
 21 |     /// <returns>The number of encoded bytes.</returns>
 22 |     public static nint GetBytes(scoped ref char source, nint sourceLength, scoped ref byte destination)
 23 |     {
 24 |         if (sourceLength < 4)
 25 |         {
 26 |             if (sourceLength <= 0)
 27 |             {
 28 |                 return 0;
 29 |             }
 30 | 
 31 |             sourceLength <<= 1;
 32 |             Unsafe.CopyBlockUnaligned(ref destination, ref Unsafe.As<char, byte>(ref source), (uint)sourceLength);
 33 |             return sourceLength;
 34 |         }
 35 | 
 36 |         scoped ref byte initialDestination = ref destination;
 37 |         scoped ref char sourceEnd = ref Unsafe.Add(ref source, sourceLength);
 38 |         var status = 0U;
 39 |         if (Vector256.IsHardwareAccelerated && Unsafe.ByteOffset(ref source, ref sourceEnd) >= Unsafe.SizeOf<Vector256<byte>>())
 40 |         {
 41 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<byte>>());
 42 |             var filter = Vector256.Create((ushort)0xff80);
 43 |             while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
 44 |             {
 45 |                 var vec = Vector256.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
 46 |                 // 00 → unicode, 11 → ascii
 47 |                 var bits = Vector256.Equals(vec & filter, Vector256<ushort>.Zero).AsByte().ExtractMostSignificantBits();
 48 |                 if (bits == 0U)
 49 |                 {
 50 |                     destination = ref TransitFromAsciiToUnicode(ref destination, ref status);
 51 |                     destination = ref Write(ref destination, ref vec);
 52 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<byte>>());
 53 |                     continue;
 54 |                 }
 55 |                 if (bits == uint.MaxValue)
 56 |                 {
 57 |                     destination = ref TransitFromUnicodeToAscii(ref destination, ref status);
 58 |                     var narrow = Vector256.Narrow(vec, vec).GetLower();
 59 |                     destination = ref Write(ref destination, ref narrow);
 60 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<byte>>());
 61 |                     continue;
 62 |                 }
 63 |                 if ((bits & 1U) == 0U)
 64 |                 {
 65 |                     destination = ref TransitFromAsciiToUnicode(ref destination, ref status);
 66 |                     var shift = bits & (bits >>> 2) & (bits >>> 4) & (bits >>> 6);
 67 |                     if (shift == 0U)
 68 |                     {
 69 |                         destination = ref Write(ref destination, ref vec);
 70 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<byte>>());
 71 |                         status = (uint)(BitOperations.LeadingZeroCount(~bits) >>> 1);
 72 |                         continue;
 73 |                     }
 74 | 
 75 |                     var copySize = BitOperations.TrailingZeroCount(shift);
 76 |                     destination = ref Copy(ref destination, ref vec, (uint)copySize);
 77 |                     source = ref Unsafe.AddByteOffset(ref source, copySize);
 78 |                     destination = ref Write(ref destination, ushort.MaxValue);
 79 |                     status = 4;
 80 |                     continue;
 81 |                 }
 82 | 
 83 |                 var asciiLength = BitOperations.TrailingZeroCount(~bits) >>> 1;
 84 |                 if (asciiLength + status < 4)
 85 |                 {
 86 |                     status = 0;
 87 |                     var copySize = (uint)(asciiLength + 1) << 1;
 88 |                     destination = ref Copy(ref destination, ref vec, copySize);
 89 |                     source = ref Unsafe.AddByteOffset(ref source, copySize);
 90 |                 }
 91 |                 else
 92 |                 {
 93 |                     destination = ref TransitFromUnicodeToAscii(ref destination, ref status);
 94 |                     var narrow = Vector256.Narrow(vec, vec).GetLower();
 95 |                     destination = ref Copy(ref destination, ref narrow, (uint)asciiLength);
 96 |                     destination = ref Write(ref destination, byte.MaxValue);
 97 |                     source = ref Unsafe.Add(ref source, asciiLength);
 98 |                     status = 0;
 99 |                 }
100 |             }
101 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<byte>>());
102 |         }
103 |         if (Vector128.IsHardwareAccelerated && Unsafe.ByteOffset(ref source, ref sourceEnd) >= Unsafe.SizeOf<Vector128<byte>>())
104 |         {
105 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<byte>>());
106 |             var filter = Vector128.Create((ushort)0xff80);
107 |             while (!Unsafe.IsAddressGreaterThan(ref source, ref sourceEnd))
108 |             {
109 |                 var vec = Vector128.LoadUnsafe(ref Unsafe.As<char, ushort>(ref source));
110 |                 // 00 → unicode, 11 → ascii
111 |                 var bits = Vector128.Equals(vec & filter, Vector128<ushort>.Zero).AsByte().ExtractMostSignificantBits();
112 |                 if (bits == 0U)
113 |                 {
114 |                     destination = ref TransitFromAsciiToUnicode(ref destination, ref status);
115 |                     destination = ref Write(ref destination, ref vec);
116 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<byte>>());
117 |                     continue;
118 |                 }
119 |                 if (bits == ushort.MaxValue)
120 |                 {
121 |                     destination = ref TransitFromUnicodeToAscii(ref destination, ref status);
122 |                     destination = ref Write(ref destination, Vector128.Narrow(vec, vec).AsUInt64().GetElement(0));
123 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<byte>>());
124 |                     continue;
125 |                 }
126 |                 if ((bits & 1U) == 0U)
127 |                 {
128 |                     destination = ref TransitFromAsciiToUnicode(ref destination, ref status);
129 |                     var shift = bits & (bits >>> 2) & (bits >>> 4) & (bits >>> 6);
130 |                     if (shift == 0U)
131 |                     {
132 |                         destination = ref Write(ref destination, ref vec);
133 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<byte>>());
134 |                         status = (uint)(BitOperations.LeadingZeroCount((ushort)~bits) >>> 1) - 8U;
135 |                         continue;
136 |                     }
137 | 
138 |                     var copySize = BitOperations.TrailingZeroCount(shift);
139 |                     destination = ref Copy(ref destination, ref vec, (uint)copySize);
140 |                     source = ref Unsafe.AddByteOffset(ref source, copySize);
141 |                     destination = ref Write(ref destination, ushort.MaxValue);
142 |                     status = 4;
143 |                     continue;
144 |                 }
145 | 
146 |                 var asciiLength = BitOperations.TrailingZeroCount(~bits) >>> 1;
147 |                 if (asciiLength + status < 4)
148 |                 {
149 |                     status = 0;
150 |                     var copySize = (uint)(asciiLength + 1) << 1;
151 |                     destination = ref Copy(ref destination, ref vec, copySize);
152 |                     source = ref Unsafe.AddByteOffset(ref source, copySize);
153 |                 }
154 |                 else
155 |                 {
156 |                     destination = ref TransitFromUnicodeToAscii(ref destination, ref status);
157 |                     var narrow = Vector128.Narrow(vec, vec);
158 |                     destination = ref Copy(ref destination, ref narrow, (uint)asciiLength);
159 |                     destination = ref Write(ref destination, byte.MaxValue);
160 |                     source = ref Unsafe.Add(ref source, asciiLength);
161 |                     status = 0;
162 |                 }
163 |             }
164 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<byte>>());
165 |         }
166 | 
167 |         while (!Unsafe.AreSame(ref source, ref sourceEnd))
168 |         {
169 |             var value = (ushort)source;
170 |             source = ref Unsafe.Add(ref source, 1);
171 |             if (value >= 0x80)
172 |             {
173 |                 destination = ref TransitFromAsciiToUnicode(ref destination, ref status);
174 |                 destination = ref Write(ref destination, value);
175 |                 continue;
176 |             }
177 | 
178 |             if (++status < 4)
179 |             {
180 |                 destination = ref Write(ref destination, value);
181 |                 continue;
182 |             }
183 | 
184 |             if (status == 4)
185 |             {
186 |                 destination = ref TransitFromUnicodeToAscii(ref destination);
187 |             }
188 |             
189 |             status = 4;
190 |             destination = ref Write(ref destination, (byte)value);
191 |         }
192 | 
193 |         return Unsafe.ByteOffset(ref initialDestination, ref destination);
194 |     }
195 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
196 |     private static ref byte Write(ref byte destination, byte value)
197 |     {
198 |         destination = value;
199 |         return ref Unsafe.AddByteOffset(ref destination, 1);
200 |     }
201 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
202 |     private static ref byte Write(ref byte destination, ushort value)
203 |     {
204 |         Unsafe.WriteUnaligned(ref destination, value);
205 |         return ref Unsafe.AddByteOffset(ref destination, 2);
206 |     }
207 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
208 |     private static ref byte Write(ref byte destination, ulong value)
209 |     {
210 |         Unsafe.WriteUnaligned(ref destination, value);
211 |         return ref Unsafe.AddByteOffset(ref destination, 8);
212 |     }
213 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
214 |     private static ref byte Write(ref byte destination, scoped ref Vector256<ushort> value)
215 |     {
216 |         value.AsByte().StoreUnsafe(ref destination);
217 |         return ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector256<ushort>>());
218 |     }
219 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
220 |     private static ref byte Write(ref byte destination, scoped ref Vector256<byte> value)
221 |     {
222 |         value.StoreUnsafe(ref destination);
223 |         return ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector256<ushort>>());
224 |     }
225 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
226 |     private static ref byte Write(ref byte destination, scoped ref Vector128<ushort> value)
227 |     {
228 |         value.AsByte().StoreUnsafe(ref destination);
229 |         return ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector128<ushort>>());
230 |     }
231 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
232 |     private static ref byte Write(ref byte destination, scoped ref Vector128<byte> value)
233 |     {
234 |         value.StoreUnsafe(ref destination);
235 |         return ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector128<ushort>>());
236 |     }
237 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
238 |     private static ref byte Copy<T>(ref byte destination, scoped ref T value, uint size) where T : unmanaged
239 |     {
240 |         Unsafe.CopyBlockUnaligned(ref destination, ref Unsafe.As<T, byte>(ref value), size);
241 |         return ref Unsafe.AddByteOffset(ref destination, size);
242 |     }
243 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
244 |     private static ref byte TransitFromUnicodeToAscii(ref byte destination)
245 |     {
246 |         // ST0L, ST0U, ST1L, ST1U, ST2L, ST2U, destination
247 |         // 0xff, 0xff, ST0L, ST1L, ST2L, destination
248 |         ref var sub4 = ref Unsafe.SubtractByteOffset(ref destination, 4);
249 |         ref var sub6 = ref Unsafe.SubtractByteOffset(ref destination, 6);
250 |         Unsafe.WriteUnaligned(ref sub4, (ushort)((sub4 << 8) | sub6));
251 |         Unsafe.WriteUnaligned(ref sub6, ushort.MaxValue);
252 |         return ref Unsafe.SubtractByteOffset(ref destination, 1);
253 |     }
254 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
255 |     private static ref byte TransitFromUnicodeToAscii(ref byte destination, scoped ref uint status)
256 |     {
257 |         var temp = status;
258 |         status = 4;
259 |         switch (temp)
260 |         {
261 |             case 0: return ref Write(ref destination, ushort.MaxValue);
262 |             case 1:
263 |             {
264 |                 ref var sub2 = ref Unsafe.SubtractByteOffset(ref destination, 2);
265 |                 var value = sub2;
266 |                 Unsafe.WriteUnaligned(ref sub2, ushort.MaxValue);
267 |                 return ref Write(ref destination, value);
268 |             }
269 |             case 2:
270 |             {
271 |                 ref var sub2 = ref Unsafe.SubtractByteOffset(ref destination, 2);
272 |                 ref var sub4 = ref Unsafe.SubtractByteOffset(ref destination, 4);
273 |                 Unsafe.WriteUnaligned(ref sub2, (ushort)((sub2 << 8) | sub4));
274 |                 Unsafe.WriteUnaligned(ref sub4, ushort.MaxValue);
275 |                 break;
276 |             }
277 |             case 3: return ref TransitFromUnicodeToAscii(ref destination);
278 |         }
279 |         return ref destination;
280 |     }
281 |     [MethodImpl(MethodImplOptions.AggressiveInlining)]
282 |     private static ref byte TransitFromAsciiToUnicode(ref byte destination, ref uint status)
283 |     {
284 |         var temp = status;
285 |         status = 0;
286 |         return ref (temp >= 4 ? ref Write(ref destination, byte.MaxValue) : ref destination);
287 |     }
288 | }
289 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/GetCharCount.cs:
--------------------------------------------------------------------------------
  1 | namespace Utf16StringFastCompression;
  2 | 
  3 | partial class Utf16CompressionEncoding
  4 | {
  5 |     public static int GetCharCount(ReadOnlySpan<byte> source) => GetCharCount(ref MemoryMarshal.GetReference(source), source.Length).ToInt32();
  6 | 
  7 |     public static int GetCharCount(byte[] source, int sourceLength) => GetCharCount(ref MemoryMarshal.GetArrayDataReference(source), sourceLength).ToInt32();
  8 | 
  9 |     public static nint GetCharCount(scoped ref byte source, nint sourceLength)
 10 |     {
 11 |         var state = new ToCharState();
 12 |         return GetCharCountStateful(ref source, sourceLength, ref state);
 13 |     }
 14 | 
 15 |     public static long GetCharCount(in ReadOnlySequence<byte> source)
 16 |     {
 17 |         var state = new ToCharState();
 18 |         if (source.IsSingleSegment)
 19 |         {
 20 |             var span = source.FirstSpan;
 21 |             return GetCharCountStateful(ref MemoryMarshal.GetReference(span), span.Length, ref state);
 22 |         }
 23 |         long answer = default;
 24 |         var enumerator = source.GetEnumerator();
 25 |         while (enumerator.MoveNext())
 26 |         {
 27 |             var span = enumerator.Current.Span;
 28 |             answer += GetCharCountStateful(ref MemoryMarshal.GetReference(span), span.Length, ref state);
 29 |         }
 30 |         return answer;
 31 |     }
 32 | 
 33 |     public static nint GetCharCountStateful(scoped ref byte source, nint sourceLength, scoped ref ToCharState state)
 34 |     {
 35 |         if (Unsafe.IsNullRef(ref source) || sourceLength <= 0)
 36 |         {
 37 |             return 0;
 38 |         }
 39 | 
 40 |         nint answer = 0;
 41 |         scoped ref var sourceEnd = ref Unsafe.AddByteOffset(ref source, sourceLength);
 42 |         ushort value;
 43 |         var isAscii = state.IsAsciiMode;
 44 |         if (state.HasRemainingByte)
 45 |         {
 46 |             state.HasRemainingByte = false;
 47 |             value = (ushort)((source << 8) | state.RemainingByte);
 48 |             if (value == ushort.MaxValue)
 49 |             {
 50 |                 isAscii = true;
 51 |             }
 52 |             else
 53 |             {
 54 |                 answer += 1;
 55 |             }
 56 |             source = ref Unsafe.AddByteOffset(ref source, 1);
 57 |         }
 58 | 
 59 |         if (Vector256.IsHardwareAccelerated && sourceLength >= Unsafe.SizeOf<Vector256<ushort>>())
 60 |         {
 61 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
 62 |             var filter = Vector256<byte>.AllBitsSet;
 63 |             while (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
 64 |             {
 65 |                 var vector = Vector256.LoadUnsafe(ref source);
 66 |                 if (isAscii)
 67 |                 {
 68 |                     if (Vector256.EqualsAny(vector, filter))
 69 |                     {
 70 |                         isAscii = false;
 71 |                         var tzc = BitOperations.TrailingZeroCount(Vector256.Equals(vector, filter).ExtractMostSignificantBits());
 72 |                         answer += tzc;
 73 |                         source = ref Unsafe.AddByteOffset(ref source, tzc + 1);
 74 |                         continue;
 75 |                     }
 76 | 
 77 |                     answer += Unsafe.SizeOf<Vector256<ushort>>();
 78 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
 79 |                 }
 80 |                 else
 81 |                 {
 82 |                     if (Vector256.EqualsAny(vector.AsUInt16(), filter.AsUInt16()))
 83 |                     {
 84 |                         isAscii = true;
 85 |                         var tzc = BitOperations.TrailingZeroCount(Vector256.Equals(vector.AsUInt16(), filter.AsUInt16()).AsByte().ExtractMostSignificantBits());
 86 |                         answer += tzc >>> 1;
 87 |                         source = ref Unsafe.AddByteOffset(ref source, tzc + 2);
 88 |                         continue;
 89 |                     }
 90 | 
 91 |                     answer += Unsafe.SizeOf<Vector256<ushort>>() >> 1;
 92 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
 93 |                 }
 94 |             }
 95 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
 96 |         }
 97 | 
 98 |         if (Vector128.IsHardwareAccelerated && sourceLength >= Unsafe.SizeOf<Vector128<ushort>>())
 99 |         {
100 |             sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
101 |             var filter = Vector128<byte>.AllBitsSet;
102 |             while (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
103 |             {
104 |                 var vector = Vector128.LoadUnsafe(ref source);
105 |                 if (isAscii)
106 |                 {
107 |                     if (Vector128.EqualsAny(vector, filter))
108 |                     {
109 |                         isAscii = false;
110 |                         var tzc = BitOperations.TrailingZeroCount(Vector128.Equals(vector, filter).ExtractMostSignificantBits());
111 |                         answer += tzc;
112 |                         source = ref Unsafe.AddByteOffset(ref source, tzc + 1);
113 |                         continue;
114 |                     }
115 | 
116 |                     answer += Unsafe.SizeOf<Vector128<ushort>>();
117 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
118 |                 }
119 |                 else
120 |                 {
121 |                     if (Vector128.EqualsAny(vector.AsUInt16(), filter.AsUInt16()))
122 |                     {
123 |                         isAscii = true;
124 |                         var tzc = BitOperations.TrailingZeroCount(Vector128.Equals(vector.AsUInt16(), filter.AsUInt16()).AsByte().ExtractMostSignificantBits());
125 |                         answer += tzc >>> 1;
126 |                         source = ref Unsafe.AddByteOffset(ref source, tzc + 2);
127 |                         continue;
128 |                     }
129 | 
130 |                     answer += Unsafe.SizeOf<Vector128<ushort>>() >> 1;
131 |                     source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
132 |                 }
133 |             }
134 |             sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
135 |         }
136 | 
137 |         if (!Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
138 |         {
139 |             goto RETURN;
140 |         }
141 |         if (isAscii)
142 |         {
143 |             goto ASCII_LOOP;
144 |         }
145 |         if (Unsafe.ByteOffset(ref source, ref sourceEnd) == 1)
146 |         {
147 |             goto REMAINDER;
148 |         }
149 | 
150 |     UNICODE_LOOP:
151 |         value = Unsafe.ReadUnaligned<ushort>(ref source);
152 |         source = ref Unsafe.AddByteOffset(ref source, 2);
153 |         if (value == ushort.MaxValue)
154 |         {
155 |             isAscii = true;
156 |             if (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
157 |             {
158 |                 goto ASCII_LOOP;
159 |             }
160 |             goto RETURN;
161 |         }
162 | 
163 |         answer += 1;
164 |         switch (Unsafe.ByteOffset(ref source, ref sourceEnd))
165 |         {
166 |             case 0: goto RETURN;
167 |             case 1: goto REMAINDER;
168 |             default: goto UNICODE_LOOP;
169 |         }
170 |         
171 |     ASCII_LOOP:
172 |         value = source;
173 |         source = ref Unsafe.AddByteOffset(ref source, 1);
174 |         if (value == byte.MaxValue)
175 |         {
176 |             isAscii = false;
177 |             switch (Unsafe.ByteOffset(ref source, ref sourceEnd))
178 |             {
179 |                 case 0: goto RETURN;
180 |                 case 1: goto REMAINDER;
181 |                 default: goto UNICODE_LOOP;
182 |             }
183 |         }
184 | 
185 |         answer += 1;
186 |         if (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
187 |         {
188 |             goto ASCII_LOOP;
189 |         }
190 |         goto RETURN;
191 | 
192 |     REMAINDER:
193 |         state.HasRemainingByte = true;
194 |         state.RemainingByte = source;
195 | 
196 |     RETURN:
197 |         state.IsAsciiMode = isAscii;
198 |         return answer;
199 |     }
200 | }
201 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/GetChars.cs:
--------------------------------------------------------------------------------
  1 | using System.Globalization;
  2 | 
  3 | namespace Utf16StringFastCompression;
  4 | 
  5 | partial class Utf16CompressionEncoding
  6 | {
  7 |     public static string GetString(ReadOnlySpan<byte> source) => GetString(ref MemoryMarshal.GetReference(source), source.Length);
  8 | 
  9 |     public static unsafe string GetString(ref byte source, nint sourceLength)
 10 |     {
 11 |         if (Unsafe.IsNullRef(ref source) || sourceLength <= 0)
 12 |         {
 13 |             return string.Empty;
 14 |         }
 15 |         var length = GetCharCount(ref source, sourceLength);
 16 |         return string.Create((int)length, ((IntPtr)Unsafe.AsPointer(ref source), sourceLength), CreateString);
 17 |     }
 18 | 
 19 |     private static unsafe void CreateString(Span<char> span, ValueTuple<IntPtr, IntPtr> arg)
 20 |     {
 21 |         GetChars(ref *((byte*)(void*)arg.Item1), arg.Item2, ref MemoryMarshal.GetReference(span));
 22 |     }
 23 | 
 24 |     /// <summary>
 25 |     /// Deserialize to span
 26 |     /// </summary>
 27 |     /// <param name="source">Source byte span</param>
 28 |     /// <param name="destination">Destination char span</param>
 29 |     /// <returns>The number of decoded chars.</returns>
 30 |     public static int GetChars(ReadOnlySpan<byte> source, Span<char> destination) => GetChars(ref MemoryMarshal.GetReference(source), source.Length, ref MemoryMarshal.GetReference(destination)).ToInt32();
 31 | 
 32 |     /// <summary>
 33 |     /// Deserialize to array
 34 |     /// </summary>
 35 |     /// <param name="source">Source byte array</param>
 36 |     /// <param name="sourceLength">Source byte length</param>
 37 |     /// <param name="destination">Destination char array</param>
 38 |     /// <returns>The number of decoded chars.</returns>
 39 |     public static int GetChars(byte[] source, int sourceLength, char[] destination) => GetChars(ref MemoryMarshal.GetArrayDataReference(source), sourceLength, ref MemoryMarshal.GetArrayDataReference(destination)).ToInt32();
 40 | 
 41 |     /// <summary>
 42 |     /// Deserialize to reference
 43 |     /// </summary>
 44 |     /// <param name="source">Source byte reference</param>
 45 |     /// <param name="sourceLength">Source byte length</param>
 46 |     /// <param name="destination">Destination char reference</param>
 47 |     /// <returns>The number of decoded chars.</returns>
 48 |     public static nint GetChars(scoped ref byte source, nint sourceLength, scoped ref char destination)
 49 |     {
 50 |         var state = new ToCharState();
 51 |         return GetCharsStateful(ref source, sourceLength, ref destination, ref state);
 52 |     }
 53 | 
 54 |     public static long GetChars(in ReadOnlySequence<byte> source, scoped ref char destination)
 55 |     {
 56 |         var state = new ToCharState();
 57 |         if (source.IsSingleSegment)
 58 |         {
 59 |             var span = source.FirstSpan;
 60 |             return GetCharsStateful(ref MemoryMarshal.GetReference(span), span.Length, ref destination, ref state);
 61 |         }
 62 |         long answer = default;
 63 |         var enumerator = source.GetEnumerator();
 64 |         while (enumerator.MoveNext())
 65 |         {
 66 |             var span = enumerator.Current.Span;
 67 |             var count = GetCharsStateful(ref MemoryMarshal.GetReference(span), span.Length, ref destination, ref state);
 68 |             destination = ref Unsafe.Add(ref destination, count);
 69 |             answer += count;
 70 |         }
 71 |         return answer;
 72 |     }
 73 | 
 74 |     public static nint GetCharsStateful(scoped ref byte source, nint sourceLength, scoped ref char destination, scoped ref ToCharState state)
 75 |     {
 76 |         if (Unsafe.IsNullRef(ref source) || sourceLength <= 0 || Unsafe.IsNullRef(ref destination))
 77 |         {
 78 |             return 0;
 79 |         }
 80 | 
 81 |         unchecked
 82 |         {
 83 |             scoped ref var initialDestination = ref destination;
 84 |             scoped ref var sourceEnd = ref Unsafe.AddByteOffset(ref source, sourceLength);
 85 |             ushort value;
 86 |             var isAscii = state.IsAsciiMode;
 87 |             if (state.HasRemainingByte)
 88 |             {
 89 |                 state.HasRemainingByte = false;
 90 |                 value = (ushort)((source << 8) | state.RemainingByte);
 91 |                 if (value == ushort.MaxValue)
 92 |                 {
 93 |                     isAscii = true;
 94 |                 }
 95 |                 else
 96 |                 {
 97 |                     destination = (char)value;
 98 |                     destination = ref Unsafe.Add(ref destination, 1);
 99 |                 }
100 |                 source = ref Unsafe.AddByteOffset(ref source, 1);
101 |             }
102 |             if (Vector256.IsHardwareAccelerated && sourceLength >= Unsafe.SizeOf<Vector256<ushort>>())
103 |             {
104 |                 sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
105 |                 var filter = Vector256<byte>.AllBitsSet;
106 |                 while (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
107 |                 {
108 |                     var vector = Vector256.LoadUnsafe(ref source);
109 |                     if (isAscii)
110 |                     {
111 |                         if (Vector256.EqualsAny(vector, filter))
112 |                         {
113 |                             isAscii = false;
114 |                             var tzc = BitOperations.TrailingZeroCount(Vector256.Equals(vector, filter).ExtractMostSignificantBits());
115 |                             for (; --tzc >= 0; source = ref Unsafe.AddByteOffset(ref source, 1), destination = ref Unsafe.Add(ref destination, 1))
116 |                             {
117 |                                 destination = (char)source;
118 |                             }
119 | 
120 |                             source = ref Unsafe.AddByteOffset(ref source, 1);
121 |                             continue;
122 |                         }
123 | 
124 |                         var (v0, v1) = Vector256.Widen(vector);
125 |                         v0.StoreUnsafe(ref Unsafe.As<char, ushort>(ref destination));
126 |                         v1.StoreUnsafe(ref Unsafe.As<char, ushort>(ref destination), (uint)Unsafe.SizeOf<Vector256<ushort>>() >>> 1);
127 |                         destination = ref Unsafe.Add(ref destination, Unsafe.SizeOf<Vector256<ushort>>());
128 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
129 |                     }
130 |                     else
131 |                     {
132 |                         if (Vector256.EqualsAny(vector.AsUInt16(), filter.AsUInt16()))
133 |                         {
134 |                             isAscii = true;
135 |                             var tzc = BitOperations.TrailingZeroCount(Vector256.Equals(vector.AsUInt16(), filter.AsUInt16()).AsByte().ExtractMostSignificantBits());
136 |                             if (tzc != 0)
137 |                             {
138 |                                 Unsafe.CopyBlockUnaligned(ref Unsafe.As<char, byte>(ref destination), ref source, (uint)tzc);
139 |                             }
140 | 
141 |                             destination = ref Unsafe.AddByteOffset(ref destination, tzc);
142 |                             source = ref Unsafe.AddByteOffset(ref source, tzc + 2);
143 |                             continue;
144 |                         }
145 | 
146 |                         vector.StoreUnsafe(ref Unsafe.As<char, byte>(ref destination));
147 |                         destination = ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector256<ushort>>());
148 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector256<ushort>>());
149 |                     }
150 |                 }
151 |                 sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector256<ushort>>());
152 |             }
153 | 
154 |             if (Vector128.IsHardwareAccelerated && sourceLength >= Unsafe.SizeOf<Vector128<ushort>>())
155 |             {
156 |                 sourceEnd = ref Unsafe.SubtractByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
157 |                 var filter = Vector128<byte>.AllBitsSet;
158 |                 while (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
159 |                 {
160 |                     var vector = Vector128.LoadUnsafe(ref source);
161 |                     if (isAscii)
162 |                     {
163 |                         if (Vector128.EqualsAny(vector, filter))
164 |                         {
165 |                             isAscii = false;
166 |                             var tzc = BitOperations.TrailingZeroCount(Vector128.Equals(vector, filter).ExtractMostSignificantBits());
167 |                             for (; --tzc >= 0; source = ref Unsafe.AddByteOffset(ref source, 1), destination = ref Unsafe.Add(ref destination, 1))
168 |                             {
169 |                                 destination = (char)source;
170 |                             }
171 | 
172 |                             source = ref Unsafe.AddByteOffset(ref source, 1);
173 |                             continue;
174 |                         }
175 | 
176 |                         var (v0, v1) = Vector128.Widen(vector);
177 |                         v0.StoreUnsafe(ref Unsafe.As<char, ushort>(ref destination));
178 |                         v1.StoreUnsafe(ref Unsafe.As<char, ushort>(ref destination), (uint)Unsafe.SizeOf<Vector128<ushort>>() >>> 1);
179 |                         destination = ref Unsafe.Add(ref destination, Unsafe.SizeOf<Vector128<ushort>>());
180 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
181 |                     }
182 |                     else
183 |                     {
184 |                         if (Vector128.EqualsAny(vector.AsUInt16(), filter.AsUInt16()))
185 |                         {
186 |                             isAscii = true;
187 |                             var tzc = BitOperations.TrailingZeroCount(Vector128.Equals(vector.AsUInt16(), filter.AsUInt16()).AsByte().ExtractMostSignificantBits());
188 |                             if (tzc != 0)
189 |                             {
190 |                                 Unsafe.CopyBlockUnaligned(ref Unsafe.As<char, byte>(ref destination), ref source, (uint)tzc);
191 |                             }
192 | 
193 |                             destination = ref Unsafe.AddByteOffset(ref destination, tzc);
194 |                             source = ref Unsafe.AddByteOffset(ref source, tzc + 2);
195 |                             continue;
196 |                         }
197 | 
198 |                         vector.StoreUnsafe(ref Unsafe.As<char, byte>(ref destination));
199 |                         destination = ref Unsafe.AddByteOffset(ref destination, Unsafe.SizeOf<Vector128<ushort>>());
200 |                         source = ref Unsafe.AddByteOffset(ref source, Unsafe.SizeOf<Vector128<ushort>>());
201 |                     }
202 |                 }
203 |                 sourceEnd = ref Unsafe.AddByteOffset(ref sourceEnd, Unsafe.SizeOf<Vector128<ushort>>());
204 |             }
205 | 
206 |             if (!Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
207 |             {
208 |                 goto RETURN;
209 |             }
210 |             if (isAscii)
211 |             {
212 |                 goto ASCII_LOOP;
213 |             }
214 |             if (Unsafe.ByteOffset(ref source, ref sourceEnd) == 1)
215 |             {
216 |                 goto REMAINDER;
217 |             }
218 | 
219 |         UNICODE_LOOP:
220 |             value = Unsafe.ReadUnaligned<ushort>(ref source);
221 |             source = ref Unsafe.AddByteOffset(ref source, 2);
222 |             if (value == ushort.MaxValue)
223 |             {
224 |                 isAscii = true;
225 |                 if (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
226 |                 {
227 |                     goto ASCII_LOOP;
228 |                 }
229 |                 goto RETURN;
230 |             }
231 | 
232 |             destination = (char)value;
233 |             destination = ref Unsafe.Add(ref destination, 1);
234 |             switch (Unsafe.ByteOffset(ref source, ref sourceEnd))
235 |             {
236 |                 case 0: goto RETURN;
237 |                 case 1: goto REMAINDER;
238 |                 default: goto UNICODE_LOOP;
239 |             }
240 |             
241 |         ASCII_LOOP:
242 |             value = source;
243 |             source = ref Unsafe.AddByteOffset(ref source, 1);
244 |             if (value == byte.MaxValue)
245 |             {
246 |                 isAscii = false;
247 |                 switch (Unsafe.ByteOffset(ref source, ref sourceEnd))
248 |                 {
249 |                     case 0: goto RETURN;
250 |                     case 1: goto REMAINDER;
251 |                     default: goto UNICODE_LOOP;
252 |                 }
253 |             }
254 | 
255 |             destination = (char)value;
256 |             destination = ref Unsafe.Add(ref destination, 1);
257 |             if (Unsafe.IsAddressLessThan(ref source, ref sourceEnd))
258 |             {
259 |                 goto ASCII_LOOP;
260 |             }
261 |             goto RETURN;
262 | 
263 |         REMAINDER:
264 |             state.HasRemainingByte = true;
265 |             state.RemainingByte = source;
266 | 
267 |         RETURN:
268 |             state.IsAsciiMode = isAscii;
269 |             return Unsafe.ByteOffset(ref initialDestination, ref destination) >>> 1;
270 |         }
271 |     }
272 | }
273 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/ToCharState.cs:
--------------------------------------------------------------------------------
1 | namespace Utf16StringFastCompression;
2 | 
3 | public struct ToCharState
4 | {
5 |     public bool IsAsciiMode;
6 |     public bool HasRemainingByte;
7 |     public byte RemainingByte;
8 | }
9 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/Utf16CompressionEncoding.cs:
--------------------------------------------------------------------------------
1 | namespace Utf16StringFastCompression;
2 | 
3 | public static partial class Utf16CompressionEncoding
4 | {
5 |     public static T GetMaxByteCount<T>(T charCount) where T : IShiftOperators<T, int, T> => charCount << 1;
6 | 
7 |     public static T GetMaxCharCount<T>(T byteCount) => byteCount;
8 | }
9 | 


--------------------------------------------------------------------------------
/src/Utf16StringFastCompression/Utf16StringFastCompression.csproj:
--------------------------------------------------------------------------------
 1 | <Project Sdk="Microsoft.NET.Sdk">
 2 | 
 3 | 	<PropertyGroup>
 4 | 		<TargetFrameworks>net7.0</TargetFrameworks>
 5 | 		<Nullable>enable</Nullable>
 6 | 		<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
 7 | 		<LangVersion>11</LangVersion>
 8 | 		<Version>0.0.8</Version>
 9 | 		<Authors>pCYSl5EDgo</Authors>
10 | 		<PackageId>Utf16StringFastCompression</PackageId>
11 | 		<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
12 | 		<Description>Very Fast Utf16 String Encoder</Description>
13 | 		<Copyright>pCYSl5EDgo</Copyright>
14 | 		<PackageProjectUrl>https://github.com/pCYSl5EDgo/Utf16StringFastCompression</PackageProjectUrl>
15 | 		<PackageLicenseExpression>MIT</PackageLicenseExpression>
16 | 		<WarningLevel>9999</WarningLevel>
17 | 		<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
18 | 	</PropertyGroup>
19 | 
20 | 	<ItemGroup>
21 | 		<Using Include="System" />
22 | 		<Using Include="System.Buffers" />
23 | 		<Using Include="System.ComponentModel" />
24 | 		<Using Include="System.Numerics" />
25 | 		<Using Include="System.Runtime.CompilerServices" />
26 | 		<Using Include="System.Runtime.InteropServices" />
27 | 		<Using Include="System.Runtime.Intrinsics" />
28 | 		<Using Include="System.Runtime.Intrinsics.X86" />
29 | 	</ItemGroup>
30 | 
31 | </Project>


--------------------------------------------------------------------------------