├── .gitignore ├── LICENSE.txt ├── README.md └── Shaman.BlobStore ├── BlobPackage.cs ├── BlobStore.cs ├── BlobStream.cs ├── ExtensionMethods.cs ├── FileLocation.cs ├── PackageDirectory.cs ├── Shaman.BlobStore.csproj ├── ShamanOpenSourceKey.snk └── UnseekableStreamWrapper.cs /.gitignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | bin 3 | obj 4 | *.user 5 | *.xproj 6 | project.lock.json 7 | Shaman.BlobStore/Program.cs 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Andrea Martinelli 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shaman.BlobStore 2 | Provides an almost drop-in replacement for `System.IO.File`, optimized for a very large number of small files. 3 | Data is batched in package files, and can be read sequentially. 4 | 5 | ## Usage 6 | 7 | ```csharp 8 | using Shaman.Runtime; 9 | 10 | // Physical structure on file system: 11 | // C:\\Example\\.shaman-blob-index <- index, automatically regenerated if missing 12 | // C:\\Example\\044cf7e7-d3a8-4499-b04d-0f5bab478e64.shaman-blobs <- package #1 13 | // C:\\Example\\37adb1a9-d365-48ab-bfb1-ff6c0fc7b552.shaman-blobs <- package #2 14 | // C:\\Example\\fc08265d-1a1c-4fc9-86b8-86475486491a.shaman-blobs <- package #3 15 | 16 | foreach (Blob b in BlobStore.EnumerateFiles("C:\\Example")) 17 | { 18 | // reading thousands or millions of small blobs, stored sequentially in .shaman-blobs files 19 | using (Stream s = b.OpenRead()) 20 | { 21 | // Process blob 22 | } 23 | } 24 | 25 | // Not actual files on the file system, they're located in one of the packages (*.shaman-blobs) in C:\Example 26 | BlobStore.Delete("C:\\Example\\MyFile1.txt"); 27 | BlobStore.WriteAllText("C:\\Example\\MyFile2.txt", "content", Encoding.UTF8); 28 | BlobStore.FlushDirectory("C:\\Example"); 29 | 30 | // Each package is flushed and saved as a file once it grows above 1.5 MB (by default) 31 | // You can change the batch size (in KB) with: 32 | BlobStore.SetConfigurationForDirectory(@"C:\\Example\\", 10 * 1024); 33 | ``` -------------------------------------------------------------------------------- /Shaman.BlobStore/BlobPackage.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Shaman.Runtime 8 | { 9 | internal class BlobPackage 10 | { 11 | internal PackageDirectory directory; 12 | internal string fileName; 13 | internal volatile MemoryStream ms; 14 | internal long lastCommittedLength; 15 | internal long startOfBlobHeader; 16 | internal long startOfBlobData; 17 | 18 | 19 | internal void Release(bool commit = true) 20 | { 21 | lock (BlobStore._lock) 22 | { 23 | BlobStore.Delete(Path.Combine(directory.path, this.currentFileName)); 24 | if (commit) Commit(true); 25 | directory.busyPackages.Remove(this); 26 | 27 | if (ms.Length < (directory.packageSize ?? BlobStore.Configuration_PackageSize)) 28 | { 29 | directory.availablePackages.Add(this); 30 | } 31 | else 32 | { 33 | WriteToFile(false); 34 | this.ms = null; 35 | } 36 | } 37 | } 38 | 39 | internal BlobStream OpenWrite(bool autocommit) 40 | { 41 | return new BlobStream(this, autocommit); 42 | } 43 | 44 | public MemoryStream CopyMemoryStream() 45 | { 46 | return new MemoryStream(ms.GetBuffer(), 0, (int)ms.Length, false, true); 47 | 48 | } 49 | 50 | 51 | private string currentFileName; 52 | private DateTime? currentFileTime; 53 | 54 | internal MemoryStream EnsureAdditionalCapacity(int capacity) 55 | { 56 | return EnsureCapacity((int)ms.Length + capacity); 57 | } 58 | 59 | internal MemoryStream EnsureCapacity(int capacity) 60 | { 61 | if (capacity > ms.Capacity) 62 | { 63 | lock (BlobStore._lock) 64 | { 65 | var arr = ms.GetBuffer(); 66 | var c = ms.Capacity; 67 | while (c < capacity) 68 | { 69 | c *= 2; 70 | } 71 | var newarr = new byte[c]; 72 | var pos = (int)ms.Position; 73 | var len = (int)ms.Length; 74 | Buffer.BlockCopy(arr, 0, newarr, 0, len); 75 | ms = new MemoryStream(newarr, 0, newarr.Length, true, true); 76 | ms.Seek(pos, SeekOrigin.Begin); 77 | ms.SetLength(len); 78 | } 79 | } 80 | return ms; 81 | } 82 | 83 | // must hold lock 84 | internal void WriteHeader(string name, DateTime? date) 85 | { 86 | if (ms == null) 87 | { 88 | var arr = new byte[directory.memoryStreamCapacity ?? BlobStore.Configuration_MemoryStreamCapacity]; 89 | ms = new MemoryStream(arr, 0, arr.Length, true, true); 90 | } 91 | lastFileCommitted = false; 92 | startOfBlobHeader = lastCommittedLength; 93 | ms.Seek(startOfBlobHeader, SeekOrigin.Begin); 94 | ms.SetLength(startOfBlobHeader); 95 | 96 | 97 | var nameBytes = Encoding.UTF8.GetBytes(name); 98 | EnsureAdditionalCapacity(nameBytes.Length + 60); 99 | ms.WriteByte(0); 100 | ms.WriteByte(0); 101 | ms.WriteByte(0); 102 | ms.WriteByte(0); 103 | currentFileName = name; 104 | currentFileTime = date; 105 | var nameLength = nameBytes.Length; 106 | if (date != null) nameLength += 8 + 1; 107 | ms.WriteByte((byte)(nameLength >> 0)); 108 | ms.WriteByte((byte)(nameLength >> 8)); 109 | ms.WriteByte(0); 110 | ms.WriteByte(0); 111 | ms.Write(nameBytes, 0, nameBytes.Length); 112 | if (date != null) 113 | { 114 | var dateBytes = BitConverter.GetBytes(date.Value.Ticks); 115 | ms.Write(dateBytes, 0, dateBytes.Length); 116 | ms.WriteByte(0); 117 | } 118 | startOfBlobData = ms.Length; 119 | } 120 | 121 | bool lastFileCommitted; 122 | 123 | // must hold lock 124 | internal long Commit(bool perfect) 125 | { 126 | if (lastFileCommitted) return ms.Length; 127 | 128 | ms.Seek(startOfBlobHeader, SeekOrigin.Begin); 129 | var length = (int)ms.Length; 130 | var blobLength = length - startOfBlobData; 131 | ms.WriteByte((byte)(blobLength >> 0)); 132 | ms.WriteByte((byte)(blobLength >> 8)); 133 | ms.WriteByte((byte)(blobLength >> 16)); 134 | ms.WriteByte((byte)(blobLength >> 24)); 135 | ms.Seek(2, SeekOrigin.Current); 136 | ms.WriteByte(perfect ? (byte)1 : (byte)0); 137 | ms.Seek(length, SeekOrigin.Begin); 138 | if (perfect) 139 | { 140 | lastCommittedLength = length; 141 | var location = new FileLocation() 142 | { 143 | BlobName = currentFileName, 144 | PackageFileName = this.fileName, 145 | DataStartOffset = (int)startOfBlobData, 146 | Package = this, 147 | Length = length - (int)startOfBlobData, 148 | Date = currentFileTime 149 | }; 150 | directory.locations[currentFileName] = location; 151 | directory.locationsToAdd.Add(location); 152 | } 153 | lastFileCommitted = true; 154 | return length; 155 | } 156 | internal long bytesWrittenToDisk; 157 | 158 | 159 | 160 | // must hold lock 161 | internal void WriteToFile(bool close) 162 | { 163 | if (close) 164 | { 165 | if (!lastFileCommitted) 166 | { 167 | ms.SetLength(startOfBlobHeader); 168 | lastFileCommitted = true; 169 | } 170 | } 171 | 172 | var len = Commit(false); 173 | if (len != bytesWrittenToDisk) 174 | { 175 | using (var fs = File.Open(Path.Combine(directory.path, fileName), FileMode.OpenOrCreate, FileAccess.Write, FileShare.Delete)) 176 | { 177 | fs.Seek(bytesWrittenToDisk, SeekOrigin.Begin); 178 | ms.Seek(bytesWrittenToDisk, SeekOrigin.Begin); 179 | byte[] array = new byte[81920]; 180 | int count; 181 | var remaining = len - bytesWrittenToDisk; 182 | while (remaining != 0 && (count = ms.Read(array, 0, (int)Math.Min(array.Length, remaining))) != 0) 183 | { 184 | fs.Write(array, 0, count); 185 | } 186 | } 187 | } 188 | bytesWrittenToDisk = len; 189 | if (close) 190 | { 191 | ms.Dispose(); 192 | } 193 | else 194 | { 195 | ms.Seek(len, SeekOrigin.Begin); 196 | } 197 | } 198 | 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /Shaman.BlobStore/BlobStore.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Diagnostics; 4 | using System.IO; 5 | using System.Linq; 6 | using System.Text; 7 | 8 | namespace Shaman.Runtime 9 | { 10 | public static class BlobStore 11 | { 12 | public static void WriteAllBytes(string path, byte[] data) 13 | { 14 | using (var stream = OpenWriteNoAutoCommit(path)) 15 | { 16 | stream.Write(data, 0, data.Length); 17 | stream.Commit(); 18 | } 19 | Debug.Assert(Exists(path)); 20 | } 21 | 22 | private static void IndexDirectory(PackageDirectory dir, string directory) 23 | { 24 | dir.locations = new Dictionary(); 25 | var indexFile = Path.Combine(directory, ".shaman-blob-index"); 26 | using (var stream = File.Open(indexFile + "$", FileMode.Create, FileAccess.Write, FileShare.Delete)) 27 | using (var bw = new BinaryWriter(stream, Encoding.UTF8)) 28 | { 29 | string prevPackage = null; 30 | foreach (var item in EnumerateFiles(directory, includeDeleted: false, orderByDate: false, singlePackage: null, allowIndex: false)) 31 | { 32 | if (prevPackage != item.PackageFileName) 33 | { 34 | prevPackage = item.PackageFileName; 35 | bw.Write((byte)1); 36 | bw.Write(item.PackageFileName ?? string.Empty); 37 | } 38 | else 39 | { 40 | bw.Write((byte)0); 41 | } 42 | 43 | 44 | bw.Write((byte)0); 45 | 46 | var length = (uint)item.Length.Value; 47 | if (item.LastWriteTime != null) length |= (uint)1 << 31; 48 | 49 | bw.Write(length); 50 | 51 | if (item.LastWriteTime != null) 52 | { 53 | bw.Write(item.LastWriteTime.Value.Ticks); 54 | } 55 | 56 | bw.Write(item.Name); 57 | bw.Write(item.DataStartOffset); 58 | 59 | dir.locations[item.Name] = new FileLocation() 60 | { 61 | BlobName = item.Name, 62 | DataStartOffset = item.DataStartOffset, 63 | PackageFileName = item.PackageFileName, 64 | Date = item.LastWriteTime, 65 | Length = item.Length.Value 66 | }; 67 | } 68 | } 69 | File.Delete(indexFile); 70 | File.Move(indexFile + "$", indexFile); 71 | 72 | } 73 | 74 | 75 | internal static void Write7BitEncodedInt(BinaryWriter bw, int value) 76 | { 77 | // Write out an int 7 bits at a time. The high bit of the byte, 78 | // when on, tells reader to continue reading more bytes. 79 | uint v = (uint)value; // support negative numbers 80 | while (v >= 0x80) 81 | { 82 | bw.Write((byte)(v | 0x80)); 83 | v >>= 7; 84 | } 85 | bw.Write((byte)v); 86 | } 87 | 88 | 89 | internal static int Read7BitEncodedInt(BinaryReader br) 90 | { 91 | // Read out an Int32 7 bits at a time. The high bit 92 | // of the byte when on means to continue reading more bytes. 93 | int count = 0; 94 | int shift = 0; 95 | byte b; 96 | do 97 | { 98 | // Check for a corrupted stream. Read a max of 5 bytes. 99 | // In a future version, add a DataFormatException. 100 | if (shift == 5 * 7) // 5 bytes max per Int32, shift += 7 101 | throw new FormatException("Format_Bad7BitInt32"); 102 | 103 | // ReadByte handles end of stream cases for us. 104 | b = br.ReadByte(); 105 | count |= (b & 0x7F) << shift; 106 | shift += 7; 107 | } while ((b & 0x80) != 0); 108 | return count; 109 | } 110 | 111 | public static void WriteAllText(string path, string text, Encoding encoding) 112 | { 113 | using (var stream = OpenWriteNoAutoCommit(path)) 114 | using (var streamWriter = new StreamWriter(stream, encoding)) 115 | { 116 | streamWriter.Write(text); 117 | streamWriter.Flush(); 118 | stream.Commit(); 119 | } 120 | Debug.Assert(Exists(path)); 121 | } 122 | 123 | 124 | public static void SetConfigurationForDirectory(string directory, int packageSizeApproxKB) 125 | { 126 | if (directory == null) throw new ArgumentNullException(); 127 | directory = Path.GetFullPath(directory); 128 | var d = GetDirectory(directory); 129 | 130 | d.packageSize = (int)(0.8 * packageSizeApproxKB); 131 | d.memoryStreamCapacity = (int)(1 * packageSizeApproxKB); 132 | } 133 | 134 | public static BlobStream OpenWrite(string path) 135 | { 136 | var store = AcquireAndWriteHeader(path); 137 | return store.OpenWrite(true); 138 | } 139 | public static BlobStream OpenWriteNoAutoCommit(string path) 140 | { 141 | var store = AcquireAndWriteHeader(path); 142 | return store.OpenWrite(false); 143 | } 144 | 145 | public static BlobStream OpenWriteNoAutoCommit(string path, DateTime date) 146 | { 147 | var store = AcquireAndWriteHeader(path, date); 148 | return store.OpenWrite(false); 149 | } 150 | 151 | public static IEnumerable GetOpenDirectories() 152 | { 153 | lock (_lock) 154 | { 155 | return directories.Keys.ToList(); 156 | } 157 | } 158 | 159 | public static void CloseDirectory(string path) 160 | { 161 | if (path == null) throw new ArgumentNullException(); 162 | lock (_lock) 163 | { 164 | path = Path.GetFullPath(path); 165 | 166 | Debug.Assert(path[0] == '/' || path[1] == ':'); 167 | path = path.TrimEnd(trimChars); 168 | PackageDirectory dir; 169 | directories.TryGetValue(path, out dir); 170 | if (dir != null) 171 | { 172 | dir.Flush(true); 173 | directories.Remove(path); 174 | foreach (var b in dir.busyPackages.Union(dir.availablePackages)) 175 | { 176 | b.directory = null; 177 | b.fileName = null; 178 | b.ms?.Dispose(); 179 | b.ms = null; 180 | } 181 | dir.availablePackages = null; 182 | dir.busyPackages = null; 183 | dir.locations = null; 184 | dir.locationsToAdd = null; 185 | } 186 | } 187 | } 188 | 189 | public class Blob 190 | { 191 | public int DataStartOffset { get; internal set; } 192 | 193 | public string Name { get; internal set; } 194 | 195 | public string Path { get; internal set; } 196 | public int? Length { get; internal set; } 197 | public bool? ClosedCleanly { get; internal set; } 198 | public string PackagePath { get; internal set; } 199 | public string PackageFileName { get; internal set; } 200 | public bool Deleted { get; internal set; } 201 | 202 | public DateTime? LastWriteTime { get; internal set; } 203 | 204 | internal Stream fileStream; 205 | 206 | public Stream OpenRead() 207 | { 208 | if (fileStream == null) 209 | { 210 | return BlobStore.OpenRead(this.Path); 211 | } 212 | 213 | int len; 214 | if (Length != null) len = Length.Value; 215 | else 216 | { 217 | throw new Exception("Length should've been available, since fileStream is not null."); 218 | } 219 | 220 | fileStream.Seek(DataStartOffset, SeekOrigin.Begin); 221 | 222 | var f = new BlobStream(fileStream, len, true); 223 | fileStream = null; 224 | return f; 225 | } 226 | 227 | public string ReadAllText() 228 | { 229 | using (var sr = new StreamReader(OpenRead(), Encoding.UTF8, true)) 230 | { 231 | return sr.ReadToEnd(); 232 | } 233 | } 234 | public byte[] ReadAllBytes() 235 | { 236 | using (var stream = OpenRead()) 237 | { 238 | var arr = new byte[stream.Length]; 239 | int offset = 0; 240 | int readBytes; 241 | while ((readBytes = stream.Read(arr, offset, arr.Length - offset)) != 0) 242 | { 243 | offset += readBytes; 244 | } 245 | return arr; 246 | } 247 | } 248 | } 249 | 250 | public static IEnumerable EnumerateFiles(Stream package) 251 | { 252 | if (!package.CanSeek) package = new UnseekableStreamWrapper(package, package.Length); 253 | return EnumerateFiles(package, includeDeleted: false); 254 | } 255 | public static IEnumerable EnumerateFiles(Stream package, bool includeDeleted) 256 | { 257 | if (!package.CanSeek) package = new UnseekableStreamWrapper(package, package.Length); 258 | return EnumerateFiles(null, includeDeleted: includeDeleted, orderByDate: false, singlePackage: package, allowIndex: false); 259 | } 260 | public static IEnumerable EnumerateFiles(Stream package, bool includeDeleted, long packageLength) 261 | { 262 | if (!package.CanSeek) package = new UnseekableStreamWrapper(package, packageLength); 263 | return EnumerateFiles(null, includeDeleted: includeDeleted, orderByDate: false, singlePackage: package, allowIndex: false); 264 | } 265 | 266 | public static IEnumerable EnumerateFiles(string directory) 267 | { 268 | return EnumerateFiles(directory, includeDeleted: false, orderByDate: false, singlePackage: null, allowIndex: true); 269 | } 270 | public static IEnumerable EnumerateFiles(string directory, string pattern) 271 | { 272 | return EnumerateFiles(directory, pattern: pattern, includeDeleted: false, orderByDate: false, singlePackage: null, allowIndex: true); 273 | } 274 | 275 | public static IEnumerable EnumerateFiles(string directory, bool includeDeleted) 276 | { 277 | return EnumerateFiles(directory, includeDeleted: true, orderByDate: false, singlePackage: null, allowIndex: true); 278 | } 279 | 280 | public static IEnumerable EnumerateFiles(string directory, string pattern, bool includeDeleted, bool orderByDate, Stream singlePackage, bool allowIndex) 281 | { 282 | if (pattern == "*") pattern = null; 283 | string prefix = null; 284 | string suffix = null; 285 | if (pattern != null) 286 | { 287 | var star = pattern.IndexOf('*'); 288 | if (star == -1 || pattern.LastIndexOf('*') != star) throw new ArgumentException("Exactly one wildcard (*) character must be specified."); 289 | prefix = pattern.Substring(0, star); 290 | suffix = pattern.Substring(star + 1); 291 | } 292 | var items = EnumerateFiles(directory, includeDeleted: includeDeleted, orderByDate: orderByDate, singlePackage: null, allowIndex: allowIndex); 293 | if (pattern != null) items = items.Where(x => x.Name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && x.Name.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); 294 | return items; 295 | } 296 | 297 | 298 | public static IEnumerable EnumerateFiles(string directory, bool includeDeleted, bool orderByDate, Stream singlePackage, bool allowIndex) 299 | { 300 | if ((directory == null) == (singlePackage == null)) throw new ArgumentException(); 301 | 302 | if (allowIndex && singlePackage == null) 303 | { 304 | lock (_lock) 305 | { 306 | var dir = GetDirectory(directory); 307 | var pkgs = dir.locations.Values.GroupBy(x => x.PackageFileName); 308 | if (orderByDate) pkgs = pkgs.Select(x => 309 | { 310 | var z = Path.Combine(directory, x.Key); 311 | return new 312 | { 313 | Date = File.Exists(z) ? File.GetCreationTimeUtc(z) : DateTime.MaxValue, 314 | Package = x 315 | }; 316 | }).OrderBy(x => x.Date).Select(x => x.Package); 317 | string currentPackageName = null; 318 | string currentPackagePath = null; 319 | var k = pkgs.SelectMany(x => x.OrderBy(y => y.DataStartOffset)).Select(x => 320 | { 321 | if (x.PackageFileName != currentPackageName) 322 | { 323 | currentPackagePath = Path.Combine(directory, x.PackageFileName); 324 | currentPackageName = x.PackageFileName; 325 | } 326 | return new Blob() 327 | { 328 | LastWriteTime = x.Date, 329 | DataStartOffset = x.DataStartOffset, 330 | Deleted = x.DataStartOffset == -1, 331 | Name = x.BlobName, 332 | Length = x.Length, 333 | PackageFileName = x.PackageFileName, 334 | PackagePath = currentPackagePath, 335 | Path = Path.Combine(directory, x.BlobName), 336 | ClosedCleanly = null, 337 | }; 338 | }); 339 | if (!includeDeleted) k = k.Where(x => !x.Deleted); 340 | return k.ToList(); 341 | } 342 | 343 | } 344 | return EnumerateFilesNoIndex(directory, includeDeleted, orderByDate, singlePackage); 345 | 346 | } 347 | 348 | private static IEnumerable EnumerateFilesNoIndex(string directory, bool includeDeleted, bool orderByDate, Stream singlePackage) 349 | { 350 | 351 | 352 | 353 | IEnumerable packages; 354 | if (directory != null) 355 | { 356 | directory = Path.GetFullPath(directory); 357 | 358 | packages = 359 | #if NET35 360 | Directory.GetFiles( 361 | #else 362 | Directory.EnumerateFiles( 363 | #endif 364 | directory, "*.shaman-blobs"); 365 | 366 | if (orderByDate) 367 | { 368 | packages = packages.Select(x => new { Path = x, Date = File.GetCreationTimeUtc(x) }).OrderBy(x => x.Date).Select(x => x.Path); 369 | } 370 | 371 | } 372 | else 373 | { 374 | packages = new string[1]; 375 | } 376 | foreach (var pkg in packages) 377 | { 378 | using (var fs = singlePackage ?? File.Open(pkg, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite)) 379 | { 380 | Blob previous = null; 381 | var packagePath = pkg; 382 | while (true) 383 | { 384 | var first = fs.ReadByte(); 385 | if (first == -1) break; 386 | var len = 0; 387 | len |= first << 0; 388 | len |= ReadByteChecked(fs) << 8; 389 | len |= ReadByteChecked(fs) << 16; 390 | len |= ReadByteChecked(fs) << 24; 391 | 392 | 393 | var nl = 0; 394 | nl |= ReadByteChecked(fs) << 0; 395 | var d = ReadByteChecked(fs); 396 | // var wasNameCorrupted = false; 397 | // if (!SupportCorruptedFileNameLengths) 398 | { 399 | nl |= d << 8; 400 | } 401 | /* else 402 | { 403 | if (d != 0) 404 | { 405 | wasNameCorrupted = true; 406 | Console.WriteLine("Fixing corrupted entry..."); 407 | fs.Seek(-1, SeekOrigin.Current); 408 | fs.WriteByte(0); 409 | } 410 | }*/ 411 | 412 | var closedCleanly = ReadByteChecked(fs) == 1; 413 | var deleted = ReadByteChecked(fs) == 1; 414 | var nameBytes = new byte[nl]; 415 | while (nl != 0) 416 | { 417 | nl -= fs.Read(nameBytes, nameBytes.Length - nl, nl); 418 | } 419 | DateTime? date = null; 420 | nl = nameBytes.Length; 421 | if (nameBytes[nl - 1] == 0) 422 | { 423 | date = new DateTime(BitConverter.ToInt64(nameBytes, nl - 9), DateTimeKind.Utc); 424 | nl -= 9; 425 | } 426 | var name = Encoding.UTF8.GetString(nameBytes, 0, nl); 427 | var next = fs.Position + len; 428 | var blob = new Blob() 429 | { 430 | PackageFileName = directory != null ? Path.GetFileName(pkg) : null, 431 | PackagePath = packagePath, 432 | Length = len, 433 | Name = name, 434 | Path = directory != null ? Path.Combine(directory, name) : null, 435 | ClosedCleanly = closedCleanly, 436 | DataStartOffset = (int)fs.Position, 437 | fileStream = fs, 438 | Deleted = deleted, 439 | LastWriteTime = date 440 | }; 441 | if (previous != null) previous.fileStream = null; 442 | previous = blob; 443 | if (!deleted || includeDeleted) 444 | yield return blob; 445 | fs.Seek(next, SeekOrigin.Begin); 446 | } 447 | if (previous != null) previous.fileStream = null; 448 | } 449 | } 450 | if (directory != null) 451 | { 452 | var additional = new List(); 453 | lock (_lock) 454 | { 455 | var dir = GetDirectory(directory); 456 | 457 | foreach (var file in dir.locationsToAdd) 458 | { 459 | FileLocation mostRecent; 460 | dir.locations.TryGetValue(file.BlobName, out mostRecent); 461 | if (mostRecent == file) 462 | { 463 | additional.Add(new Blob() 464 | { 465 | DataStartOffset = file.DataStartOffset, 466 | PackageFileName = file.PackageFileName, 467 | Name = file.BlobName, 468 | ClosedCleanly = true, 469 | Length = file.Length, 470 | Path = Path.Combine(directory, file.BlobName) 471 | }); 472 | } 473 | } 474 | } 475 | foreach (var item in additional) 476 | { 477 | yield return item; 478 | } 479 | } 480 | } 481 | 482 | // public static bool SupportCorruptedFileNameLengths; 483 | 484 | public static Stream OpenRead(string path) 485 | { 486 | if (path == null) throw new ArgumentNullException(); 487 | path = Path.GetFullPath(path); 488 | var dirpath = Path.GetDirectoryName(path); 489 | var name = Path.GetFileName(path); 490 | lock (_lock) 491 | { 492 | var dir = GetDirectory(dirpath); 493 | FileLocation loc; 494 | if (!dir.locations.TryGetValue(name, out loc)) throw new FileNotFoundException("BlobStore file not found. ", path); 495 | if (loc.Package != null && loc.Package.ms != null) 496 | { 497 | var ms2 = loc.Package.CopyMemoryStream(); 498 | ms2.Seek(loc.DataStartOffset, SeekOrigin.Begin); 499 | return new BlobStream(ms2, loc.Length.Value, true); 500 | } 501 | else 502 | { 503 | var fs = File.Open(Path.Combine(dirpath, loc.PackageFileName), FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite); 504 | SeekToBeginOfBlobName(loc, fs); 505 | fs.Seek(-8, SeekOrigin.Current); 506 | var len = 0; 507 | len |= fs.ReadByte() << 0; 508 | len |= fs.ReadByte() << 8; 509 | len |= fs.ReadByte() << 16; 510 | len |= fs.ReadByte() << 24; 511 | fs.Seek(loc.DataStartOffset, SeekOrigin.Begin); 512 | return new BlobStream(fs, len, false); 513 | } 514 | 515 | } 516 | } 517 | 518 | public static bool Exists(string path) 519 | { 520 | if (string.IsNullOrEmpty(path)) return false; 521 | path = Path.GetFullPath(path); 522 | var blobName = Path.GetFileName(path); 523 | var dirpath = Path.GetDirectoryName(path); 524 | lock (_lock) 525 | { 526 | var dir = GetDirectory(dirpath); 527 | return dir.locations.ContainsKey(blobName); 528 | } 529 | } 530 | 531 | private static int ReadByteChecked(Stream fs) 532 | { 533 | var k = fs.ReadByte(); 534 | if (k == -1) throw new EndOfStreamException(); 535 | return k; 536 | } 537 | 538 | private static Dictionary directories = new Dictionary(StringComparer.OrdinalIgnoreCase); 539 | internal static object _lock = new object(); 540 | 541 | [Configuration] 542 | internal static int Configuration_PackageSize = (512 + 1024) * 1024; 543 | 544 | [Configuration] 545 | internal static int Configuration_MemoryStreamCapacity = (1024 + 1024) * 1024; 546 | 547 | private static BlobPackage AcquireAndWriteHeader(string path, DateTime? date = null) 548 | { 549 | if (path == null) throw new ArgumentNullException(); 550 | if (date != null && date.Value.Ticks == 0) date = null; 551 | path = Path.GetFullPath(path); 552 | var name = Path.GetFileName(path); 553 | var dirpath = Path.GetDirectoryName(path); 554 | lock (_lock) 555 | { 556 | var dir = GetDirectory(dirpath); 557 | BlobPackage pkg; 558 | if (dir.availablePackages.Count != 0) 559 | { 560 | var idx = dir.availablePackages.Count - 1; 561 | pkg = dir.availablePackages[idx]; 562 | dir.availablePackages.RemoveAt(idx); 563 | } 564 | else 565 | { 566 | pkg = new BlobPackage() { fileName = Guid.NewGuid() + ".shaman-blobs", directory = dir }; 567 | } 568 | dir.busyPackages.Add(pkg); 569 | pkg.WriteHeader(name, date); 570 | return pkg; 571 | } 572 | } 573 | 574 | private static char[] trimChars = new[] { '\\', '/' }; 575 | 576 | 577 | 578 | 579 | 580 | // must hold lock 581 | private static PackageDirectory GetDirectory(string dirpath) 582 | { 583 | Debug.Assert(dirpath[0] == '/' || dirpath[1] == ':'); 584 | dirpath = dirpath.TrimEnd(trimChars); 585 | PackageDirectory dir; 586 | directories.TryGetValue(dirpath, out dir); 587 | if (dir == null) 588 | { 589 | dir = new PackageDirectory(); 590 | dir.path = dirpath; 591 | directories[dirpath] = dir; 592 | var indexFile = Path.Combine(dirpath, ".shaman-blob-index"); 593 | if (File.Exists(indexFile)) 594 | { 595 | using (var stream = File.Open(indexFile, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read)) 596 | using (var br = new BinaryReader(stream, Encoding.UTF8)) 597 | { 598 | dir.locations = new Dictionary(); 599 | string package = null; 600 | while (true) 601 | { 602 | var tag = stream.ReadByte(); 603 | if (tag == -1) break; 604 | if (tag == 1) 605 | { 606 | package = br.ReadString(); 607 | } 608 | else if (tag != 0) throw new InvalidDataException(); 609 | 610 | DateTime? date = null; 611 | var name = br.ReadString(); 612 | int? length = null; 613 | if (name.Length == 0) 614 | { 615 | uint len = br.ReadUInt32(); 616 | var hasDate = (len & (1 << 31)) != 0; 617 | length = (int)(len & ~(1 << 31)); 618 | if (hasDate) 619 | date = new DateTime(br.ReadInt64(), DateTimeKind.Utc); 620 | name = br.ReadString(); 621 | } 622 | var offset = br.ReadInt32(); 623 | if (offset == -1) 624 | { 625 | dir.locations.Remove(name); 626 | } 627 | else 628 | { 629 | 630 | dir.locations[name] = new FileLocation() 631 | { 632 | BlobName = name, 633 | DataStartOffset = offset, 634 | PackageFileName = package, 635 | Date = date, 636 | Length = length 637 | }; 638 | } 639 | } 640 | 641 | } 642 | } 643 | else 644 | { 645 | dir.locations = new Dictionary(); 646 | 647 | if ( 648 | #if NET35 649 | Directory.GetFiles( 650 | #else 651 | Directory.EnumerateFiles( 652 | #endif 653 | dirpath, "*.shaman-blobs" 654 | ).Any()) 655 | { 656 | IndexDirectory(dir, dirpath); 657 | } 658 | } 659 | } 660 | return dir; 661 | } 662 | 663 | public static void FlushAll() 664 | { 665 | lock (_lock) 666 | { 667 | foreach (var dir in directories.Values) 668 | { 669 | dir.Flush(false); 670 | } 671 | } 672 | } 673 | 674 | public static void CloseAll() 675 | { 676 | lock (_lock) 677 | { 678 | foreach (var dir in directories.Values) 679 | { 680 | dir.Flush(true); 681 | } 682 | directories.Clear(); 683 | } 684 | } 685 | 686 | public static void CloseAllForShutdown() 687 | { 688 | lock (_lock) 689 | { 690 | CloseAll(); 691 | directories = null; 692 | _lock = null; 693 | } 694 | } 695 | 696 | 697 | public static void Delete(string path) 698 | { 699 | if (path == null) throw new ArgumentNullException(); 700 | path = Path.GetFullPath(path); 701 | var blobName = Path.GetFileName(path); 702 | var dirpath = Path.GetDirectoryName(path); 703 | lock (_lock) 704 | { 705 | var dir = GetDirectory(dirpath); 706 | FileLocation location; 707 | dir.locations.TryGetValue(blobName, out location); 708 | if (location == null) return; 709 | dir.locationsToAdd.Add(new FileLocation() 710 | { 711 | BlobName = blobName, 712 | DataStartOffset = -1 713 | }); 714 | dir.locations.Remove(blobName); 715 | using (var fs = location.Package?.ms == null ? (Stream)File.Open(Path.Combine(dir.path, location.PackageFileName), FileMode.Open, FileAccess.ReadWrite, FileShare.Delete | FileShare.ReadWrite) : null) 716 | { 717 | 718 | var oldposition = fs == null ? location.Package.ms.Position : -1; 719 | var stream = fs; 720 | if (fs == null) 721 | { 722 | stream = location.Package.ms; 723 | } 724 | SeekToBeginOfBlobName(location, stream); 725 | stream.Seek(-1, SeekOrigin.Current); 726 | stream.WriteByte(1); 727 | 728 | if (fs == null) 729 | stream.Seek(oldposition, SeekOrigin.Begin); 730 | } 731 | 732 | 733 | } 734 | Debug.Assert(!Exists(path)); 735 | } 736 | 737 | private static void SeekToBeginOfBlobName(FileLocation location, Stream stream) 738 | { 739 | var idx = 1; 740 | while (true) 741 | { 742 | stream.Seek(location.DataStartOffset - idx, SeekOrigin.Begin); 743 | var b = stream.ReadByte(); 744 | if (b == 0 && idx == 1) 745 | { 746 | idx += 8; 747 | continue; 748 | } 749 | if (b == 0 || b == 1) break; 750 | idx++; 751 | } 752 | } 753 | 754 | public static string ReadAllText(string path) 755 | { 756 | using (var sr = new StreamReader(OpenRead(path), Encoding.UTF8, true)) 757 | { 758 | return sr.ReadToEnd(); 759 | } 760 | } 761 | 762 | public static int GetLength(string path) 763 | { 764 | using (var stream = OpenRead(path)) 765 | { 766 | return (int)stream.Length; 767 | } 768 | } 769 | 770 | 771 | #if NET35 772 | private static void InternalCopyTo(Stream source, Stream destination, int bufferSize) 773 | { 774 | byte[] array = new byte[bufferSize]; 775 | int count; 776 | while ((count = source.Read(array, 0, array.Length)) != 0) 777 | { 778 | destination.Write(array, 0, count); 779 | } 780 | } 781 | 782 | #endif 783 | 784 | public static void MigrateFolder(string folder) 785 | { 786 | if (folder == null) throw new ArgumentNullException(); 787 | folder = Path.GetFullPath(folder); 788 | #if NET35 789 | foreach (var file in Directory.GetFiles(folder)) 790 | #else 791 | foreach (var file in Directory.EnumerateFiles(folder)) 792 | #endif 793 | { 794 | var ext = Path.GetExtension(file); 795 | if (ext == ".shaman-blobs") continue; 796 | if (ext == ".shaman-blob-index") continue; 797 | if (BlobStore.Exists(file)) continue; 798 | Console.WriteLine(file); 799 | using (var fs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.Read)) 800 | { 801 | using (var blob = OpenWrite(file)) 802 | { 803 | #if NET35 804 | InternalCopyTo(fs, blob, 81920); 805 | #else 806 | fs.CopyTo(blob); 807 | #endif 808 | blob.Commit(); 809 | } 810 | } 811 | } 812 | var dir = GetDirectory(folder); 813 | dir.Flush(true); 814 | Console.WriteLine("Migration done, deleting files..."); 815 | foreach (var file in Directory 816 | #if NET35 817 | .GetFiles( 818 | #else 819 | .EnumerateFiles( 820 | #endif 821 | folder)) 822 | { 823 | var ext = Path.GetExtension(file); 824 | 825 | if (ext == ".shaman-blobs") continue; 826 | if (ext == ".shaman-blob-index") continue; 827 | File.Delete(file); 828 | } 829 | Console.WriteLine("Files deleted."); 830 | } 831 | 832 | public static void FlushDirectory(string path) 833 | { 834 | if (path == null) throw new ArgumentNullException(); 835 | path = Path.GetFullPath(path); 836 | var dir = GetDirectory(path); 837 | dir.Flush(false); 838 | } 839 | } 840 | } 841 | -------------------------------------------------------------------------------- /Shaman.BlobStore/BlobStream.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | using System.Threading; 7 | 8 | namespace Shaman.Runtime 9 | { 10 | public class BlobStream : Stream 11 | { 12 | private Stream readStream; 13 | private int length; 14 | private int remainingToRead; 15 | private BlobPackage package; 16 | private bool leaveOpen; 17 | private bool autocommit; 18 | 19 | internal BlobStream(Stream readStream, int length, bool leaveOpen) 20 | { 21 | this.readStream = readStream; 22 | this.length = length; 23 | this.remainingToRead = length; 24 | this.leaveOpen = leaveOpen; 25 | } 26 | internal BlobStream(BlobPackage pkg, bool autocommit) 27 | { 28 | this.package = pkg; 29 | this.autocommit = autocommit; 30 | } 31 | public override int ReadByte() 32 | { 33 | if (remainingToRead-- == 0) return -1; 34 | return readStream.ReadByte(); 35 | } 36 | 37 | public override void WriteByte(byte value) 38 | { 39 | package.EnsureAdditionalCapacity(1).WriteByte(value); 40 | } 41 | public override bool CanRead 42 | { 43 | get 44 | { 45 | return readStream != null; 46 | } 47 | } 48 | 49 | public override bool CanSeek 50 | { 51 | get 52 | { 53 | return CanRead; 54 | } 55 | } 56 | 57 | public override bool CanWrite 58 | { 59 | get 60 | { 61 | return package != null; 62 | } 63 | } 64 | 65 | public override long Length 66 | { 67 | get 68 | { 69 | if (package != null) return package.ms.Length - package.startOfBlobData; 70 | return length; 71 | } 72 | } 73 | 74 | public override long Position 75 | { 76 | get 77 | { 78 | if (package != null) return Length; 79 | return length - remainingToRead; 80 | } 81 | 82 | set 83 | { 84 | throw new NotSupportedException(); 85 | } 86 | } 87 | 88 | public override void Flush() 89 | { 90 | } 91 | 92 | public override int Read(byte[] buffer, int offset, int count) 93 | { 94 | if (remainingToRead == 0) return 0; 95 | var r = readStream.Read(buffer, offset, Math.Min(remainingToRead, count)); 96 | remainingToRead -= r; 97 | return r; 98 | } 99 | 100 | public override long Seek(long offset, SeekOrigin origin) 101 | { 102 | if (readStream == null) throw new NotSupportedException(); 103 | var newPosition = origin == SeekOrigin.Begin ? 0 : origin == SeekOrigin.Current ? Position : Length; 104 | newPosition += offset; 105 | if (newPosition < 0 || newPosition >= Length) throw new ArgumentOutOfRangeException(); 106 | var diff = (int)(newPosition - Position); 107 | remainingToRead -= diff; 108 | readStream.Seek(diff, SeekOrigin.Current); 109 | return Position; 110 | } 111 | 112 | public override void SetLength(long value) 113 | { 114 | throw new NotSupportedException(); 115 | } 116 | 117 | public override void Write(byte[] buffer, int offset, int count) 118 | { 119 | package.EnsureAdditionalCapacity(count).Write(buffer, offset, count); 120 | } 121 | 122 | protected override void Dispose(bool disposing) 123 | { 124 | if (disposing) 125 | { 126 | if (readStream != null) 127 | { 128 | if (!leaveOpen) 129 | readStream.Dispose(); 130 | } 131 | else 132 | { 133 | var pkg = Interlocked.Exchange(ref package, null); 134 | if (pkg != null) 135 | { 136 | pkg.Release(autocommit); 137 | } 138 | } 139 | } 140 | } 141 | 142 | 143 | public void Commit() 144 | { 145 | var pkg = Interlocked.Exchange(ref package, null); 146 | if (pkg != null) 147 | { 148 | pkg.Release(true); 149 | } 150 | else 151 | { 152 | throw new InvalidOperationException(); 153 | } 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /Shaman.BlobStore/ExtensionMethods.cs: -------------------------------------------------------------------------------- 1 | #if CORECLR 2 | 3 | using System; 4 | using System.IO; 5 | 6 | namespace Shaman.Runtime 7 | { 8 | internal static class ExtensionMethods 9 | { 10 | public static byte[] GetBuffer(this MemoryStream ms) 11 | { 12 | ArraySegment buffer; 13 | if(!ms.TryGetBuffer(out buffer)) throw new ArgumentException(); 14 | return buffer.Array; 15 | } 16 | 17 | } 18 | } 19 | 20 | #endif -------------------------------------------------------------------------------- /Shaman.BlobStore/FileLocation.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.Linq; 4 | using System.Text; 5 | 6 | namespace Shaman.Runtime 7 | { 8 | internal class FileLocation 9 | { 10 | public int DataStartOffset; 11 | public string PackageFileName; 12 | public string BlobName; 13 | internal BlobPackage Package; 14 | internal int? Length; 15 | public DateTime? Date; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Shaman.BlobStore/PackageDirectory.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Linq; 5 | using System.Text; 6 | 7 | namespace Shaman.Runtime 8 | { 9 | class PackageDirectory 10 | { 11 | internal List availablePackages = new List(); 12 | internal List busyPackages = new List(); 13 | internal string path; 14 | internal Dictionary locations; 15 | internal List locationsToAdd = new List(); 16 | internal int? packageSize; 17 | internal int? memoryStreamCapacity; 18 | 19 | 20 | 21 | public void Flush(bool close) 22 | { 23 | lock(BlobStore._lock) 24 | { 25 | foreach (var pkg in this.availablePackages) 26 | { 27 | pkg.WriteToFile(close); 28 | if (close) pkg.ms = null; 29 | } 30 | if (close) availablePackages.Clear(); 31 | foreach (var pkg in this.busyPackages) 32 | { 33 | pkg.WriteToFile(close); 34 | } 35 | if (this.locationsToAdd.Count != 0) 36 | { 37 | using (var stream = File.Open(Path.Combine(this.path, ".shaman-blob-index"), FileMode.Append, FileAccess.Write, FileShare.Delete)) 38 | using (var bw = new BinaryWriter(stream, Encoding.UTF8)) 39 | { 40 | string prevPackage = null; 41 | foreach (var item in locationsToAdd) 42 | { 43 | if (prevPackage != item.PackageFileName) 44 | { 45 | prevPackage = item.PackageFileName; 46 | bw.Write((byte)1); 47 | bw.Write(item.PackageFileName ?? string.Empty); 48 | } 49 | else 50 | { 51 | bw.Write((byte)0); 52 | } 53 | 54 | bw.Write((byte)0); 55 | var length = 56 | item.DataStartOffset == -1 ? 0 : 57 | (uint)item.Length.Value; 58 | if (item.Date != null) length |= (uint)1 << 31; 59 | 60 | bw.Write(length); 61 | 62 | if (item.Date != null) 63 | { 64 | bw.Write(item.Date.Value.Ticks); 65 | } 66 | 67 | 68 | bw.Write(item.BlobName); 69 | bw.Write(item.DataStartOffset); 70 | } 71 | } 72 | locationsToAdd.Clear(); 73 | } 74 | } 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /Shaman.BlobStore/Shaman.BlobStore.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | Provides an almost drop-in replacement for System.IO.File, optimized for a very large number of small files. Data is batched in package files, and can be read sequentially. 4 | 1.0.1.11 5 | Andrea Martinelli 6 | net35;net45;netstandard2.0 7 | $(DefineConstants);SHAMAN_BLOBSTORE 8 | Shaman.BlobStore 9 | 1.0.0.0 10 | ShamanOpenSourceKey.snk 11 | true 12 | true 13 | Shaman.BlobStore 14 | http://shaman.io/images/shaman-nuget-icon.png 15 | http://shaman.io 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | $(DefineConstants);NET35 30 | 31 | 32 | $(DefineConstants);CORECLR 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /Shaman.BlobStore/ShamanOpenSourceKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/antiufo/Shaman.BlobStore/f69fb0f4873b0bb76b8d8a1dca69b8c4cb2a1424/Shaman.BlobStore/ShamanOpenSourceKey.snk -------------------------------------------------------------------------------- /Shaman.BlobStore/UnseekableStreamWrapper.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Text; 5 | using System.Threading; 6 | #if !NET35 7 | using System.Threading.Tasks; 8 | #endif 9 | 10 | namespace Shaman.Runtime 11 | { 12 | #if SHAMAN_BLOBSTORE 13 | internal 14 | #else 15 | public 16 | #endif 17 | class UnseekableStreamWrapper : Stream 18 | { 19 | public UnseekableStreamWrapper(Stream stream, long length) 20 | { 21 | this.inner = stream; 22 | this._length = length; 23 | } 24 | 25 | public UnseekableStreamWrapper(Stream stream) 26 | { 27 | this.inner = stream; 28 | this._length = -1; 29 | } 30 | 31 | private Stream inner; 32 | private long _length; 33 | public override bool CanRead 34 | { 35 | get 36 | { 37 | return inner.CanRead; 38 | } 39 | } 40 | 41 | public override bool CanSeek 42 | { 43 | get 44 | { 45 | return true; 46 | } 47 | } 48 | 49 | public override bool CanWrite 50 | { 51 | get 52 | { 53 | return false; 54 | } 55 | } 56 | 57 | public override long Length 58 | { 59 | get 60 | { 61 | if (_length == -1) throw new NotSupportedException(); 62 | return _length; 63 | } 64 | } 65 | 66 | public override long Position 67 | { 68 | get 69 | { 70 | return _position; 71 | } 72 | 73 | set 74 | { 75 | Seek(value, SeekOrigin.Begin); 76 | } 77 | } 78 | private long _position; 79 | 80 | public override void Flush() 81 | { 82 | throw new NotSupportedException(); 83 | } 84 | 85 | public override int Read(byte[] buffer, int offset, int count) 86 | { 87 | var v = inner.Read(buffer, offset, count); 88 | _position += v; 89 | return v; 90 | } 91 | 92 | public override long Seek(long offset, SeekOrigin origin) 93 | { 94 | var pos = 95 | origin == SeekOrigin.Begin ? offset : 96 | origin == SeekOrigin.Current ? Position + offset : 97 | origin == SeekOrigin.End ? Length + offset : 98 | -1; 99 | if (pos < 0 || (_length != -1 && pos > _length)) throw new ArgumentException(); 100 | if (pos == _position) return _position; 101 | if (pos < _position) throw new NotSupportedException("Cannot rewind stream."); 102 | var diff = pos - _position; 103 | var buffer = new byte[Math.Min(81920, diff)]; 104 | while (true) 105 | { 106 | var read = Read(buffer, 0, (int)Math.Min((long)diff, buffer.Length)); 107 | if (read == 0) 108 | { 109 | if (_position == pos) return pos; 110 | throw new EndOfStreamException(); 111 | } 112 | diff -= read; 113 | if (diff == 0) return pos; 114 | } 115 | 116 | } 117 | 118 | public override void SetLength(long value) 119 | { 120 | throw new NotSupportedException(); 121 | } 122 | 123 | public override void Write(byte[] buffer, int offset, int count) 124 | { 125 | throw new NotSupportedException(); 126 | } 127 | 128 | #if !NET35 129 | public async override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) 130 | { 131 | var k = await inner.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); 132 | _position += k; 133 | return k; 134 | } 135 | #endif 136 | 137 | public override int ReadTimeout 138 | { 139 | get 140 | { 141 | return inner.ReadTimeout; 142 | } 143 | 144 | set 145 | { 146 | inner.ReadTimeout = value; 147 | } 148 | } 149 | 150 | #if !CORECLR 151 | public override void Close() 152 | { 153 | inner.Close(); 154 | } 155 | #endif 156 | 157 | protected override void Dispose(bool disposing) 158 | { 159 | if (disposing) 160 | inner.Dispose(); 161 | } 162 | public override bool CanTimeout 163 | { 164 | get 165 | { 166 | return inner.CanTimeout; 167 | } 168 | } 169 | public override int ReadByte() 170 | { 171 | var k = inner.ReadByte(); 172 | if (k != -1) 173 | { 174 | _position++; 175 | } 176 | return k; 177 | } 178 | 179 | } 180 | } 181 | --------------------------------------------------------------------------------