├── tests ├── files │ ├── empty_file │ └── ascii_table.txt ├── test_pipelines.nim.cfg ├── all_tests.nim ├── nim.cfg ├── test_vm_readme.nim ├── test_readme_examples.nim ├── test_vm_textio.nim ├── test_vm.nim ├── base64.nim ├── test_pipelines.nim ├── test_inputs.nim ├── test_buffers.nim └── test_outputs.nim ├── faststreams ├── std_adapters.nim ├── stdin.nim ├── stdout.nim ├── multisync.nim ├── async_backend.nim ├── asynctools_adapters.nim ├── chronos_adapters.nim ├── textio.nim ├── pipelines.nim ├── buffers.nim ├── outputs.nim └── inputs.nim ├── .gitignore ├── config.nims ├── faststreams.nim ├── nim.cfg ├── .github └── workflows │ └── ci.yml ├── LICENSE-MIT ├── faststreams.nimble ├── LICENSE-APACHEv2 └── README.md /tests/files/empty_file: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /faststreams/std_adapters.nim: -------------------------------------------------------------------------------- 1 | import 2 | async_backend 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | nimcache/ 2 | *.exe 3 | nimble.develop 4 | nimble.paths 5 | build/ 6 | -------------------------------------------------------------------------------- /tests/test_pipelines.nim.cfg: -------------------------------------------------------------------------------- 1 | --linedir:off 2 | --linetrace:off 3 | --stacktrace:off 4 | 5 | -------------------------------------------------------------------------------- /faststreams/stdin.nim: -------------------------------------------------------------------------------- 1 | import 2 | inputs 3 | 4 | let fsStdIn* {.threadvar.} = fileInput(system.stdin) 5 | 6 | -------------------------------------------------------------------------------- /config.nims: -------------------------------------------------------------------------------- 1 | # begin Nimble config (version 1) 2 | when fileExists("nimble.paths"): 3 | include "nimble.paths" 4 | # end Nimble config 5 | -------------------------------------------------------------------------------- /faststreams.nim: -------------------------------------------------------------------------------- 1 | import 2 | faststreams/[inputs, outputs, pipelines, multisync] 3 | 4 | export 5 | inputs, outputs, pipelines, multisync 6 | 7 | -------------------------------------------------------------------------------- /tests/all_tests.nim: -------------------------------------------------------------------------------- 1 | import 2 | test_buffers, 3 | test_inputs, 4 | test_outputs, 5 | test_pipelines, 6 | test_readme_examples, 7 | test_vm_readme, 8 | test_vm_textio, 9 | test_vm 10 | -------------------------------------------------------------------------------- /nim.cfg: -------------------------------------------------------------------------------- 1 | # Avoid some rare stack corruption while using exceptions with a SEH-enabled 2 | # toolchain: https://github.com/status-im/nimbus-eth2/issues/3121 3 | @if windows and not vcc: 4 | --define:nimRawSetjmp 5 | @end 6 | -------------------------------------------------------------------------------- /tests/nim.cfg: -------------------------------------------------------------------------------- 1 | --threads:on 2 | 3 | # Avoid some rare stack corruption while using exceptions with a SEH-enabled 4 | # toolchain: https://github.com/status-im/nimbus-eth2/issues/3121 5 | @if windows and not vcc: 6 | --define:nimRawSetjmp 7 | @end 8 | -------------------------------------------------------------------------------- /faststreams/stdout.nim: -------------------------------------------------------------------------------- 1 | import 2 | outputs 3 | 4 | var fsStdOut* {.threadvar.}: OutputStream 5 | 6 | proc initFsStdOut* = 7 | ## This proc must be called in each thread where 8 | ## the `fsStdOut` variable will be used. 9 | if fsStdOut == nil: 10 | fsStdOut = fileOutput(system.stdout) 11 | 12 | initFsStdOut() 13 | 14 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | jobs: 10 | build: 11 | uses: status-im/nimbus-common-workflow/.github/workflows/common.yml@main 12 | with: 13 | test-command: | 14 | env NIMLANG=c nimble test 15 | nimble install chronos 16 | env NIMLANG=c nimble testChronos || true 17 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Status Research & Development GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/test_vm_readme.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import 4 | typetraits, ../faststreams 5 | 6 | proc writeNimRepr*(stream: OutputStream, str: string) = 7 | stream.write '"' 8 | 9 | for c in str: 10 | if c == '"': 11 | stream.write ['\'', '"'] 12 | else: 13 | stream.write c 14 | 15 | stream.write '"' 16 | 17 | proc writeNimRepr*(stream: OutputStream, x: char) = 18 | stream.write ['\'', x, '\''] 19 | 20 | proc writeNimRepr*(stream: OutputStream, x: int) = 21 | stream.write $x # Making this more optimal has been left 22 | # as an exercise for the reader 23 | 24 | proc writeNimRepr*[T](stream: OutputStream, obj: T) = 25 | stream.write typetraits.name(T) 26 | stream.write '(' 27 | 28 | var firstField = true 29 | for name, val in fieldPairs(obj): 30 | if not firstField: 31 | stream.write ", " 32 | 33 | stream.write name 34 | stream.write ": " 35 | stream.writeNimRepr val 36 | 37 | firstField = false 38 | 39 | stream.write ')' 40 | 41 | type 42 | ABC = object 43 | a: int 44 | b: char 45 | c: string 46 | 47 | static: 48 | var stream = memoryOutput() 49 | stream.writeNimRepr(ABC(a: 1, b: 'b', c: "str")) 50 | var repr = stream.getOutput(string) 51 | 52 | doAssert repr == "ABC(a: 1, b: 'b', c: \"str\")" 53 | 54 | -------------------------------------------------------------------------------- /tests/test_readme_examples.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import 4 | typetraits, ../faststreams 5 | 6 | proc writeNimRepr*(stream: OutputStream, str: string) = 7 | stream.write '"' 8 | 9 | for c in str: 10 | if c == '"': 11 | stream.write ['\'', '"'] 12 | else: 13 | stream.write c 14 | 15 | stream.write '"' 16 | 17 | proc writeNimRepr*(stream: OutputStream, x: char) = 18 | stream.write ['\'', x, '\''] 19 | 20 | proc writeNimRepr*(stream: OutputStream, x: int) = 21 | stream.write $x # Making this more optimal has been left 22 | # as an exercise for the reader 23 | 24 | proc writeNimRepr*[T](stream: OutputStream, obj: T) = 25 | stream.write typetraits.name(T) 26 | stream.write '(' 27 | 28 | var firstField = true 29 | for name, val in fieldPairs(obj): 30 | if not firstField: 31 | stream.write ", " 32 | 33 | stream.write name 34 | stream.write ": " 35 | stream.writeNimRepr val 36 | 37 | firstField = false 38 | 39 | stream.write ')' 40 | 41 | type 42 | ABC = object 43 | a: int 44 | b: char 45 | c: string 46 | 47 | block: 48 | var stream = memoryOutput() 49 | stream.writeNimRepr(ABC(a: 1, b: 'b', c: "str")) 50 | var repr = stream.getOutput(string) 51 | 52 | doAssert repr == "ABC(a: 1, b: 'b', c: \"str\")" 53 | 54 | -------------------------------------------------------------------------------- /faststreams/multisync.nim: -------------------------------------------------------------------------------- 1 | import 2 | async_backend 3 | 4 | export 5 | async_backend 6 | 7 | when fsAsyncSupport: 8 | import 9 | stew/shims/macros, 10 | "."/[inputs, outputs] 11 | 12 | macro fsMultiSync*(body: untyped) = 13 | # We will produce an identical copy of the annotated proc, 14 | # but taking async parameters and having the async pragma. 15 | var 16 | asyncProcBody = copy body 17 | asyncProcParams = asyncProcBody[3] 18 | 19 | asyncProcBody.addPragma(bindSym"async") 20 | 21 | # The return types becomes Future[T] 22 | if asyncProcParams[0].kind == nnkEmpty: 23 | asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", ident"void") 24 | else: 25 | asyncProcParams[0] = newTree(nnkBracketExpr, bindSym"Future", asyncProcParams[0]) 26 | 27 | # We replace all stream inputs with their async counterparts 28 | for i in 1 ..< asyncProcParams.len: 29 | let paramsDef = asyncProcParams[i] 30 | let typ = paramsDef[^2] 31 | if eqIdent(typ, "InputStream"): 32 | paramsDef[^2] = bindSym "AsyncInputStream" 33 | elif eqIdent(typ, "OutputStream"): 34 | paramsDef[^2] = bindSym "AsyncOutputStream" 35 | 36 | result = newStmtList(body, asyncProcBody) 37 | when defined(debugSupportAsync): 38 | echo result.repr 39 | else: 40 | macro fsMultiSync*(body: untyped) = body 41 | -------------------------------------------------------------------------------- /tests/test_vm_textio.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import unittest2, ../faststreams, ../faststreams/textio 4 | 5 | proc readAll(s: InputStream): string = 6 | while s.readable: 7 | result.add s.read.char 8 | 9 | suite "TextIO": 10 | dualTest "writeText int": 11 | let s = memoryOutput() 12 | var n = 123 13 | s.writeText(n) 14 | check s.getOutput(string) == "123" 15 | 16 | dualTest "writeText uint": 17 | let s = memoryOutput() 18 | var n = 123'u64 19 | s.writeText(n) 20 | check s.getOutput(string) == "123" 21 | 22 | dualTest "writeText float64": 23 | let s = memoryOutput() 24 | var n = 1.23'f64 25 | s.writeText(n) 26 | check s.getOutput(string) == "1.23" 27 | 28 | dualTest "writeText float32": 29 | let s = memoryOutput() 30 | var n = 1.23'f32 31 | s.writeText(n) 32 | check s.getOutput(string) == "1.23" 33 | 34 | dualTest "writeText string": 35 | let s = memoryOutput() 36 | var n = "abc" 37 | s.writeText(n) 38 | check s.getOutput(string) == "abc" 39 | 40 | dualTest "writeHex": 41 | let s = memoryOutput() 42 | var n = "abc" 43 | s.writeHex(n) 44 | check s.getOutput(string) == "616263" 45 | 46 | dualTest "readLine": 47 | let s = memoryInput("abc\ndef") 48 | check s.readLine() == "abc" 49 | check s.readLine() == "def" 50 | 51 | dualTest "readUntil": 52 | let s = memoryInput("abc\ndef") 53 | check s.readUntil(@['d']).get == "abc\n" 54 | 55 | dualTest "nextLine": 56 | let s = memoryInput("abc\ndef") 57 | check s.nextLine().get == "abc" 58 | check s.nextLine().get == "def" 59 | check s.nextLine() == none(string) 60 | 61 | dualTest "lines": 62 | let s = memoryInput("abc\ndef") 63 | var found = newSeq[string]() 64 | for line in lines(s): 65 | found.add line 66 | check found == @["abc", "def"] 67 | -------------------------------------------------------------------------------- /faststreams/async_backend.nim: -------------------------------------------------------------------------------- 1 | const 2 | # To compile with async support, use `-d:asyncBackend=chronos|asyncdispatch` 3 | asyncBackend {.strdefine.} = "none" 4 | 5 | const 6 | faststreams_async_backend {.strdefine.} = "" 7 | 8 | when faststreams_async_backend != "": 9 | {.fatal: "use `-d:asyncBackend` instead".} 10 | 11 | type 12 | CloseBehavior* = enum 13 | waitAsyncClose 14 | dontWaitAsyncClose 15 | 16 | const 17 | debugHelpers* = defined(debugHelpers) 18 | fsAsyncSupport* = asyncBackend != "none" 19 | 20 | when asyncBackend == "none": 21 | discard 22 | elif asyncBackend == "chronos": 23 | {.warning: "chronos backend uses nested calls to `waitFor` which is not supported by chronos - it is not recommended to use it until this has been resolved".} 24 | 25 | import 26 | chronos 27 | 28 | export 29 | chronos 30 | 31 | template fsAwait*(f: Future): untyped = 32 | await f 33 | 34 | elif asyncBackend == "asyncdispatch": 35 | {.warning: "asyncdispatch backend currently fails tests - it may or may not work as expected".} 36 | 37 | import 38 | std/asyncdispatch 39 | 40 | export 41 | asyncdispatch 42 | 43 | template fsAwait*(awaited: Future): untyped = 44 | # TODO revisit after https://github.com/nim-lang/Nim/pull/12085/ is merged 45 | let f = awaited 46 | yield f 47 | if not isNil(f.error): 48 | raise f.error 49 | f.read 50 | 51 | type Duration* = int 52 | 53 | else: 54 | {.fatal: "Unrecognized network backend: " & asyncBackend .} 55 | 56 | when defined(danger): 57 | template fsAssert*(x) = discard 58 | template fsAssert*(x, msg) = discard 59 | else: 60 | template fsAssert*(x) = doAssert(x) 61 | template fsAssert*(x, msg) = doAssert(x, msg) 62 | 63 | template fsTranslateErrors*(errMsg: string, body: untyped) = 64 | try: 65 | body 66 | except IOError as err: 67 | raise err 68 | except Exception as err: 69 | if err[] of Defect: 70 | raise (ref Defect)(err) 71 | else: 72 | raise newException(IOError, errMsg, err) 73 | 74 | template noAwait*(expr: untyped): untyped = 75 | expr 76 | 77 | -------------------------------------------------------------------------------- /faststreams.nimble: -------------------------------------------------------------------------------- 1 | mode = ScriptMode.Verbose 2 | 3 | packageName = "faststreams" 4 | version = "0.5.0" 5 | author = "Status Research & Development GmbH" 6 | description = "Nearly zero-overhead input/output streams for Nim" 7 | license = "Apache License 2.0" 8 | skipDirs = @["tests"] 9 | 10 | requires "nim >= 1.6.0", 11 | "stew >= 0.2.0", 12 | "unittest2" 13 | 14 | let nimc = getEnv("NIMC", "nim") # Which nim compiler to use 15 | let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js) 16 | let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler 17 | let verbose = getEnv("V", "") notin ["", "0"] 18 | 19 | let cfg = 20 | " --styleCheck:usages --styleCheck:error" & 21 | (if verbose: "" else: " --verbosity:0 --hints:off") & 22 | " --skipParentCfg --skipUserCfg --outdir:build --nimcache:build/nimcache -f" 23 | 24 | proc build(args, path: string) = 25 | exec nimc & " " & lang & " " & cfg & " " & flags & " " & args & " " & path 26 | 27 | import strutils 28 | proc run(args, path: string) = 29 | build args & " --mm:refc -r", path 30 | if (NimMajor, NimMinor) >= (2, 0): 31 | build args & " --mm:orc -r", path 32 | 33 | if (NimMajor, NimMinor) >= (2, 2) and defined(linux) and defined(amd64) and "danger" in args: 34 | # Test with AddressSanitizer 35 | build args & " --mm:orc -d:useMalloc --cc:clang --passc:-fsanitize=address --passl:-fsanitize=address --debugger:native -r", path 36 | 37 | task test, "Run all tests": 38 | # TODO asyncdispatch backend is broken / untested 39 | # TODO chronos backend uses nested waitFor which is not supported 40 | for backend in ["-d:asyncBackend=none"]: 41 | for threads in ["--threads:off", "--threads:on"]: 42 | for mode in ["-d:debug", "-d:release", "-d:danger"]: 43 | run backend & " " & threads & " " & mode, "tests/all_tests" 44 | 45 | task testChronos, "Run chronos tests": 46 | # TODO chronos backend uses nested waitFor which is not supported 47 | for backend in ["-d:asyncBackend=chronos"]: 48 | for threads in ["--threads:off", "--threads:on"]: 49 | for mode in ["-d:debug", "-d:release", "-d:danger"]: 50 | run backend & " " & threads & " " & mode, "tests/all_tests" 51 | -------------------------------------------------------------------------------- /tests/files/ascii_table.txt: -------------------------------------------------------------------------------- 1 | |000 nul|001 soh|002 stx|003 etx|004 eot|005 enq|006 ack|007 bel| 2 | |010 bs |011 ht |012 nl |013 vt |014 np |015 cr |016 so |017 si | 3 | |020 dle|021 dc1|022 dc2|023 dc3|024 dc4|025 nak|026 syn|027 etb| 4 | |030 can|031 em |032 sub|033 esc|034 fs |035 gs |036 rs |037 us | 5 | |040 sp |041 ! |042 " |043 # |044 $ |045 % |046 & |047 ' | 6 | |050 ( |051 ) |052 * |053 + |054 , |055 - |056 . |057 / | 7 | |060 0 |061 1 |062 2 |063 3 |064 4 |065 5 |066 6 |067 7 | 8 | |070 8 |071 9 |072 : |073 ; |074 < |075 = |076 > |077 ? | 9 | |100 @ |101 A |102 B |103 C |104 D |105 E |106 F |107 G | 10 | |110 H |111 I |112 J |113 K |114 L |115 M |116 N |117 O | 11 | |120 P |121 Q |122 R |123 S |124 T |125 U |126 V |127 W | 12 | |130 X |131 Y |132 Z |133 [ |134 \ |135 ] |136 ^ |137 _ | 13 | |140 ` |141 a |142 b |143 c |144 d |145 e |146 f |147 g | 14 | |150 h |151 i |152 j |153 k |154 l |155 m |156 n |157 o | 15 | |160 p |161 q |162 r |163 s |164 t |165 u |166 v |167 w | 16 | |170 x |171 y |172 z |173 { |174 | |175 } |176 ~ |177 del| 17 | 18 | | 00 nul| 01 soh| 02 stx| 03 etx| 04 eot| 05 enq| 06 ack| 07 bel| 19 | | 08 bs | 09 ht | 0a nl | 0b vt | 0c np | 0d cr | 0e so | 0f si | 20 | | 10 dle| 11 dc1| 12 dc2| 13 dc3| 14 dc4| 15 nak| 16 syn| 17 etb| 21 | | 18 can| 19 em | 1a sub| 1b esc| 1c fs | 1d gs | 1e rs | 1f us | 22 | | 20 sp | 21 ! | 22 " | 23 # | 24 $ | 25 % | 26 & | 27 ' | 23 | | 28 ( | 29 ) | 2a * | 2b + | 2c , | 2d - | 2e . | 2f / | 24 | | 30 0 | 31 1 | 32 2 | 33 3 | 34 4 | 35 5 | 36 6 | 37 7 | 25 | | 38 8 | 39 9 | 3a : | 3b ; | 3c < | 3d = | 3e > | 3f ? | 26 | | 40 @ | 41 A | 42 B | 43 C | 44 D | 45 E | 46 F | 47 G | 27 | | 48 H | 49 I | 4a J | 4b K | 4c L | 4d M | 4e N | 4f O | 28 | | 50 P | 51 Q | 52 R | 53 S | 54 T | 55 U | 56 V | 57 W | 29 | | 58 X | 59 Y | 5a Z | 5b [ | 5c \ | 5d ] | 5e ^ | 5f _ | 30 | | 60 ` | 61 a | 62 b | 63 c | 64 d | 65 e | 66 f | 67 g | 31 | | 68 h | 69 i | 6a j | 6b k | 6c l | 6d m | 6e n | 6f o | 32 | | 70 p | 71 q | 72 r | 73 s | 74 t | 75 u | 76 v | 77 w | 33 | | 78 x | 79 y | 7a z | 7b { | 7c | | 7d } | 7e ~ | 7f del| 34 | 35 | -------------------------------------------------------------------------------- /tests/test_vm.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import unittest2, ../faststreams 4 | 5 | proc readAll(s: InputStream): string = 6 | while s.readable: 7 | result.add s.read.char 8 | 9 | suite "Inputs": 10 | dualTest "readableNow": 11 | let s = memoryInput("abc") 12 | check readableNow(s) 13 | discard readAll(s) 14 | check not readableNow(s) 15 | 16 | dualTest "totalUnconsumedBytes": 17 | let s = memoryInput("abc") 18 | check totalUnconsumedBytes(s) == 3 19 | discard s.read() 20 | check totalUnconsumedBytes(s) == 2 21 | discard s.read() 22 | check totalUnconsumedBytes(s) == 1 23 | discard s.read() 24 | check totalUnconsumedBytes(s) == 0 25 | 26 | dualTest "len": 27 | let s = memoryInput("abc") 28 | check s.len == some 3.Natural 29 | discard s.read() 30 | check s.len == some 2.Natural 31 | discard s.read() 32 | check s.len == some 1.Natural 33 | discard s.read() 34 | check s.len == some 0.Natural 35 | 36 | dualTest "readable": 37 | let text = "abc" 38 | let s = memoryInput(text) 39 | for _ in 0 ..< text.len: 40 | check readable(s) 41 | discard s.read() 42 | check(not readable(s)) 43 | 44 | dualTest "readable N": 45 | let s = memoryInput("abc") 46 | check readable(s, 0) 47 | check readable(s, 1) 48 | check readable(s, 2) 49 | check readable(s, 3) 50 | check(not readable(s, 4)) 51 | 52 | dualTest "peek": 53 | let text = "abc" 54 | let s = memoryInput(text) 55 | for i in 0 ..< text.len: 56 | check s.peek() == text[i].byte 57 | discard s.read() 58 | 59 | dualTest "read": 60 | let text = "abc" 61 | let s = memoryInput(text) 62 | for i in 0 ..< text.len: 63 | check s.read() == text[i].byte 64 | 65 | dualTest "peekAt": 66 | let text = "abc" 67 | let s = memoryInput(text) 68 | check s.peekAt(0) == 'a'.byte 69 | check s.peekAt(1) == 'b'.byte 70 | check s.peekAt(2) == 'c'.byte 71 | 72 | dualTest "advance": 73 | let text = "abc" 74 | let s = memoryInput(text) 75 | for i in 0 ..< text.len: 76 | check s.peek() == text[i].byte 77 | advance(s) 78 | 79 | dualTest "readIntoEx": 80 | let text = "abc" 81 | let s = memoryInput(text) 82 | var ss = newSeq[byte](2) 83 | check readIntoEx(s, ss) == 2 84 | check ss == toOpenArrayByte("ab", 0, 1) 85 | check readIntoEx(s, ss) == 1 86 | check ss[0] == 'c'.byte 87 | 88 | dualTest "readInto": 89 | let text = "abc" 90 | let s = memoryInput(text) 91 | var ss = newSeq[byte](2) 92 | check readInto(s, ss) 93 | check ss == toOpenArrayByte("ab", 0, 1) 94 | check(not readInto(s, ss)) 95 | check ss[0] == 'c'.byte 96 | 97 | dualTest "pos": 98 | let text = "abc" 99 | let s = memoryInput(text) 100 | for i in 0 ..< text.len: 101 | check s.pos() == i 102 | discard s.read() 103 | 104 | dualTest "close": 105 | let s = memoryInput("abc") 106 | check readable(s) 107 | s.close() 108 | check(not readable(s)) 109 | 110 | dualTest "unsafe read": 111 | let text = "abc" 112 | let s = unsafeMemoryInput(text) 113 | for i in 0 ..< text.len: 114 | check s.read() == text[i].byte 115 | 116 | suite "Outputs": 117 | dualTest "write byte": 118 | let s = memoryOutput() 119 | s.write('a'.byte) 120 | check s.getOutput() == @['a'.byte] 121 | 122 | dualTest "write char": 123 | let s = memoryOutput() 124 | s.write('a') 125 | check s.getOutput(string) == "a" 126 | 127 | dualTest "write byte seq": 128 | let s = memoryOutput() 129 | s.write(@['a'.byte, 'b'.byte]) 130 | check s.getOutput() == @['a'.byte, 'b'.byte] 131 | 132 | dualTest "write string": 133 | let s = memoryOutput() 134 | s.write("ab") 135 | check s.getOutput(string) == "ab" 136 | -------------------------------------------------------------------------------- /tests/base64.nim: -------------------------------------------------------------------------------- 1 | import 2 | ../faststreams 3 | 4 | template cbBase(a, b): untyped = [ 5 | 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 6 | 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 7 | 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 8 | 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 9 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', a, b] 10 | 11 | const 12 | cb64 = cbBase('+', '/') 13 | invalidChar = 255 14 | paddingByte = byte('=') 15 | 16 | template encodeSize(size: int): int = (size * 4 div 3) + 6 17 | 18 | import 19 | ../faststreams/buffers 20 | 21 | proc base64encode*(i: InputStream, o: OutputStream) {.fsMultiSync.} = 22 | var 23 | n: uint32 24 | b: uint32 25 | 26 | template inputByte(exp: untyped) = 27 | b = uint32(i.read) 28 | n = exp 29 | 30 | template outputChar(x: typed) = 31 | o.write cb64[x and 63] 32 | 33 | let inputLen = i.len 34 | if inputLen.isSome: 35 | o.ensureRunway encodeSize(inputLen.get) 36 | 37 | while i.readable(3): 38 | inputByte(b shl 16) 39 | inputByte(n or b shl 8) 40 | inputByte(n or b shl 0) 41 | outputChar(n shr 18) 42 | outputChar(n shr 12) 43 | outputChar(n shr 6) 44 | outputChar(n shr 0) 45 | 46 | if i.readable: 47 | inputByte(b shl 16) 48 | if i.readable: 49 | inputByte(n or b shl 8) 50 | outputChar(n shr 18) 51 | outputChar(n shr 12) 52 | outputChar(n shr 6) 53 | o.write paddingByte 54 | else: 55 | outputChar(n shr 18) 56 | outputChar(n shr 12) 57 | o.write paddingByte 58 | o.write paddingByte 59 | 60 | close o 61 | 62 | proc initDecodeTable*(): array[256, char] = 63 | # computes a decode table at compile time 64 | for i in 0 ..< 256: 65 | let ch = char(i) 66 | var code = invalidChar 67 | if ch >= 'A' and ch <= 'Z': code = i - 0x00000041 68 | if ch >= 'a' and ch <= 'z': code = i - 0x00000047 69 | if ch >= '0' and ch <= '9': code = i + 0x00000004 70 | if ch == '+' or ch == '-': code = 0x0000003E 71 | if ch == '/' or ch == '_': code = 0x0000003F 72 | result[i] = char(code) 73 | 74 | const 75 | decodeTable = initDecodeTable() 76 | 77 | proc base64decode*(i: InputStream, o: OutputStream) {.fsMultiSync.} = 78 | proc decodeSize(size: int): int = 79 | return (size * 3 div 4) + 6 80 | 81 | proc raiseInvalidChar(c: byte, pos: int) {.noreturn.} = 82 | raise newException(ValueError, 83 | "Invalid base64 format character `" & char(c) & "` at location " & $pos & ".") 84 | 85 | template inputChar(x: untyped) = 86 | let c = i.read() 87 | let x = int decodeTable[c] 88 | if x == invalidChar: 89 | raiseInvalidChar(c, i.pos - 1) 90 | 91 | template outputChar(x: untyped) = 92 | o.write char(x and 255) 93 | 94 | let inputLen = i.len 95 | if inputLen.isSome: 96 | o.ensureRunway decodeSize(inputLen.get) 97 | 98 | # hot loop: read 4 characters at at time 99 | while i.readable(8): 100 | inputChar(a) 101 | inputChar(b) 102 | inputChar(c) 103 | inputChar(d) 104 | outputChar(a shl 2 or b shr 4) 105 | outputChar(b shl 4 or c shr 2) 106 | outputChar(c shl 6 or d shr 0) 107 | 108 | if i.readable(4): 109 | inputChar(a) 110 | inputChar(b) 111 | outputChar(a shl 2 or b shr 4) 112 | 113 | if i.peek == paddingByte: 114 | let next = i.peekAt(1) 115 | if next != paddingByte: 116 | raiseInvalidChar(next, i.pos + 1) 117 | else: 118 | inputChar(c) 119 | outputChar(b shl 4 or c shr 2) 120 | if i.peek != paddingByte: 121 | inputChar(d) 122 | outputChar(c shl 6 or d shr 0) 123 | elif i.readable: 124 | raise newException(ValueError, "The input stream has insufficient number of bytes for base64 decoding") 125 | 126 | close o 127 | 128 | -------------------------------------------------------------------------------- /faststreams/asynctools_adapters.nim: -------------------------------------------------------------------------------- 1 | import 2 | asynctools/asyncpipe, 3 | inputs, outputs, buffers, multisync, async_backend 4 | 5 | when (not fsAsyncSupport): 6 | {.fatal: "`-d:async_backend` has be to set".} 7 | 8 | export 9 | inputs, outputs, asyncpipe, fsMultiSync 10 | 11 | {.pragma: iocall, nimcall, gcsafe, raises: [IOError].} 12 | 13 | type 14 | AsyncPipeInput* = ref object of InputStream 15 | pipe: AsyncPipe 16 | allowWaitFor: bool 17 | 18 | AsyncPipeOutput* = ref object of OutputStream 19 | pipe: AsyncPipe 20 | allowWaitFor: bool 21 | 22 | const 23 | readingErrMsg = "Failed to read from AsyncPipe" 24 | writingErrMsg = "Failed to write to AsyncPipe" 25 | closingErrMsg = "Failed to close AsyncPipe" 26 | writeIncompleteErrMsg = "Failed to write all bytes to AsyncPipe" 27 | 28 | proc closeAsyncPipe(pipe: AsyncPipe) 29 | {.raises: [IOError].} = 30 | fsTranslateErrors closingErrMsg: 31 | close pipe 32 | 33 | proc readOnce(s: AsyncPipeInput, 34 | dst: pointer, dstLen: Natural): Future[Natural] {.async.} = 35 | fsTranslateErrors readingErrMsg: 36 | return implementSingleRead(s.buffers, dst, dstLen, ReadFlags {}, 37 | readStartAddr, readLen): 38 | await s.pipe.readInto(readStartAddr, readLen) 39 | 40 | proc write(s: AsyncPipeOutput, src: pointer, srcLen: Natural) {.async.} = 41 | fsTranslateErrors writeIncompleteErrMsg: 42 | implementWrites(s.buffers, src, srcLen, "AsyncPipe", 43 | writeStartAddr, writeLen): 44 | await s.pipe.write(writeStartAddr, writeLen) 45 | 46 | # TODO: Use the Raising type here 47 | const asyncPipeInputVTable = InputStreamVTable( 48 | readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural 49 | {.iocall.} = 50 | fsTranslateErrors "Unexpected exception from asyncdispatch": 51 | var cs = AsyncPipeInput(s) 52 | fsAssert cs.allowWaitFor 53 | return waitFor readOnce(cs, dst, dstLen) 54 | , 55 | readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural] 56 | {.iocall.} = 57 | fsTranslateErrors "Unexpected exception from merely forwarding a future": 58 | return readOnce(AsyncPipeInput s, dst, dstLen) 59 | , 60 | closeSync: proc (s: InputStream) 61 | {.iocall.} = 62 | closeAsyncPipe AsyncPipeInput(s).pipe 63 | , 64 | closeAsync: proc (s: InputStream): Future[void] 65 | {.iocall.} = 66 | closeAsyncPipe AsyncPipeInput(s).pipe 67 | ) 68 | 69 | func asyncPipeInput*(pipe: AsyncPipe, 70 | pageSize = defaultPageSize, 71 | allowWaitFor = false): AsyncInputStream = 72 | AsyncInputStream AsyncPipeInput( 73 | vtable: vtableAddr asyncPipeInputVTable, 74 | buffers: initPageBuffers(pageSize), 75 | pipe: pipe, 76 | allowWaitFor: allowWaitFor) 77 | 78 | const asyncPipeOutputVTable = OutputStreamVTable( 79 | writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural) 80 | {.iocall.} = 81 | fsTranslateErrors "Unexpected exception from asyncdispatch": 82 | var cs = AsyncPipeOutput(s) 83 | fsAssert cs.allowWaitFor 84 | waitFor write(cs, src, srcLen) 85 | , 86 | writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void] 87 | {.iocall.} = 88 | fsTranslateErrors "Unexpected exception from merely forwarding a future": 89 | return write(AsyncPipeOutput s, src, srcLen) 90 | , 91 | closeSync: proc (s: OutputStream) 92 | {.iocall.} = 93 | closeAsyncPipe AsyncPipeOutput(s).pipe 94 | , 95 | closeAsync: proc (s: OutputStream): Future[void] 96 | {.iocall.} = 97 | closeAsyncPipe AsyncPipeOutput(s).pipe 98 | ) 99 | 100 | func asyncPipeOutput*(pipe: AsyncPipe, 101 | pageSize = defaultPageSize, 102 | allowWaitFor = false): AsyncOutputStream = 103 | AsyncOutputStream AsyncPipeOutput( 104 | vtable: vtableAddr(asyncPipeOutputVTable), 105 | buffers: initPageBuffers(pageSize), 106 | pipe: pipe, 107 | allowWaitFor: allowWaitFor) 108 | 109 | -------------------------------------------------------------------------------- /faststreams/chronos_adapters.nim: -------------------------------------------------------------------------------- 1 | import 2 | chronos, 3 | inputs, outputs, buffers, multisync 4 | 5 | export 6 | chronos, fsMultiSync 7 | 8 | {.pragma: iocall, nimcall, gcsafe, raises: [IOError].} 9 | 10 | type 11 | ChronosInputStream* = ref object of InputStream 12 | transport: StreamTransport 13 | allowWaitFor: bool 14 | 15 | ChronosOutputStream* = ref object of OutputStream 16 | transport: StreamTransport 17 | allowWaitFor: bool 18 | 19 | const 20 | readingErrMsg = "Failed to read from Chronos transport" 21 | writingErrMsg = "Failed to write to Chronos transport" 22 | closingErrMsg = "Failed to close Chronos transport" 23 | writeIncompleteErrMsg = "Failed to write all bytes to Chronos transport" 24 | 25 | proc chronosCloseWait(t: StreamTransport) 26 | {.async, raises: [IOError].} = 27 | fsTranslateErrors closingErrMsg: 28 | await t.closeWait() 29 | 30 | proc chronosReadOnce(s: ChronosInputStream, 31 | dst: pointer, dstLen: Natural): Future[Natural] {.async.} = 32 | fsTranslateErrors readingErrMsg: 33 | return implementSingleRead(s.buffers, dst, dstLen, ReadFlags {}, 34 | readStartAddr, readLen): 35 | await s.transport.readOnce(readStartAddr, readLen) 36 | 37 | proc chronosWrites(s: ChronosOutputStream, src: pointer, srcLen: Natural) {.async.} = 38 | fsTranslateErrors writeIncompleteErrMsg: 39 | implementWrites(s.buffers, src, srcLen, "StreamTransport", 40 | writeStartAddr, writeLen): 41 | await s.transport.write(writeStartAddr, writeLen) 42 | 43 | # TODO: Use the Raising type here 44 | const chronosInputVTable = InputStreamVTable( 45 | readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural 46 | {.iocall.} = 47 | fsTranslateErrors "Unexpected exception from Chronos async macro": 48 | var cs = ChronosInputStream(s) 49 | fsAssert cs.allowWaitFor 50 | return waitFor chronosReadOnce(cs, dst, dstLen) 51 | , 52 | readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural] 53 | {.iocall.} = 54 | fsTranslateErrors "Unexpected exception from merely forwarding a future": 55 | return chronosReadOnce(ChronosInputStream s, dst, dstLen) 56 | , 57 | closeSync: proc (s: InputStream) 58 | {.iocall.} = 59 | fsTranslateErrors closingErrMsg: 60 | s.closeFut = ChronosInputStream(s).transport.close() 61 | , 62 | closeAsync: proc (s: InputStream): Future[void] 63 | {.iocall.} = 64 | chronosCloseWait ChronosInputStream(s).transport 65 | ) 66 | 67 | func chronosInput*(s: StreamTransport, 68 | pageSize = defaultPageSize, 69 | allowWaitFor = false): AsyncInputStream = 70 | AsyncInputStream ChronosInputStream( 71 | vtable: vtableAddr chronosInputVTable, 72 | buffers: initPageBuffers(pageSize), 73 | transport: s, 74 | allowWaitFor: allowWaitFor) 75 | 76 | const chronosOutputVTable = OutputStreamVTable( 77 | writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural) 78 | {.iocall.} = 79 | var cs = ChronosOutputStream(s) 80 | fsAssert cs.allowWaitFor 81 | waitFor chronosWrites(cs, src, srcLen) 82 | , 83 | writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void] 84 | {.iocall.} = 85 | chronosWrites(ChronosOutputStream s, src, srcLen) 86 | , 87 | closeSync: proc (s: OutputStream) 88 | {.iocall.} = 89 | fsTranslateErrors closingErrMsg: 90 | s.closeFut = close ChronosOutputStream(s).transport 91 | , 92 | closeAsync: proc (s: OutputStream): Future[void] 93 | {.iocall.} = 94 | chronosCloseWait ChronosOutputStream(s).transport 95 | ) 96 | 97 | func chronosOutput*(s: StreamTransport, 98 | pageSize = defaultPageSize, 99 | allowWaitFor = false): AsyncOutputStream = 100 | AsyncOutputStream ChronosOutputStream( 101 | vtable: vtableAddr(chronosOutputVTable), 102 | buffers: initPageBuffers(pageSize), 103 | transport: s, 104 | allowWaitFor: allowWaitFor) 105 | 106 | -------------------------------------------------------------------------------- /tests/test_pipelines.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import 4 | unittest2, 5 | 6 | # FastStreams modules: 7 | ../faststreams/[pipelines, multisync] 8 | 9 | when fsAsyncSupport: 10 | import 11 | # Std lib: 12 | std/[strutils, random, base64, terminal], 13 | # FastStreams modules: 14 | ../faststreams/[pipelines, multisync], 15 | # Testing modules: 16 | ./base64 as fsBase64, 17 | chronos/unittest2/asynctests 18 | 19 | include system/timers 20 | 21 | type 22 | TestTimes = object 23 | fsPipeline: Nanos 24 | fsAsyncPipeline: Nanos 25 | stdFunctionCalls: Nanos 26 | 27 | proc upcaseAllCharacters(i: InputStream, o: OutputStream) {.fsMultiSync.} = 28 | let inputLen = i.len 29 | if inputLen.isSome: 30 | o.ensureRunway inputLen.get 31 | 32 | while i.readable: 33 | o.write toUpperAscii(i.read.char) 34 | 35 | close o 36 | 37 | proc printTimes(t: TestTimes) = 38 | styledEcho " cpu time [FS Sync ]: ", styleBright, $t.fsPipeline, "ms" 39 | styledEcho " cpu time [FS Async ]: ", styleBright, $t.fsAsyncPipeline, "ms" 40 | styledEcho " cpu time [Std Lib ]: ", styleBright, $t.stdFunctionCalls, "ms" 41 | 42 | template timeit(timerVar: var Nanos, code: untyped) = 43 | let t0 = getTicks() 44 | code 45 | timerVar = int64(getTicks() - t0) div 1000000 46 | 47 | proc getOutput(sp: AsyncInputStream, T: type string): Future[string] {.async.} = 48 | # this proc is a quick hack to let the test pass 49 | # do not use it in production code 50 | let size = sp.totalUnconsumedBytes() 51 | if size > 0: 52 | var data = newSeq[byte](size) 53 | discard sp.readInto(data) 54 | result = cast[string](data) 55 | 56 | suite "pipelines": 57 | const loremIpsum = """ 58 | Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod 59 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim 60 | veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex 61 | ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate 62 | velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat 63 | cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id 64 | est laborum. 65 | 66 | """ 67 | 68 | test "upper-case/base64 pipeline benchmark": 69 | var 70 | times: TestTimes 71 | stdRes: string 72 | fsRes: string 73 | fsAsyncRes: string 74 | 75 | let inputText = loremIpsum.repeat(5000) 76 | 77 | timeit times.stdFunctionCalls: 78 | stdRes = base64.decode(base64.encode(toUpperAscii(inputText))) 79 | 80 | timeit times.fsPipeline: 81 | fsRes = executePipeline(unsafeMemoryInput(inputText), 82 | upcaseAllCharacters, 83 | base64encode, 84 | base64decode, 85 | getOutput string) 86 | 87 | timeit times.fsAsyncPipeline: 88 | fsAsyncRes = waitFor executePipeline(Async unsafeMemoryInput(inputText), 89 | upcaseAllCharacters, 90 | base64encode, 91 | base64decode, 92 | getOutput string) 93 | 94 | check fsAsyncRes == stdRes 95 | check fsRes == stdRes 96 | 97 | printTimes times 98 | 99 | asyncTest "upper-case/base64 async pipeline": 100 | let pipe = asyncPipe() 101 | let inputText = repeat(loremIpsum, 100) 102 | 103 | proc pipeFeeder(s: AsyncOutputStream) {.gcsafe, async.} = 104 | randomize 1234 105 | var pos = 0 106 | 107 | while pos != inputText.len: 108 | let bytesToWrite = rand(15) 109 | 110 | if bytesToWrite == 0: 111 | s.write inputText[pos] 112 | inc pos 113 | else: 114 | let endPos = min(pos + bytesToWrite, inputText.len) 115 | s.writeAndWait inputText[pos ..< endPos] 116 | pos = endPos 117 | 118 | let sleep = rand(50) - 45 119 | if sleep > 0: 120 | await sleepAsync(sleep.milliseconds) 121 | 122 | close s 123 | 124 | asyncCheck pipeFeeder(pipe.initWriter) 125 | 126 | let f = executePipeline(pipe.initReader, 127 | upcaseAllCharacters, 128 | base64encode, 129 | base64decode, 130 | getOutput string) 131 | 132 | let fsAsyncres = await f 133 | 134 | check fsAsyncres == toUpperAscii(inputText) 135 | else: 136 | test "pipelines": 137 | skip 138 | -------------------------------------------------------------------------------- /faststreams/textio.nim: -------------------------------------------------------------------------------- 1 | import 2 | stew/ptrops, 3 | inputs, outputs, buffers, async_backend, multisync 4 | 5 | from std/strutils import Digits, Newlines 6 | 7 | when (NimMajor, NimMinor) < (2, 0): 8 | import system/formatfloat 9 | else: 10 | import std/formatfloat 11 | 12 | 13 | template matchingIntType(T: type int64): type = uint64 14 | template matchingIntType(T: type int32): type = uint32 15 | template matchingIntType(T: type uint64): type = int64 16 | template matchingIntType(T: type uint32): type = int32 17 | 18 | # To reduce the produce code bloat, we will compile the integer 19 | # handling functions only for the native type of the platform. 20 | # Smaller int types will be automatically promoted to the native. 21 | # On a 32-bit platforms, we'll also compile support for 64-bit types. 22 | when sizeof(int) == sizeof(int64): 23 | type 24 | CompiledIntTypes = int64 25 | CompiledUIntTypes = uint64 26 | else: 27 | type 28 | CompiledIntTypes = int32|int64 29 | CompiledUIntTypes = uint32|uint64 30 | 31 | # The following code implements writing numbers to a stream without going 32 | # through Nim's `$` operator which will allocate memory. 33 | # It's based on some speed comparisons of different methods presented here: 34 | # http://www.zverovich.net/2013/09/07/integer-to-string-conversion-in-cplusplus.html 35 | 36 | const 37 | digitsTable = block: 38 | var s = "" 39 | for i in 0..99: 40 | if i < 10: s.add '0' 41 | s.add $i 42 | s 43 | 44 | maxLen = ($BiggestInt.high).len + 4 # null terminator, sign 45 | 46 | proc writeText*(s: OutputStream, x: CompiledUIntTypes) = 47 | var 48 | num: array[maxLen, char] 49 | pos = num.len 50 | 51 | template writeByteInReverse(c: char) = 52 | dec pos 53 | num[pos] = c 54 | 55 | var val = x 56 | while val > 99: 57 | # Integer division is slow so do it for a group of two digits instead 58 | # of for every digit. The idea comes from the talk by Alexandrescu 59 | # "Three Optimization Tips for C++". 60 | let base100digitIdx = (val mod 100) * 2 61 | val = val div 100 62 | 63 | writeByteInReverse digitsTable[base100digitIdx + 1] 64 | writeByteInReverse digitsTable[base100digitIdx] 65 | 66 | when true: 67 | if val < 10: 68 | writeByteInReverse char(ord('0') + val) 69 | else: 70 | let base100digitIdx = val * 2 71 | writeByteInReverse digitsTable[base100digitIdx + 1] 72 | writeByteInReverse digitsTable[base100digitIdx] 73 | else: 74 | # Alternative idea: 75 | # We now know enough to write digits directly to the stream. 76 | if val < 10: 77 | write s, byte(ord('\0') + val) 78 | else: 79 | let base100digitIdx = val * 2 80 | write s, digitsTable[base100digitIdx] 81 | write s, digitsTable[base100digitIdx + 1] 82 | 83 | write s, num.toOpenArray(pos, static(num.len - 1)) 84 | 85 | proc writeText*(s: OutputStream, x: CompiledIntTypes) = 86 | type MatchingUInt = matchingIntType typeof(x) 87 | 88 | if x < 0: 89 | s.write '-' 90 | # The `0 - x` trick below takes care of one corner case: 91 | # How do we get the abs value of low(int)? 92 | # The naive `-x` triggers an overflow, because low(int8) 93 | # is -128, while high(int8) is 127. 94 | writeText(s, MatchingUInt(0) - MatchingUInt(x)) 95 | else: 96 | writeText(s, MatchingUInt(x)) 97 | 98 | template writeText*(s: OutputStream, str: string) = 99 | write s, str 100 | 101 | template writeText*(s: OutputStream, val: auto) = 102 | # TODO https://github.com/nim-lang/Nim/issues/25166 103 | when val is SomeFloat: 104 | var buffer: array[65, char] 105 | let blen = writeFloatToBufferRoundtrip(buffer, val) 106 | write s, buffer.toOpenArray(0, blen - 1) 107 | else: 108 | write s, $val 109 | 110 | proc writeHex*(s: OutputStream, bytes: openArray[byte]) = 111 | const hexChars = "0123456789abcdef" 112 | 113 | for b in bytes: 114 | s.write hexChars[int b shr 4 and 0xF] 115 | s.write hexChars[int b and 0xF] 116 | 117 | proc writeHex*(s: OutputStream, chars: openArray[char]) = 118 | writeHex s, chars.toOpenArrayByte(0, chars.high()) 119 | 120 | proc readLine*(s: InputStream, keepEol = false): string {.fsMultiSync.} = 121 | fsAssert readableNow(s) 122 | 123 | while s.readable: 124 | let c = s.peek.char 125 | if c in Newlines: 126 | if keepEol: 127 | result.add c 128 | if c == '\r' and s.readable and s.peek.char == '\n': 129 | result.add s.read.char 130 | else: 131 | advance s 132 | if c == '\r' and s.readable and s.peek.char == '\n': 133 | advance s 134 | return 135 | 136 | result.add s.read.char 137 | 138 | proc readUntil*(s: InputStream, 139 | sep: openArray[char]): Option[string] = 140 | fsAssert readableNow(s) 141 | var res = "" 142 | while s.readable(sep.len): 143 | if s.lookAheadMatch(sep.toOpenArrayByte(0, sep.high())): 144 | return some(res) 145 | res.add s.read.char 146 | 147 | template nextLine*(sp: InputStream, keepEol = false): Option[string] = 148 | let s = sp 149 | if s.readable: 150 | some readLine(s, keepEol) 151 | else: 152 | none string 153 | 154 | iterator lines*(s: InputStream, keepEol = false): string = 155 | while s.readable: 156 | yield readLine(s, keepEol) 157 | 158 | proc readUnsignedInt*(s: InputStream, T: type[CompiledUIntTypes]): T = 159 | fsAssert s.readable and s.peek.char in Digits 160 | 161 | template eatDigitAndPeek: char = 162 | advance s 163 | if not s.readable: return 164 | s.peek.char 165 | 166 | var c = s.peek.char 167 | result = T(ord(c) - ord('0')) 168 | c = eatDigitAndPeek() 169 | while c.isDigit: 170 | # TODO: How do we handle the possible overflow here? 171 | result = result * 10 + T(ord(c) - ord('0')) 172 | c = eatDigitAndPeek() 173 | 174 | template readUnsignedInt*(s: InputStream): uint = 175 | readUnsignedInt(s, uint) 176 | 177 | proc readSignedInt*(s: InputStream, T: type[CompiledIntTypes]): Option[T] = 178 | if s.readable: 179 | let maybeSign = s.read.peek 180 | if maybeSign in {'-', '+'}: 181 | if not s.readable(2) or s.peekAt(1).char notin Digits: 182 | return 183 | else: 184 | advance s 185 | elif maybeSign notin Digits: 186 | return 187 | 188 | type UIntType = matchingIntType T 189 | let uintVal = readUnsignedInt(s, UIntType) 190 | 191 | if maybeSign == '-': 192 | if uintVal > UIntType(max(T)) + 1: 193 | return # Overflow. We've consumed part of the stream though. 194 | # TODO: Should we rewind it to a previous state? 195 | return some cast[T](UIntType(0) - uintVal) 196 | else: 197 | if uintVal > UIntType(max(T)): 198 | return # Overflow. We've consumed part of the stream though. 199 | # TODO: Should we rewind it to a previous state? 200 | return some T(uintVal) 201 | 202 | -------------------------------------------------------------------------------- /tests/test_inputs.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import 4 | os, unittest2, strutils, sequtils, random, 5 | ../faststreams, ../faststreams/textio 6 | 7 | setCurrentDir currentSourcePath.parentDir 8 | 9 | proc str(bytes: openArray[byte]): string = 10 | result = newStringOfCap(bytes.len) 11 | for b in items(bytes): 12 | result.add b.char 13 | 14 | proc countLines(s: InputStream): Natural = 15 | for s in lines(s): 16 | inc result 17 | 18 | proc readAll(s: InputStream): seq[byte] = 19 | while s.readable: 20 | result.add s.read 21 | 22 | const 23 | asciiTableFile = "files" / "ascii_table.txt" 24 | asciiTableContents = slurp(asciiTableFile) 25 | 26 | suite "input stream": 27 | template emptyInputTests(suiteName, setupCode: untyped) = 28 | suite suiteName & " empty inputs": 29 | setup setupCode 30 | 31 | test "input is not readable with read": 32 | check not input.readable 33 | when not defined(danger): 34 | expect Defect: 35 | echo "This read should not complete: ", input.read 36 | 37 | test "input is not readable with read(n)": 38 | check not input.readable(10) 39 | when not defined(danger): 40 | expect Defect: 41 | echo "This read should not complete: ", input.read(10) 42 | 43 | test "next returns none": 44 | check input.next.isNone 45 | 46 | test "input can be closed": 47 | input.close() 48 | 49 | emptyInputTests "memoryInput": 50 | var input = InputStream() 51 | 52 | emptyInputTests "unsafeMemoryInput": 53 | var str = "" 54 | var input = unsafeMemoryInput(str) 55 | 56 | emptyInputTests "fileInput": 57 | var input = fileInput("files" / "empty_file") 58 | 59 | emptyInputTests "memFileInput": 60 | var input = memFileInput("files" / "empty_file") 61 | 62 | template asciiTableFileTest(name: string, body: untyped) = 63 | suite name: 64 | test name & " of ascii table with regular pageSize": 65 | var input {.inject.} = fileInput(asciiTableFile) 66 | try: 67 | body 68 | finally: 69 | close input 70 | 71 | test name & " of ascii table with pageSize = 10": 72 | var input {.inject.} = fileInput(asciiTableFile, pageSize = 10) 73 | try: 74 | body 75 | finally: 76 | close input 77 | 78 | test name & " of ascii table with pageSize = 1": 79 | var input {.inject.} = fileInput(asciiTableFile, pageSize = 1) 80 | try: 81 | body 82 | finally: 83 | close input 84 | 85 | # TODO: fileInput with offset 86 | # - in the middle of the 87 | # - right at the end of the file 88 | # - past the end of the file 89 | 90 | asciiTableFileTest "count lines": 91 | check countLines(input) == 34 92 | 93 | asciiTableFileTest "mixed read types": 94 | randomize(10000) 95 | 96 | var fileContents = "" 97 | 98 | while true: 99 | let r = rand(100) 100 | if r < 20: 101 | let readSize = 1 + rand(10) 102 | 103 | var buf = newSeq[byte](readSize) 104 | let bytesRead = input.readIntoEx(buf) 105 | fileContents.add buf.toOpenArray(0, bytesRead - 1).str 106 | 107 | if bytesRead < buf.len: 108 | break 109 | 110 | elif r < 50: 111 | let readSize = 6 + rand(10) 112 | 113 | if input.readable(readSize): 114 | fileContents.add input.read(readSize).str 115 | else: 116 | while input.readable: 117 | fileContents.add input.read.char 118 | break 119 | 120 | elif r < 60: 121 | # Test the ability to call readable() and read() multiple times from 122 | # the same scope. 123 | let readSize = 6 + rand(10) 124 | 125 | if input.readable(readSize): 126 | fileContents.add input.read(readSize).str 127 | else: 128 | while input.readable: 129 | fileContents.add input.read.char 130 | break 131 | 132 | if input.readable(readSize): 133 | fileContents.add input.read(readSize).str 134 | else: 135 | while input.readable: 136 | fileContents.add input.read.char 137 | break 138 | 139 | else: 140 | if input.readable: 141 | fileContents.add input.read.char 142 | 143 | # You can uncomment this to get earlier failure in the test: 144 | when false: 145 | require fileContents == asciiTableContents[0 ..< fileContents.len] 146 | 147 | check fileContents == asciiTableContents 148 | 149 | suite "misc": 150 | test "reading into empty buffer": 151 | var input = memoryInput([byte 1]) 152 | 153 | var buf: seq[byte] 154 | check: 155 | # Reading into empty should succeed for open stream 156 | input.readIntoEx(buf) == 0 157 | input.readInto(buf) 158 | 159 | buf.setLen(1) 160 | check: 161 | input.readIntoEx(buf) == 1 162 | input.readIntoEx(buf) == 0 163 | not input.readInto(buf) 164 | test "missing file input": 165 | const fileName = "there-is-no-such-faststreams-file" 166 | 167 | check not fileExists(fileName) 168 | expect CatchableError: discard fileInput(fileName) 169 | 170 | check not fileExists(fileName) 171 | expect CatchableError: discard memFileInput(fileName) 172 | 173 | check not fileExists(fileName) 174 | 175 | test "can close nil InputStream": 176 | var v: InputStream 177 | v.close() 178 | 179 | test "non-blocking reads": 180 | let s = fileInput(asciiTableFile, pageSize = 100) 181 | if s.readable(20): 182 | s.withReadableRange(20, r): 183 | check r.len.get == 20 184 | check r.totalUnconsumedBytes == 20 185 | check r.readAll.len == 20 186 | 187 | if s.readable(20): 188 | s.withReadableRange(20, r): 189 | check r.len.get == 20 190 | check r.totalUnconsumedBytes == 20 191 | check r.readAll.len == 20 192 | 193 | check s.readable 194 | 195 | if s.readable(160): 196 | s.withReadableRange(160, r): 197 | check r.len.get == 160 198 | check r.totalUnconsumedBytes == 160 199 | check r.readAll.len == 160 200 | 201 | check s.readable 202 | 203 | if s.readable(20): 204 | s.withReadableRange(20, r): 205 | check r.len.get == 20 206 | check r.totalUnconsumedBytes == 20 207 | check r.readAll.len == 20 208 | 209 | check s.readable 210 | 211 | template drainTest(name: string, setup: untyped) = 212 | test "draining readable ranges " & name: 213 | setup 214 | check: 215 | s.readable(20) 216 | 217 | s.withReadableRange(20, r): 218 | var tmp: array[100, byte] 219 | 220 | check: 221 | r.drainBuffersInto(addr tmp[0], tmp.len) == 20 222 | @tmp == repeat(byte 5, 20) & repeat(byte 0, 80) 223 | 224 | check: 225 | s.readable(80) 226 | 227 | var tmp: array[10, byte] 228 | check: 229 | s.drainBuffersInto(addr tmp[0], tmp.len) == 10 230 | 231 | @tmp == repeat(byte 5, 10) 232 | 233 | s.withReadableRange(50, r): 234 | var tmp: array[100, byte] 235 | check: 236 | r.drainBuffersInto(addr tmp[0], tmp.len) == 50 237 | @tmp == repeat(byte 5, 50) & repeat(byte 0, 50) 238 | 239 | check: 240 | s.readable() 241 | 242 | check: 243 | s.drainBuffersInto(addr tmp[0], tmp.len) == 10 244 | @tmp == repeat(byte 5, 10) 245 | 246 | check: 247 | s.drainBuffersInto(addr tmp[0], tmp.len) == 10 248 | @tmp == repeat(byte 5, 10) 249 | 250 | check: 251 | not s.readable() 252 | 253 | drainTest "unsafeMemoryInput": 254 | const data = repeat(byte 5, 100) 255 | let s = unsafeMemoryInput(data) 256 | 257 | drainTest "memoryInput": 258 | var data = repeat(byte 5, 100) 259 | let s = memoryInput(data) 260 | 261 | test "simple": 262 | var input = repeat("1234 5678 90AB CDEF\n", 1000) 263 | var stream = unsafeMemoryInput(input) 264 | 265 | check: 266 | (stream.read(4) == "1234".toOpenArrayByte(0, 3)) 267 | 268 | template posTest(name: string, setup: untyped) = 269 | test name: 270 | setup 271 | check input.readable 272 | check input.pos == 0 273 | discard input.read 274 | check input.pos == 1 275 | close input 276 | 277 | posTest "unsafeMemoryInput pos": 278 | var str = "hello" 279 | var input = unsafeMemoryInput(str) 280 | 281 | posTest "fileInput pos": 282 | var input = fileInput(asciiTableFile) 283 | 284 | posTest "memFileInput pos": 285 | var input = memFileInput(asciiTableFile) 286 | 287 | test "buffered peekAt": 288 | let s = fileInput(asciiTableFile, pageSize = 3) 289 | for i in 0..10: 290 | if s.readable(i + 1): 291 | check: 292 | s.peekAt(i) == byte asciiTableContents[i] 293 | -------------------------------------------------------------------------------- /LICENSE-APACHEv2: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Status Research & Development GmbH 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /tests/test_buffers.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import unittest2, ../faststreams/buffers 4 | import std/random 5 | 6 | const bytes256 = block: 7 | var v: array[256, byte] 8 | for i in 0 ..< 256: 9 | v[i] = byte i 10 | v 11 | 12 | proc consumeAll(buf: PageBuffers): seq[byte] = 13 | for page in buf.consumePages(): 14 | result.add page.data() 15 | 16 | # Note: these tests count `consume` calls as a way to make sure we don't waste 17 | # buffers - however, the number of consume calls is not part of the stable API 18 | # and therefore, tests may have to change in the future 19 | suite "PageBuffers": 20 | test "prepare/commit/consume": 21 | const pageSize = 8 22 | let buf = PageBuffers.init(pageSize) 23 | let input = bytes256[0 .. 31] 24 | 25 | # Write in one go 26 | buf.write(input) 27 | 28 | var res: seq[byte] 29 | 30 | var r = buf.consume() 31 | res.add r.data() 32 | check res == input 33 | 34 | # Buffer should now be empty 35 | check buf.consumable() == 0 36 | 37 | test "prepare/commit/consume with partial writes and reads": 38 | const pageSize = 8 39 | let buf = PageBuffers.init(pageSize) 40 | let input = bytes256[0 .. 15] # 16 bytes, 2 pages 41 | 42 | # Write in two steps 43 | var w1 = buf.prepare(8) 44 | w1.write(input[0 .. 7]) 45 | buf.commit(8) 46 | 47 | var w2 = buf.prepare(8) 48 | w2.write(input[8 .. 15]) 49 | buf.commit(8) 50 | 51 | # Consume part of first write 52 | var r = buf.consume(4) 53 | check @(r.data()) == input[0 .. 3] 54 | 55 | r = buf.consume() 56 | check @(r.data()) == input[4 .. 7] 57 | 58 | r = buf.consume() 59 | check @(r.data()) == input[8 .. 15] 60 | 61 | # Buffer should now be empty 62 | check buf.consumable() == 0 63 | 64 | test "reserve/commit/consume blocks until commit": 65 | const pageSize = 8 66 | let buf = PageBuffers.init(pageSize) 67 | let input = bytes256[0 .. 7] 68 | 69 | var w = buf.reserve(8) 70 | w.write(input) 71 | # Not committed yet, should not be readable 72 | check buf.consumable() == 0 73 | 74 | buf.commit(w) 75 | # Now it should be readable 76 | var r = buf.consume() 77 | check @(r.data()) == input 78 | 79 | test "reserve/prepare/commit/consume interleaved": 80 | const pageSize = 8 81 | let buf = PageBuffers.init(pageSize) 82 | let a = bytes256[0 .. 7] 83 | let b = bytes256[8 .. 15] 84 | 85 | var w1 = buf.reserve(8) 86 | w1.write(a) 87 | var w2 = buf.prepare(8) 88 | w2.write(b) 89 | buf.commit(w2) 90 | # Only b is committed, but a is reserved and not yet committed, so nothing is readable 91 | check buf.consumable() == 0 92 | 93 | buf.commit(w1) 94 | 95 | # Now both a and b should be readable, in order 96 | var r = buf.consume() 97 | check @(r.data()) == a 98 | r = buf.consume() 99 | check @(r.data()) == b 100 | check buf.consumable() == 0 101 | 102 | test "multiple small writes and reads, crossing page boundaries": 103 | const pageSize = 4 104 | let buf = PageBuffers.init(pageSize) 105 | let input = bytes256[0 .. 11] # 12 bytes, 3 pages 106 | 107 | # Write 3 times, 4 bytes each 108 | for i in 0 .. 2: 109 | var w = buf.prepare(4) 110 | w.write(input[(i * 4) ..< (i * 4 + 4)]) 111 | buf.commit(4) 112 | 113 | # Read 6 times, 2 bytes each 114 | var res: seq[byte] 115 | for i in 0 .. 5: 116 | var r = buf.consume(2) 117 | res.add r.data() 118 | check res == input 119 | 120 | # Buffer should now be empty 121 | check buf.consumable() == 0 122 | 123 | test "unconsume restores data": 124 | const pageSize = 8 125 | let buf = PageBuffers.init(pageSize) 126 | let input = bytes256[0 .. 7] 127 | 128 | var w = buf.prepare(8) 129 | w.write(input) 130 | buf.commit(8) 131 | 132 | var r = buf.consume(8) 133 | check @(r.data()) == input 134 | 135 | # Unconsume 4 bytes 136 | buf.unconsume(4) 137 | var r2 = buf.consume(4) 138 | check @(r2.data()) == input[4 .. 7] 139 | 140 | # Buffer should now be empty 141 | check buf.consumable() == 0 142 | 143 | test "prepare with more than page size allocates larger page": 144 | const pageSize = 8 145 | let buf = PageBuffers.init(pageSize) 146 | let input = bytes256[0 .. 15] # 16 bytes 147 | 148 | var w = buf.prepare(16) 149 | w.write(input) 150 | buf.commit(16) 151 | 152 | var r = buf.consume(16) 153 | check @(r.data()) == input 154 | 155 | test "reserve with more than page size allocates larger page": 156 | const pageSize = 8 157 | let buf = PageBuffers.init(pageSize) 158 | let input = bytes256[0 .. 15] # 16 bytes 159 | 160 | var w = buf.reserve(16) 161 | w.write(input) 162 | buf.commit(w) 163 | 164 | var r = buf.consume(16) 165 | check @(r.data()) == input 166 | 167 | test "mix of prepare, reserve, commit, and consume with random order": 168 | const pageSize = 8 169 | let buf = PageBuffers.init(pageSize) 170 | var expected: seq[byte] 171 | var written: seq[(string, seq[byte], PageSpan)] 172 | randomize(1000) 173 | 174 | # Randomly choose prepare or reserve, then commit in random order 175 | for i in 0 .. 3: 176 | let d = bytes256[(i * 8) ..< (i * 8 + 8)] 177 | if rand(1) == 0: 178 | var w = buf.prepare(8) 179 | w.write(d) 180 | written.add(("prepare", d, w)) 181 | buf.commit(8) 182 | else: 183 | var w = buf.reserve(8) 184 | w.write(d) 185 | written.add(("reserve", d, w)) 186 | 187 | # Commit in random order 188 | var idxs = @[0, 1, 2, 3] 189 | idxs.shuffle() 190 | for i in idxs: 191 | let (kind, _, w) = written[i] 192 | if kind == "reserve": 193 | buf.commit(w) 194 | 195 | # All data should be readable in original order 196 | for i in 0 .. 3: 197 | expected.add(bytes256[(i * 8) ..< (i * 8 + 8)]) 198 | check consumeAll(buf) == expected 199 | 200 | test "consumePages iterator yields all data and recycles last page": 201 | const pageSize = 8 202 | let buf = PageBuffers.init(pageSize) 203 | let input = bytes256[0 .. 23] # 24 bytes, 3 pages 204 | 205 | for i in 0 .. 2: 206 | var w = buf.prepare() 207 | w.write(input[i * 8 ..< ((i + 1) * 8)]) 208 | buf.commit(w) 209 | 210 | var seen = buf.consumeAll() 211 | 212 | check seen == input 213 | # After consuming, the buffer should have one recycled page 214 | check buf.capacity() == pageSize 215 | 216 | test "consumePageBuffers yields correct pointers and lengths": 217 | let buf = PageBuffers.init(8) 218 | let input = bytes256[0 .. 15] # 16 bytes, 2 pages 219 | 220 | var w = buf.prepare() 221 | w.write(input[0 .. 7]) 222 | buf.commit(w) 223 | w = buf.prepare() 224 | w.write(input[8 .. 15]) 225 | buf.commit(w) 226 | 227 | var seen: seq[byte] 228 | for (p, len) in buf.consumePageBuffers(): 229 | let s = cast[ptr UncheckedArray[byte]](p) 230 | for i in 0 ..< int(len): 231 | seen.add s[i] 232 | check seen == input 233 | 234 | test "commit less than prepared within a single page": 235 | const pageSize = 8 236 | let buf = PageBuffers.init(pageSize) 237 | let input = bytes256[0 .. 7] 238 | 239 | var w = buf.prepare(8) 240 | w.write(input) 241 | # Only commit 4 bytes 242 | buf.commit(4) 243 | 244 | var r = buf.consume() 245 | check @(r.data()) == input[0 .. 3] 246 | # The rest should not be available 247 | check buf.consumable() == 0 248 | 249 | # To write more, must call prepare again 250 | var w2 = buf.prepare(4) 251 | w2.write(input[4 .. 7]) 252 | buf.commit(4) 253 | r = buf.consume() 254 | check @(r.data()) == input[4 .. 7] 255 | check buf.consumable() == 0 256 | 257 | test "commit less than reserved, then reserve again in same page": 258 | const pageSize = 8 259 | let buf = PageBuffers.init(pageSize) 260 | let input = bytes256[0 .. 7] 261 | 262 | var w = buf.reserve(8) 263 | w.write(input[0 .. 3]) 264 | # Commit only 4 bytes 265 | buf.commit(w) 266 | # The committed reserve should be available also 267 | check buf.consumable() == 4 268 | 269 | var w2 = buf.reserve(4) 270 | w2.write(input[4 .. 7]) 271 | buf.commit(w2) 272 | 273 | var r = buf.consume() 274 | check @(r.data()) == input[0 .. 3] 275 | r = buf.consume() 276 | check @(r.data()) == input[4 .. 7] 277 | check buf.consumable() == 0 278 | 279 | test "commit less than prepared, crossing page boundary": 280 | const pageSize = 8 281 | let buf = PageBuffers.init(pageSize) 282 | let input = bytes256[0 .. 15] # 16 bytes, 2 writes 283 | 284 | var w = buf.prepare(16) 285 | w.write(input) 286 | buf.commit(10) 287 | 288 | var r = buf.consume() 289 | check @(r.data()) == input[0 .. 9] 290 | # The rest should not be available 291 | check buf.consumable() == 0 292 | 293 | # To write more, must call prepare again 294 | var w2 = buf.prepare(6) 295 | w2.write(input[10 .. 15]) 296 | buf.commit(6) 297 | r = buf.consume() 298 | check @(r.data()) == input[10 .. 15] 299 | check buf.consumable() == 0 300 | 301 | test "commit less than reserved, crossing page boundary, then reserve again": 302 | const pageSize = 8 303 | let buf = PageBuffers.init(pageSize) 304 | let input = bytes256[0 .. 15] # 16 bytes 305 | 306 | var w = buf.reserve(16) 307 | w.write(input[0 .. 9]) 308 | # Commit only first 10 bytes 309 | buf.commit(w) 310 | check buf.consumable() == 10 311 | 312 | # Must reserve again 313 | var w2 = buf.reserve(6) 314 | w2.write(input[10 .. 15]) 315 | buf.commit(w2) 316 | var r = buf.consume() 317 | check @(r.data()) == input[0 .. 9] 318 | r = buf.consume() 319 | check @(r.data()) == input[10 .. 15] 320 | check buf.consumable() == 0 321 | 322 | test "commit less than prepared/reserved, then interleave with new writes": 323 | const pageSize = 8 324 | let buf = PageBuffers.init(pageSize) 325 | let a = bytes256[0 .. 7] 326 | let b = bytes256[8 .. 15] 327 | 328 | # Prepare 8, commit 4 329 | var w1 = buf.prepare(8) 330 | w1.write(a) 331 | buf.commit(4) 332 | 333 | # Must prepare again 334 | var w1b = buf.prepare(4) 335 | w1b.write(a[4 .. 7]) 336 | buf.commit(4) 337 | 338 | # Reserve 8, commit 4 339 | var w2 = buf.reserve(8) 340 | w2.write(b[0 .. 3]) 341 | buf.commit(w2) 342 | 343 | check buf.consumable() == 12 344 | 345 | # Must reserve again 346 | var w2b = buf.reserve(4) 347 | w2b.write(b[4 .. 7]) 348 | buf.commit(w2b) 349 | 350 | # All data should be readable in order 351 | var r = buf.consume() 352 | check @(r.data()) == a 353 | r = buf.consume() 354 | check @(r.data()) == b[0 .. 3] 355 | r = buf.consume() 356 | check @(r.data()) == b[4 .. 7] 357 | check buf.consumable() == 0 358 | 359 | test "recycle buffer after consuming it": 360 | let buf = PageBuffers.init(8) 361 | 362 | let p0 = buf.prepare(4) 363 | 364 | buf.commit(p0) 365 | 366 | discard buf.consume() 367 | 368 | let p1 = buf.prepare(4) 369 | check: 370 | p0.startAddr == p1.startAddr 371 | 372 | test "Consuming iterator stops at reservation": 373 | let buf = PageBuffers.init(8) 374 | 375 | var p0 = buf.prepare(4) 376 | p0.write(bytes256[0 .. 3]) 377 | buf.commit(p0) 378 | 379 | var r0 = buf.reserve(4) 380 | r0.write(bytes256[0 .. 3]) 381 | 382 | var total = 0 383 | for (_, len) in buf.consumePageBuffers: 384 | total += len 385 | 386 | check: 387 | total == 4 388 | 389 | buf.commit(r0) 390 | 391 | for (_, len) in buf.consumePageBuffers: 392 | total += len 393 | 394 | check: 395 | total == 8 396 | 397 | test "unconsume works in consumeAll": 398 | let buf = PageBuffers.init(8) 399 | 400 | buf.write(bytes256[0 .. 7]) 401 | buf.write(bytes256[8 .. 15]) 402 | 403 | for span in buf.consumeAll(): 404 | buf.unconsume(span.len()) 405 | break 406 | 407 | check: 408 | buf.consumeAll() == bytes256[0..15] 409 | -------------------------------------------------------------------------------- /faststreams/pipelines.nim: -------------------------------------------------------------------------------- 1 | import 2 | "."/[inputs, outputs, async_backend] 3 | 4 | export 5 | inputs, outputs, async_backend 6 | 7 | {.pragma: iocall, nimcall, gcsafe, raises: [IOError].} 8 | 9 | when fsAsyncSupport: 10 | import 11 | std/macros, 12 | ./buffers 13 | 14 | type 15 | Pipe* = ref object 16 | # TODO: Make these stream handles 17 | input*: AsyncInputStream 18 | output*: AsyncOutputStream 19 | buffers*: PageBuffers 20 | 21 | template enterWait(fut: var Future, context: static string) = 22 | let wait = newFuture[void](context) 23 | fut = wait 24 | try: fsAwait wait 25 | finally: fut = nil 26 | 27 | template awake(fp: Future) = 28 | let f = fp 29 | if f != nil and not finished(f): 30 | complete f 31 | 32 | proc pipeRead(s: LayeredInputStream, 33 | dst: pointer, dstLen: Natural): Future[Natural] {.async.} = 34 | let buffers = s.buffers 35 | if buffers.eofReached: return 0 36 | 37 | var 38 | bytesInBuffersAtStart = buffers.consumable() 39 | minBytesExpected = max(1, dstLen) 40 | bytesInBuffersNow = bytesInBuffersAtStart 41 | 42 | while bytesInBuffersNow < minBytesExpected: 43 | awake buffers.waitingWriter 44 | buffers.waitingReader.enterWait "waiting for writer to buffer more data" 45 | 46 | bytesInBuffersNow = buffers.consumable() 47 | if buffers.eofReached: 48 | return bytesInBuffersNow - bytesInBuffersAtStart 49 | 50 | if dst != nil: 51 | let drained {.used.} = drainBuffersInto(s, cast[ptr byte](dst), dstLen) 52 | fsAssert drained == dstLen 53 | 54 | awake buffers.waitingWriter 55 | 56 | return bytesInBuffersNow - bytesInBuffersAtStart 57 | 58 | proc pipeWrite(s: LayeredOutputStream, src: pointer, srcLen: Natural) {.async.} = 59 | let buffers = s.buffers 60 | while buffers.canAcceptWrite(srcLen) == false: 61 | buffers.waitingWriter.enterWait "waiting for reader to drain the buffers" 62 | 63 | if src != nil: 64 | buffers.appendUnbufferedWrite(src, srcLen) 65 | 66 | awake buffers.waitingReader 67 | describeBuffers "pipeWrite", buffers 68 | 69 | template completedFuture(name: static string): untyped = 70 | let fut = newFuture[void](name) 71 | complete fut 72 | fut 73 | 74 | const pipeInputVTable = InputStreamVTable( 75 | readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural {.iocall.} = 76 | fsTranslateErrors "Failed to read from pipe": 77 | let ls = LayeredInputStream(s) 78 | fsAssert ls.allowWaitFor 79 | return waitFor pipeRead(ls, dst, dstLen) 80 | , 81 | readAsync: proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural] 82 | {.iocall.} = 83 | fsTranslateErrors "Unexpected error from the async macro": 84 | let ls = LayeredInputStream(s) 85 | return pipeRead(ls, dst, dstLen) 86 | , 87 | getLenSync: proc (s: InputStream): Option[Natural] {.iocall.} = 88 | let source = LayeredInputStream(s).source 89 | if source != nil: 90 | return source.len 91 | , 92 | closeSync: proc (s: InputStream) {.iocall.} = 93 | let source = LayeredInputStream(s).source 94 | if source != nil: 95 | close source 96 | , 97 | closeAsync: proc (s: InputStream): Future[void] {.iocall.} = 98 | fsTranslateErrors "Unexpected error from the async macro": 99 | let source = LayeredInputStream(s).source 100 | if source != nil: 101 | return closeAsync(Async source) 102 | else: 103 | return completedFuture("pipeInput.closeAsync") 104 | ) 105 | 106 | const pipeOutputVTable = OutputStreamVTable( 107 | writeSync: proc (s: OutputStream, src: pointer, srcLen: Natural) {.iocall.} = 108 | fsTranslateErrors "Failed to write all bytes to pipe": 109 | var ls = LayeredOutputStream(s) 110 | fsAssert ls.allowWaitFor 111 | waitFor pipeWrite(ls, src, srcLen) 112 | , 113 | writeAsync: proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void] 114 | {.iocall.} = 115 | # TODO: The async macro is raising exceptions even when 116 | # merely forwarding a future: 117 | fsTranslateErrors "Unexpected error from the async macro": 118 | return pipeWrite(LayeredOutputStream s, src, srcLen) 119 | , 120 | flushSync: proc (s: OutputStream) 121 | {.iocall.} = 122 | let destination = LayeredOutputStream(s).destination 123 | if destination != nil: 124 | flush destination 125 | , 126 | flushAsync: proc (s: OutputStream): Future[void] {.iocall.} = 127 | fsTranslateErrors "Unexpected error from the async macro": 128 | let destination = LayeredOutputStream(s).destination 129 | if destination != nil: 130 | return flushAsync(Async destination) 131 | else: 132 | return completedFuture("pipeOutput.flushAsync") 133 | , 134 | closeSync: proc (s: OutputStream) {.iocall.} = 135 | 136 | s.buffers.eofReached = true 137 | 138 | fsTranslateErrors "Unexpected error from Future.complete": 139 | awake s.buffers.waitingReader 140 | 141 | let destination = LayeredOutputStream(s).destination 142 | if destination != nil: 143 | close destination 144 | , 145 | closeAsync: proc (s: OutputStream): Future[void] {.iocall.} = 146 | s.buffers.eofReached = true 147 | 148 | fsTranslateErrors "Unexpected error from Future.complete": 149 | awake s.buffers.waitingReader 150 | 151 | fsTranslateErrors "Unexpected error from the async macro": 152 | let destination = LayeredOutputStream(s).destination 153 | if destination != nil: 154 | return closeAsync(Async destination) 155 | else: 156 | return completedFuture("pipeOutput.closeAsync") 157 | ) 158 | 159 | func pipeInput*(source: InputStream, 160 | pageSize = defaultPageSize, 161 | allowWaitFor = false): AsyncInputStream = 162 | fsAssert pageSize > 0 163 | 164 | AsyncInputStream LayeredInputStream( 165 | vtable: vtableAddr pipeInputVTable, 166 | buffers: initPageBuffers pageSize, 167 | allowWaitFor: allowWaitFor, 168 | source: source) 169 | 170 | func pipeInput*(buffers: PageBuffers, 171 | allowWaitFor = false, 172 | source: InputStream = nil): AsyncInputStream = 173 | var spanEndPos = Natural 0 174 | var span = if buffers.len == 0: default(PageSpan) 175 | else: buffers.obtainReadableSpan(spanEndPos) 176 | 177 | AsyncInputStream LayeredInputStream( 178 | vtable: vtableAddr pipeInputVTable, 179 | buffers: buffers, 180 | span: span, 181 | spanEndPos: span.len, 182 | allowWaitFor: allowWaitFor, 183 | source: source) 184 | 185 | proc pipeOutput*(destination: OutputStream, 186 | pageSize = defaultPageSize, 187 | maxBufferedBytes = defaultPageSize * 4, 188 | allowWaitFor = false): AsyncOutputStream = 189 | fsAssert pageSize > 0 190 | 191 | var 192 | buffers = initPageBuffers pageSize 193 | span = buffers.getWritableSpan() 194 | 195 | AsyncOutputStream LayeredOutputStream( 196 | vtable: vtableAddr pipeOutputVTable, 197 | buffers: buffers, 198 | span: span, 199 | spanEndPos: span.len, 200 | allowWaitFor: allowWaitFor, 201 | destination: destination) 202 | 203 | proc pipeOutput*(buffers: PageBuffers, 204 | allowWaitFor = false, 205 | destination: OutputStream = nil): AsyncOutputStream = 206 | var span = buffers.getWritableSpan() 207 | 208 | AsyncOutputStream LayeredOutputStream( 209 | vtable: vtableAddr pipeOutputVTable, 210 | buffers: buffers, 211 | span: span, 212 | # TODO What if the buffers are partially populated? 213 | # Should we adjust the spanEndPos? This would 214 | # need the old buffers.totalBytesWritten var. 215 | spanEndPos: span.len, 216 | allowWaitFor: allowWaitFor, 217 | destination: destination) 218 | 219 | func asyncPipe*(pageSize = defaultPageSize, 220 | maxBufferedBytes = defaultPageSize * 4): Pipe = 221 | fsAssert pageSize > 0 222 | Pipe(buffers: initPageBuffers(pageSize, maxBufferedBytes)) 223 | 224 | func initReader*(pipe: Pipe): AsyncInputStream = 225 | result = pipeInput(pipe.buffers) 226 | pipe.input = result 227 | 228 | func initWriter*(pipe: Pipe): AsyncOutputStream = 229 | result = pipeOutput(pipe.buffers) 230 | pipe.output = result 231 | 232 | proc exchangeBuffersAfterPipilineStep(input: InputStream, output: OutputStream) = 233 | let formerInputBuffers = input.buffers 234 | let formerOutputBuffers = output.getBuffers 235 | 236 | input.resetBuffers formerOutputBuffers 237 | output.recycleBuffers formerInputBuffers 238 | 239 | macro executePipeline*(start: InputStream, steps: varargs[untyped]): untyped = 240 | result = newTree(nnkStmtListExpr) 241 | 242 | var 243 | inputVal = start 244 | outputVal = newCall(bindSym"memoryOutput") 245 | 246 | inputVar = genSym(nskVar, "input") 247 | outputVar = genSym(nskVar, "output") 248 | 249 | step0 = steps[0] 250 | 251 | result.add quote do: 252 | var 253 | `inputVar` = `inputVal` 254 | `outputVar` = OutputStream `outputVal` 255 | 256 | `step0`(`inputVar`, `outputVar`) 257 | 258 | if steps.len > 2: 259 | let step1 = steps[1] 260 | result.add quote do: 261 | let formerInputBuffers = `inputVar`.buffers 262 | `inputVar` = memoryInput(getBuffers `outputVar`) 263 | recycleBuffers(`outputVar`, formerInputBuffers) 264 | `step1`(`inputVar`, `outputVar`) 265 | 266 | for i in 2 .. steps.len - 2: 267 | let step = steps[i] 268 | result.add quote do: 269 | exchangeBuffersAfterPipilineStep(`inputVar`, `outputVar`) 270 | `step`(`inputVar`, `outputVar`) 271 | 272 | var closingCall = steps[^1] 273 | closingCall.insert(1, outputVar) 274 | result.add closingCall 275 | 276 | if defined(debugMacros) or defined(debugPipelines): 277 | echo result.repr 278 | 279 | macro executePipeline*(start: AsyncInputStream, steps: varargs[untyped]): untyped = 280 | var 281 | stream = ident "stream" 282 | pipelineSteps = ident "pipelineSteps" 283 | pipelineBody = newTree(nnkStmtList) 284 | 285 | step0 = steps[0] 286 | stepOutput = genSym(nskVar, "pipe") 287 | 288 | pipelineBody.add quote do: 289 | var `pipelineSteps` = newSeq[Future[void]]() 290 | var `stepOutput` = asyncPipe() 291 | add `pipelineSteps`, `step0`(`stream`, initWriter(`stepOutput`)) 292 | 293 | var 294 | stepInput = stepOutput 295 | 296 | for i in 1 .. steps.len - 2: 297 | var step = steps[i] 298 | stepOutput = genSym(nskVar, "pipe") 299 | 300 | pipelineBody.add quote do: 301 | var `stepOutput` = asyncPipe() 302 | add `pipelineSteps`, `step`(initReader(`stepInput`), initWriter(`stepOutput`)) 303 | 304 | stepInput = stepOutput 305 | 306 | var RetTypeExpr = copy steps[^1] 307 | RetTypeExpr.insert(1, newCall("default", ident"AsyncInputStream")) 308 | 309 | var closingCall = steps[^1] 310 | closingCall.insert(1, newCall(bindSym"initReader", stepInput)) 311 | 312 | pipelineBody.add quote do: 313 | fsAwait allFutures(`pipelineSteps`) 314 | `closingCall` 315 | 316 | result = quote do: 317 | type UserOpRetType = type(`RetTypeExpr`) 318 | 319 | when UserOpRetType is Future: 320 | type RetType = type(default(UserOpRetType).read) 321 | else: 322 | type RetType = UserOpRetType 323 | 324 | proc pipelineProc(`stream`: AsyncInputStream): Future[RetType] {.async.} = 325 | when UserOpRetType is Future: 326 | var f = `pipelineBody` 327 | return fsAwait(f) 328 | else: 329 | return `pipelineBody` 330 | 331 | pipelineProc(`start`) 332 | 333 | when defined(debugMacros): 334 | echo result.repr 335 | 336 | -------------------------------------------------------------------------------- /tests/test_outputs.nim: -------------------------------------------------------------------------------- 1 | {.used.} 2 | 3 | import 4 | os, 5 | unittest2, 6 | random, 7 | strformat, 8 | sequtils, 9 | strutils, 10 | algorithm, 11 | stew/ptrops, 12 | ../faststreams, 13 | ../faststreams/buffers, 14 | ../faststreams/textio 15 | 16 | proc bytes(s: string): seq[byte] = 17 | result = newSeqOfCap[byte](s.len) 18 | for c in s: 19 | result.add byte(c) 20 | 21 | proc bytes(s: cstring): seq[byte] = 22 | for c in s: 23 | result.add byte(c) 24 | 25 | template bytes(c: char): byte = 26 | byte(c) 27 | 28 | template bytes(b: seq[byte]): seq[byte] = 29 | b 30 | 31 | template bytes[N, T](b: array[N, T]): seq[byte] = 32 | @b 33 | 34 | const line = "123456789123456789123456789123456789\n\n\n\n\n" 35 | 36 | proc randomBytes(n: int): seq[byte] = 37 | result.newSeq n 38 | for i in 0 ..< n: 39 | result[i] = byte(line[rand(line.len - 1)]) 40 | 41 | proc readAllAndClose(s: InputStream): seq[byte] = 42 | while s.readable: 43 | result.add s.read 44 | 45 | close(s) 46 | 47 | import memfiles 48 | 49 | suite "output stream": 50 | setup: 51 | var 52 | nimSeq: seq[byte] = @[] 53 | 54 | memStream = memoryOutput() 55 | smallPageSizeStream = memoryOutput(pageSize = 10) 56 | largePageSizeStream = memoryOutput(pageSize = 1000000) 57 | 58 | fileOutputPath = getTempDir() / "faststreams_testfile" 59 | unbufferedFileOutputPath = getTempDir() / "faststreams_testfile_unbuffered" 60 | 61 | fileStream = fileOutput(fileOutputPath) 62 | unbufferedFileStream = fileOutput(unbufferedFileOutputPath, pageSize = 0) 63 | 64 | bufferSize = 1000000 65 | buffer = alloc(bufferSize) 66 | streamWritingToExistingBuffer = unsafeMemoryOutput(buffer, bufferSize) 67 | 68 | teardown: 69 | close fileStream 70 | close unbufferedFileStream 71 | removeFile fileOutputPath 72 | removeFile unbufferedFileOutputPath 73 | dealloc buffer 74 | 75 | template output(val: auto) {.dirty.} = 76 | nimSeq.add bytes(val) 77 | memStream.write val 78 | smallPageSizeStream.write val 79 | largePageSizeStream.write val 80 | fileStream.write val 81 | unbufferedFileStream.write val 82 | streamWritingToExistingBuffer.write val 83 | 84 | template outputText(val: auto) = 85 | let valAsStr = $val 86 | nimSeq.add valAsStr.toOpenArrayByte(0, valAsStr.len - 1) 87 | 88 | memStream.writeText val 89 | smallPageSizeStream.writeText val 90 | largePageSizeStream.writeText val 91 | 92 | fileStream.writeText val 93 | unbufferedFileStream.writeText val 94 | 95 | streamWritingToExistingBuffer.writeText val 96 | 97 | template checkOutputsMatch(showResults = false, skipUnbufferedFile = false) = 98 | close fileStream 99 | close unbufferedFileStream 100 | 101 | check fileExists(fileOutputPath) and fileExists(unbufferedFileOutputPath) 102 | 103 | let 104 | memStreamRes = memStream.getOutput 105 | readFileRes = readFile(fileOutputPath).bytes 106 | fileInputRes = fileInput(fileOutputPath).readAllAndClose 107 | memFileInputRes = memFileInput(fileOutputPath).readAllAndClose 108 | fileInputWithSmallPagesRes = 109 | fileInput(fileOutputPath, pageSize = 10).readAllAndClose 110 | 111 | when showResults: 112 | checkpoint "Nim seq result" 113 | checkpoint $nimSeq 114 | 115 | checkpoint "Writes to existing buffer result" 116 | checkpoint $makeOpenArray( 117 | cast[ptr byte](buffer), streamWritingToExistingBuffer.pos 118 | ) 119 | 120 | checkpoint "mem stream result" 121 | checkpoint $memStreamRes 122 | 123 | checkpoint "readFile result" 124 | checkpoint $readFileRes 125 | 126 | checkpoint "fileInput result" 127 | checkpoint $fileInputRes 128 | 129 | checkpoint "memFileInput result" 130 | checkpoint $memFileInputRes 131 | 132 | checkpoint "fileInput with small pageSize result" 133 | checkpoint $fileInputWithSmallPagesRes 134 | 135 | let outputsMatch = 136 | nimSeq == makeOpenArray(cast[ptr byte](buffer), streamWritingToExistingBuffer.pos) 137 | check outputsMatch 138 | 139 | check nimSeq == memStreamRes 140 | check nimSeq == readFileRes 141 | check nimSeq == fileInputRes 142 | check nimSeq == memFileInputRes 143 | check nimSeq == fileInputWithSmallPagesRes 144 | 145 | when not skipUnbufferedFile: 146 | let unbufferedFileRes = readFile(unbufferedFileOutputPath).bytes 147 | check nimSeq == unbufferedFileRes 148 | 149 | test "no appends produce an empty output": 150 | checkOutputsMatch() 151 | 152 | test "write zero length slices": 153 | output "" 154 | output newSeq[byte]() 155 | var arr: array[0, byte] 156 | output arr 157 | 158 | check nimSeq.len == 0 159 | checkOutputsMatch() 160 | 161 | test "text output": 162 | for i in 1 .. 100: 163 | outputText i 164 | outputText " bottles on the wall" 165 | outputText '\n' 166 | 167 | checkOutputsMatch() 168 | 169 | test "cstrings": 170 | for i in 1 .. 100: 171 | output cstring("cstring sent by output ") 172 | output cstring("") 173 | outputText cstring("cstring sent by outputText ") 174 | outputText cstring("") 175 | 176 | checkOutputsMatch() 177 | 178 | test "memcpy": 179 | var x = 0x42'u8 180 | 181 | nimSeq.add x 182 | memStream.writeMemCopy x 183 | let memStreamRes = memStream.getOutput 184 | 185 | check memStreamRes == nimSeq 186 | 187 | template undelayedOutput(content: seq[byte]) {.dirty.} = 188 | nimSeq.add content 189 | streamWritingToExistingBuffer.write content 190 | 191 | test "delayed write": 192 | output "initial output\n" 193 | const delayedWriteContent = bytes "delayed write\n" 194 | let memStream2 = memoryOutput() 195 | 196 | var memCursor = memStream.delayFixedSizeWrite(delayedWriteContent.len) 197 | var fileCursor = fileStream.delayVarSizeWrite(delayedWriteContent.len + 50) 198 | var memCursor2 = memStream2.delayVarSizeWrite(10) 199 | 200 | let cursorStart = memStream.pos 201 | 202 | undelayedOutput delayedWriteContent 203 | 204 | var bytesWritten = 0 205 | for i, count in [2, 12, 342, 2121, 23, 1, 34012, 932]: 206 | output repeat(byte(i), count) 207 | bytesWritten += count 208 | check memStream.pos - cursorStart == bytesWritten 209 | 210 | memCursor.finalWrite delayedWriteContent 211 | fileCursor.finalWrite delayedWriteContent 212 | memCursor2.finalWrite [] 213 | 214 | checkOutputsMatch(skipUnbufferedFile = true) 215 | 216 | test "delayed write (edge cases)": 217 | const delayedWriteContent = bytes "delayed write\n" 218 | var 219 | memCursor1 = memStream.delayFixedSizeWrite(delayedWriteContent.len) 220 | fileCursor1 = fileStream.delayFixedSizeWrite(delayedWriteContent.len) 221 | undelayedOutput delayedWriteContent 222 | output "some output\n" 223 | var 224 | memCursor2 = memStream.delayFixedSizeWrite(delayedWriteContent.len) 225 | fileCursor2 = fileStream.delayFixedSizeWrite(delayedWriteContent.len) 226 | undelayedOutput delayedWriteContent 227 | output repeat(byte(42), 10000) 228 | var 229 | memCursor3 = memStream.delayFixedSizeWrite(delayedWriteContent.len) 230 | fileCursor3 = fileStream.delayFixedSizeWrite(delayedWriteContent.len) 231 | undelayedOutput delayedWriteContent 232 | 233 | memCursor1.finalWrite delayedWriteContent 234 | memCursor2.finalWrite delayedWriteContent 235 | memCursor3.finalWrite delayedWriteContent 236 | fileCursor1.finalWrite delayedWriteContent 237 | fileCursor2.finalWrite delayedWriteContent 238 | fileCursor3.finalWrite delayedWriteContent 239 | 240 | checkOutputsMatch(skipUnbufferedFile = true) 241 | 242 | test "float output": 243 | let basic: float64 = 12345.125 244 | let small: float32 = 12345.125 245 | let large: float64 = 9.99e+20 246 | let tiny: float64 = -2.25e-35 247 | 248 | outputText basic 249 | outputText small 250 | outputText large 251 | outputText tiny 252 | 253 | checkOutputsMatch() 254 | 255 | proc writeBlock(data: openArray[byte], output: var openArray[byte]): int = 256 | doAssert data.len <= output.len 257 | copyMem(unsafeAddr output[0], unsafeAddr data[0], data.len) 258 | data.len 259 | 260 | suite "randomized tests": 261 | type 262 | WriteTypes = enum 263 | FixedSize 264 | VarSize 265 | Mixed 266 | 267 | DelayedWrite = object 268 | isFixedSize: bool 269 | fixedSizeCursor: WriteCursor 270 | varSizeCursor: VarSizeWriteCursor 271 | content: seq[byte] 272 | written: int 273 | 274 | proc randomizedCursorsTestImpl( 275 | stream: OutputStream, 276 | seed = 1000, 277 | iterations = 1000, 278 | minWriteSize = 500, 279 | maxWriteSize = 1000, 280 | writeTypes = Mixed, 281 | varSizeVariance = 50, 282 | ): seq[byte] = 283 | randomize seed 284 | 285 | var delayedWrites = newSeq[DelayedWrite]() 286 | let writeSizeSpread = maxWriteSize - minWriteSize 287 | 288 | for i in 0 ..< iterations: 289 | let decision = rand(100) 290 | 291 | if decision < 20: 292 | # Write at some random cursor 293 | if delayedWrites.len > 0: 294 | let 295 | i = rand(delayedWrites.len - 1) 296 | written = delayedWrites[i].written 297 | remaining = delayedWrites[i].content.len - written 298 | toWrite = min(rand(remaining) + 10, remaining) 299 | 300 | if delayedWrites[i].isFixedSize: 301 | delayedWrites[i].fixedSizeCursor.write delayedWrites[i].content[ 302 | written ..< written + toWrite 303 | ] 304 | 305 | delayedWrites[i].written += toWrite 306 | 307 | if remaining - toWrite == 0: 308 | if delayedWrites[i].isFixedSize: 309 | finalize delayedWrites[i].fixedSizeCursor 310 | else: 311 | finalWrite delayedWrites[i].varSizeCursor, delayedWrites[i].content 312 | 313 | if i != delayedWrites.len - 1: 314 | swap(delayedWrites[i], delayedWrites[^1]) 315 | delayedWrites.setLen(delayedWrites.len - 1) 316 | 317 | continue 318 | 319 | let 320 | size = rand(writeSizeSpread) + minWriteSize 321 | randomBytes = randomBytes(size) 322 | 323 | if decision < 90: 324 | # Normal write 325 | result.add randomBytes 326 | stream.write randomBytes 327 | else: 328 | # Create cursor 329 | result.add randomBytes 330 | 331 | let isFixedSize = 332 | case writeTypes 333 | of FixedSize: 334 | true 335 | of VarSize: 336 | false 337 | of Mixed: 338 | rand(10) > 3 339 | 340 | if isFixedSize: 341 | let cursor = stream.delayFixedSizeWrite(randomBytes.len) 342 | 343 | delayedWrites.add DelayedWrite( 344 | fixedSizeCursor: cursor, content: randomBytes, written: 0, isFixedSize: true 345 | ) 346 | else: 347 | let 348 | overestimatedBytes = rand(varSizeVariance) 349 | cursorSize = randomBytes.len + overestimatedBytes 350 | cursor = stream.delayVarSizeWrite(cursorSize) 351 | 352 | delayedWrites.add DelayedWrite( 353 | varSizeCursor: cursor, content: randomBytes, written: 0, isFixedSize: false 354 | ) 355 | 356 | # Write all unwritten data to all outstanding cursors 357 | if stream != nil: 358 | for dw in mitems(delayedWrites): 359 | if dw.isFixedSize: 360 | let remaining = dw.content.len - dw.written 361 | dw.fixedSizeCursor.write dw.content[dw.written ..< dw.written + remaining] 362 | finalize dw.fixedSizeCursor 363 | else: 364 | dw.varSizeCursor.finalWrite dw.content 365 | 366 | template randomizedCursorsTest( 367 | streamExpr: OutputStreamHandle, 368 | writeTypesExpr: WriteTypes, 369 | varSizeVarianceExpr: int, 370 | customChecks: untyped = nil, 371 | ) = 372 | const testName = 373 | "randomized cursor test [" & astToStr(streamExpr) & ";writes=" & $writeTypesExpr & 374 | ",variance=" & $varSizeVarianceExpr & "]" 375 | test testName: 376 | let s = streamExpr 377 | var referenceResult = randomizedCursorsTestImpl( 378 | stream = s, writeTypes = writeTypesExpr, varSizeVariance = varSizeVarianceExpr 379 | ) 380 | 381 | when astToStr(customChecks) == "nil": 382 | let pos = s.pos 383 | let streamResult = s.getOutput() 384 | let resultsMatch = streamResult == referenceResult 385 | when false: 386 | if not resultsMatch: 387 | writeFile("reference-result.txt", referenceResult) 388 | writeFile("stream-result.txt", streamResult) 389 | check resultsMatch 390 | else: 391 | customChecks 392 | 393 | check referenceResult.len == pos 394 | 395 | randomizedCursorsTest(memoryOutput(), FixedSize, 0) 396 | randomizedCursorsTest(memoryOutput(), VarSize, 100) 397 | randomizedCursorsTest(memoryOutput(pageSize = 10), Mixed, 10) 398 | 399 | test "randomized file roundtrip": 400 | const randomBytesFileName = "random_bytes_file" 401 | 402 | var referenceBytes, restoredBytes: seq[byte] 403 | 404 | try: 405 | let output = fileOutput randomBytesFileName 406 | 407 | for i in 0 .. 3000: 408 | let bytes = randomBytes(rand(9999) + 1) 409 | referenceBytes.add bytes 410 | 411 | var openArraySize = rand(12000) 412 | if openArraySize >= bytes.len: 413 | # Make sure that sometimes `writeBlock` populates the entire span 414 | if i < 100: 415 | openArraySize = bytes.len 416 | output.advance writeBlock(bytes, output.getWritableBytes(openArraySize)) 417 | else: 418 | output.write bytes 419 | 420 | close output 421 | 422 | let input = fileInput randomBytesFileName 423 | 424 | while input.readable(10000): 425 | let r = 1 + rand(9999) 426 | if r < 5000: 427 | restoredBytes.add input.read(8000) 428 | restoredBytes.add input.read(500) 429 | elif r < 7000: 430 | restoredBytes.add input.read(1) 431 | restoredBytes.add input.read(2) 432 | restoredBytes.add input.read(5) 433 | restoredBytes.add input.read(17) 434 | restoredBytes.add input.read(128) 435 | else: 436 | restoredBytes.add input.read(r) 437 | 438 | while input.readable: 439 | restoredBytes.add input.read 440 | 441 | close input 442 | 443 | doAssert referenceBytes == restoredBytes 444 | finally: 445 | if fileExists(randomBytesFileName): 446 | removeFile randomBytesFileName 447 | 448 | test "ensureRunway": 449 | var output = memoryOutput() 450 | 451 | const writes = 256 452 | var buffer = newSeq[byte](writes) 453 | let totalBytes = block: 454 | var tmp = 0 455 | for i in 0 ..< writes: 456 | tmp += i 457 | buffer[i] = byte(i) 458 | tmp 459 | 460 | output.ensureRunway(totalBytes) 461 | 462 | for i in 0 ..< writes: 463 | output.write(buffer.toOpenArray(0, i - 1)) 464 | 465 | output.flush() 466 | 467 | let res = output.getOutput() 468 | var j = 0 469 | for i in 0 ..< writes: 470 | check: 471 | res[j ..< j + i] == buffer[0 ..< i] 472 | j += i 473 | 474 | test "ensureRunway with delayFixedSizeWrite": 475 | var output = memoryOutput() 476 | 477 | const writes = 256 478 | var buffer = newSeq[byte](writes) 479 | let totalBytes = block: 480 | var tmp = 0 481 | for i in 0 ..< writes: 482 | tmp += i 483 | buffer[i] = byte(i) 484 | tmp 485 | 486 | var cursor = output.delayFixedSizeWrite(8) 487 | 488 | output.ensureRunway(totalBytes) 489 | 490 | for i in 0 ..< writes: 491 | output.write(buffer.toOpenArray(0, i - 1)) 492 | 493 | const data = [byte 1, 2, 3, 4, 5, 6, 7, 8] 494 | cursor.finalWrite data 495 | 496 | output.flush() 497 | 498 | let res = output.getOutput() 499 | var j = 8 500 | for i in 0 ..< writes: 501 | check: 502 | res[j ..< j + i] == buffer[0 ..< i] 503 | j += i 504 | 505 | check: 506 | res[0 ..< data.len] == data 507 | 508 | suite "output api": 509 | test "can close default OutputStream": 510 | var v: OutputStream 511 | v.close() 512 | 513 | for pageSize in [1, 2, 10, 100]: 514 | for advanceSize in [1, 2, 10]: 515 | template testAdvance(expect: untyped) = 516 | test "getWritableBytes with partial advance " & $pageSize & " " & $advanceSize: 517 | var stream = memoryOutput(pageSize = pageSize) 518 | 519 | fill(stream.getWritableBytes(10), byte 'A') 520 | stream.advance(advanceSize) 521 | 522 | check: 523 | stream.getOutput(typeof(expect)) == expect 524 | 525 | test "multiple getWritableBytes " & $pageSize & " " & $advanceSize: 526 | var stream = memoryOutput(pageSize = pageSize) 527 | var expect2: typeof(expect) 528 | for _ in 0 ..< 10: 529 | fill(stream.getWritableBytes(10), byte 'A') 530 | stream.advance(advanceSize) 531 | expect2.add expect 532 | 533 | check: 534 | stream.getOutput(typeof(expect2)) == expect2 535 | 536 | testAdvance(repeat('A', advanceSize)) 537 | testAdvance(repeat(byte 'A', advanceSize)) 538 | 539 | test "ensureRunway works before delayed write": 540 | var stream = memoryOutput(pageSize = 10) 541 | 542 | stream.ensureRunway(100) 543 | 544 | var w0 = stream.delayVarSizeWrite(10) 545 | 546 | stream.write [byte 4, 5, 6, 7] 547 | 548 | w0.finalWrite [byte 0, 1, 2, 3] 549 | 550 | check: 551 | stream.getOutput(seq[byte]) == [byte 0, 1, 2, 3, 4, 5, 6, 7] 552 | 553 | test "getOutput drains the buffer": 554 | var stream = memoryOutput(pageSize = 4) 555 | 556 | stream.write([byte 0, 1, 2, 3]) 557 | stream.write([byte 4, 5, 6, 7]) 558 | check: 559 | stream.getOutput(seq[byte]) == [byte 0, 1, 2, 3, 4, 5, 6, 7] 560 | stream.getOutput(seq[byte]) == [] 561 | 562 | stream.write([byte 0]) 563 | check: 564 | stream.getOutput(seq[byte]) == [byte 0] 565 | stream.getOutput(seq[byte]) == [] 566 | 567 | for i in 0 ..< 128: 568 | stream.write([byte i]) 569 | 570 | check: 571 | stream.getOutput(seq[byte]).len == 128 572 | stream.getOutput(seq[byte]) == [] 573 | 574 | test "consumeOutputs drains the buffer": 575 | var stream = memoryOutput(pageSize = 4) 576 | 577 | stream.write([byte 0, 1, 2, 3]) 578 | stream.write([byte 4, 5, 6, 7]) 579 | 580 | var tmp: seq[byte] 581 | stream.consumeOutputs(bytes): 582 | tmp.add bytes 583 | 584 | check: 585 | tmp == [byte 0, 1, 2, 3, 4, 5, 6, 7] 586 | reset(tmp) 587 | 588 | stream.consumeOutputs(bytes): 589 | tmp.add bytes 590 | 591 | check: 592 | tmp == [] 593 | 594 | stream.write([byte 0]) 595 | stream.consumeContiguousOutput(bytes): 596 | tmp.add bytes 597 | 598 | check: 599 | tmp == [byte 0] 600 | reset(tmp) 601 | 602 | stream.consumeContiguousOutput(bytes): 603 | tmp.add bytes 604 | 605 | check: 606 | tmp == [] 607 | 608 | for i in 0 ..< 128: 609 | stream.write([byte i]) 610 | 611 | stream.consumeContiguousOutput(bytes): 612 | tmp.add bytes 613 | 614 | check: 615 | tmp.len == 128 616 | reset(tmp) 617 | 618 | stream.consumeContiguousOutput(bytes): 619 | tmp.add bytes 620 | 621 | check: 622 | tmp == [] 623 | 624 | test "consumeContiguousOutput drains the buffer": 625 | var stream = memoryOutput(pageSize = 4) 626 | 627 | stream.write([byte 0, 1, 2, 3]) 628 | stream.write([byte 4, 5, 6, 7]) 629 | 630 | var tmp: seq[byte] 631 | stream.consumeContiguousOutput(bytes): 632 | tmp = @bytes 633 | 634 | check: 635 | tmp == [byte 0, 1, 2, 3, 4, 5, 6, 7] 636 | 637 | stream.consumeContiguousOutput(bytes): 638 | tmp = @bytes 639 | 640 | check: 641 | tmp == [] 642 | 643 | stream.write([byte 0]) 644 | stream.consumeContiguousOutput(bytes): 645 | tmp = @bytes 646 | 647 | check: 648 | tmp == [byte 0] 649 | 650 | stream.consumeContiguousOutput(bytes): 651 | tmp = @bytes 652 | 653 | check: 654 | tmp == [] 655 | 656 | 657 | for i in 0 ..< 128: 658 | stream.write([byte i]) 659 | 660 | stream.consumeContiguousOutput(bytes): 661 | tmp = @bytes 662 | 663 | check: 664 | tmp.len == 128 665 | 666 | stream.consumeContiguousOutput(bytes): 667 | tmp = @bytes 668 | 669 | check: 670 | tmp == [] 671 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nim-faststreams 2 | 3 | [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 4 | [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) 5 | ![Stability: experimental](https://img.shields.io/badge/stability-experimental-orange.svg) 6 | ![Github action](https://github.com/status-im/nim-faststreams/workflows/CI/badge.svg) 7 | 8 | FastStreams is a highly efficient library for all your I/O needs. 9 | 10 | It offers nearly zero-overhead synchronous streams for handling inputs and 11 | outputs of various types: 12 | 13 | * Memory inputs and outputs for serialization frameworks and parsers 14 | * File inputs and outputs 15 | 16 | The library aims to provide a common interface between all stream types 17 | that allows the application code to be easily portable to different back-ends. 18 | 19 | The library contains prototype-level support for `{.async.}` streams - this 20 | support should be considered experimental and may change in future releases. 21 | 22 | ## What does zero-overhead mean? 23 | 24 | Even though FastStreams support multiple stream types, the API is designed 25 | in a way that allows the read and write operations to be handled without any 26 | dynamic dispatch in the majority of cases. 27 | 28 | In particular, reading from a `memoryInput` or writing to a `memoryOutput` 29 | will have similar performance to a loop iterating over an `openArray` or 30 | another loop populating a pre-allocated `string`. `memFileInput` offers 31 | the same performance characteristics when working with files. The idiomatic 32 | use of the APIs with the rest of the stream types will result in a highly 33 | efficient memory allocation patterns and zero-copy performance in a great 34 | variety of real-world use cases such as: 35 | 36 | * Parsers for data formats and protocols employing formal grammars 37 | * Block ciphers 38 | * Compressors and decompressors 39 | * Stream multiplexers 40 | 41 | The zero-copy behavior and low-memory usage is maintained even when multiple 42 | streams are layered on top of each other while back-pressure is properly 43 | accounted for. This makes FastStreams ideal for implementing highly-flexible 44 | networking stacks such as [LibP2P](https://github.com/status-im/nim-libp2p). 45 | 46 | ## The key ideas in the FastStreams design 47 | 48 | FastStreams is heavily inspired by the `System.IO.Pipelines` API which was 49 | developed and released by Microsoft in 2018 and is considered the result of 50 | multiple years of evolution over similar APIs shipped in previous SDKs. 51 | 52 | We highly recommend reading the following two articles which provide an in-depth 53 | explanation for the benefits of the design: 54 | 55 | * https://blog.marcgravell.com/2018/07/pipe-dreams-part-1.html 56 | * https://blog.marcgravell.com/2018/07/pipe-dreams-part-2.html 57 | 58 | Here, we'll only summarize the main insights: 59 | 60 | ### Obtaining data from the input device is not the same as consuming it. 61 | 62 | When protocols and formats are layered on top of each other, it's highly 63 | inconvenient to handle a read operation that can return an arbitrary amount 64 | of data. If not enough data was returned, you may need to copy the available 65 | bytes into a local buffer and then repeat the reading operation until enough 66 | data is gathered and the local buffer can be processed. On the other hand, 67 | if more data was received, you need to complete the current stage of processing 68 | and then somehow feed the remaining bytes into the next stage of processing 69 | (e.g. this might be a nested format or a different parsing branch in the formal 70 | grammar of the protocol). Both of these scenarios require logic that is 71 | difficult to write correctly and results in unnecessary copying of the input 72 | bytes. 73 | 74 | A major difference in the FastStreams design is that the arbitrary-length 75 | data obtained from the input device is managed by the stream itself while you 76 | are provided with an API allowing you to control precisely how much data 77 | is consumed from the stream. Consuming the buffered content does not invoke 78 | costly calls and you are allowed to peek at the stream contents 79 | before deciding which step to take next (something crucial for handling formal 80 | grammars). Thus, using the FastStreams API results in code that is both highly 81 | efficient and easy to author. 82 | 83 | ### Higher efficiency is possible if we say goodbye to the good old single buffer. 84 | 85 | The buffering logic inside the stream divides the data into "pages" which 86 | are allocated with a known fast path in the Nim allocator and which can be 87 | efficiently transferred between streams and threads in the layered streams 88 | scenario or in IPC mechanisms such as `AsyncChannel`. The consuming code can 89 | be aware of this, but doesn't need to. The most idiomatic usage of the API 90 | handles the buffer switching logic automatically for the user. 91 | 92 | Nevertheless, the buffering logic can be configured for unbuffered reads 93 | and writes and it supports efficiently various common real-world patterns 94 | such as: 95 | 96 | * Length prefixes 97 | 98 | To handle protocols with length prefixes without any memory overhead, 99 | the output streams support "delayed writes" where a portion of the 100 | stream content is specified only after the prefixed content is written 101 | to the stream. 102 | 103 | * Block compressors and Block ciphers 104 | 105 | These can benefit significantly from a more precise control over 106 | the stride of the buffered pages which can be configured to match 107 | the block size of the encoder. 108 | 109 | * Content with known length 110 | 111 | Some streams have a known length which allows us to accurately estimate 112 | the size of the transformed content. The `len` and `ensureRunway` APIs 113 | make sure such cases are handled as optimally as possible. 114 | 115 | ## Basic API usage 116 | 117 | The FastStreams API consists of ony few major object types: 118 | 119 | ### `InputStream` 120 | 121 | An `InputStream` manages a particular input device. The library offers out 122 | of the box the following input stream types: 123 | 124 | * `fileInput` 125 | 126 | For reading files through the familiar `fread` API from the C run-time. 127 | 128 | * `memFileInput` 129 | 130 | For reading memory mapped files which provides the best performance. 131 | 132 | * `unsafeMemoryInput` 133 | 134 | For handling strings, sequences and openarrays as an input stream.
135 | You are responsible for ensuring that the backing buffer won't be invalidated 136 | while the stream is being used. 137 | 138 | * `memoryInput` 139 | 140 | Primarily used to consume the contents written to a previously populated 141 | output stream, but it can also be used to consume the contents of strings 142 | and sequences in a memory-safe way (by creating a copy). 143 | 144 | * `pipeInput` (async) 145 | 146 | For arbitrary communication between a producer and a consumer. 147 | 148 | * `chronosInput` (async) 149 | 150 | Enabled by importing `faststreams/chronos_adapters`.
151 | It can represent any Chronos `Transport` as an input stream. 152 | 153 | * `asyncSocketInput` (async) 154 | 155 | Enabled by importing `faststreams/std_adapters`.
156 | Allows using Nim's standard library `AsyncSocket` type as an input stream. 157 | 158 | You can extend the library with new `InputStream` types without modifying it. 159 | Please see the inline code documentation of `InputStreamVTable` for more details. 160 | 161 | All of the above APIs are possible constructors for creating an `InputStream`. 162 | The stream instances will manage their resources through destructors, but you 163 | might want to `close` them explicitly in async context or when you need to 164 | handle the possible errors from the closing operation. 165 | 166 | Here is an example usage: 167 | 168 | ```nim 169 | import 170 | faststreams/inputs 171 | 172 | var 173 | jsonString = "[1, 2, 3]" 174 | jsonNodes = parseJson(unsafeMemoryInput(jsonString)) 175 | moreNodes = parseJson(fileInput("data.json")) 176 | ``` 177 | 178 | The example above assumes we might have a `parseJson` function accepting an 179 | `InputStream`. Here how this function could be defined: 180 | 181 | ```nim 182 | import 183 | faststreams/inputs 184 | 185 | proc scanString(stream: InputStream): JsonToken {.fsMultiSync.} = 186 | result = newStringToken() 187 | 188 | advance stream # skip the opening quote 189 | 190 | while stream.readable: 191 | let nextChar = stream.read.char 192 | case nextChar 193 | of '\'': 194 | if stream.readable: 195 | let escaped = stream.read.char 196 | case escaped 197 | of 'n': result.add '\n' 198 | of 't': result.add '\t' 199 | else: result.add escaped 200 | else: 201 | error(UnexpectedEndOfFile) 202 | of '"' 203 | return 204 | else: 205 | result.add nextChar 206 | 207 | error(UnexpectedEndOfFile) 208 | 209 | proc nextToken(stream: InputStream): JsonToken {.fsMultiSync.} = 210 | while stream.readable: 211 | case stream.peek.char 212 | of '"': 213 | result = scanString(stream) 214 | of '0'..'9': 215 | result = scanNumber(stream) 216 | of 'a'..'z', 'A'..'Z', '_': 217 | result = scanIdentifier(stream) 218 | of '{': 219 | advance stream # skip the character 220 | result = objectStartToken 221 | ... 222 | 223 | return eofToken 224 | 225 | proc parseJson(stream: InputStream): JsonNode {.fsMultiSync.} = 226 | while (let token = nextToken(stream); token != eofToken): 227 | case token 228 | of numberToken: 229 | result = newJsonNumber(token.num) 230 | of stringToken: 231 | result = newJsonString(token.str) 232 | of objectStartToken: 233 | result = parseObject(stream) 234 | ... 235 | ``` 236 | 237 | The above example is nothing but a toy program, but we can already see many 238 | usage patterns of the `InputStream` type. For a more sophisticated and complete 239 | implementation of a JSON parser, please see the [nim-json-serialization](https://github.com/status-im/nim-json-serialization) 240 | package. 241 | 242 | As we can see from the example above, calling `stream.read` should always be 243 | preceded by a call to `stream.readable`. When the stream is in the readable 244 | state, we can also `peek` at the next character before we decide how to 245 | proceed. Besides calling `read`, we can also mark the data as consumed by 246 | calling `stream.advance`. 247 | 248 | The above APIs demonstrate how you can consume the data one byte at the time. 249 | Common wisdom might tell you that this should be inefficient, but that's not 250 | the case with FastStreams. The loop `while stream.readable: stream.read` will 251 | compile to very efficient inlined code that performs nothing more than pointer 252 | increments and comparisons. This will be true even when working with async 253 | streams. 254 | 255 | The `readable` check is the only place where our code may block (or await). 256 | Only when all the data in the stream buffers have been consumed, the stream 257 | will invoke a new read operation on the backing input device and this may 258 | repopulate the buffers with an arbitrary number of new bytes. 259 | 260 | Sometimes, you need to check whether the stream contains at least a specific 261 | number of bytes. You can use the `stream.readable(N)` API to achieve this. 262 | 263 | Reading multiple bytes at once is then possible with `stream.read(N)`, but 264 | if you need to store the bytes in an object field or another long-term storage 265 | location, consider using `stream.readInto(destination)` which may result in 266 | zero-copy operation. It can also be used to implement unbuffered reading. 267 | 268 | #### `AsyncInputStream` and `fsMultiSync` 269 | 270 | An astute reader might have wondered what is the purpose of the custom pragma 271 | `fsMultiSync` used in the examples above? It is a simple macro generating an 272 | additional `async` copy of our stream processing functions where all the input 273 | types are replaced by their async counterparts (e.g. `AsyncInputStream`) and 274 | the return type is wrapped in a `Future` as usual. 275 | 276 | The standard API of `InputStream` and `AsyncInputStream` is exactly the same. 277 | Operations such as `readable` will just invoke `await` behind the scenes, but 278 | there is one key difference - the `await` will be triggered only when there 279 | is not enough data already stored in the stream buffers. Thus, in the great 280 | majority of cases, we avoid the high cost of instantiating a `Future` and 281 | yielding control to the event loop. 282 | 283 | We highly recommend implementing most of your stream processing code through 284 | the `fsMultiSync` pragma. This ensures the best possible performance and makes 285 | the code more easily testable (e.g. with inputs stored on disk). FastStreams 286 | ships with a set of fuzzing tools that will help you ensure that your code 287 | behaves correctly with arbitrary data and/or arbitrary interruption points. 288 | 289 | Nevertheless, if you need a more traditional async API, please be aware that 290 | all of the functions discussed in this README also have an `*Async` suffix 291 | form that returns a `Future` (e.g. `readableAsync`, `readAsync`, etc). 292 | 293 | One exception to the above rule is the helper `stream.timeoutToNextByte(t)` 294 | which can be used to detect situations where your communicating party is 295 | failing to send data in time. It accepts a `Duration` or an existing deadline 296 | `Future` and it's usually used like this: 297 | 298 | ```nim 299 | proc performHandshake(c: Connection): bool {.async.} = 300 | if c.inputStream.timeoutToNextByte(HANDSHAKE_TIMEOUT): 301 | # The other party didn't send us anything in time, 302 | # We close the connection: 303 | close c 304 | return false 305 | 306 | while c.inputStream.readable: 307 | ... 308 | ``` 309 | 310 | It is assumed that in traditional async code, timeouts will be managed more 311 | explicitly with `sleepAsync` and the `or` operator defined over futures. 312 | 313 | #### Range-restricted reads 314 | 315 | Protocols transmitting serialized payloads often provide information regarding 316 | the size of the payload. When you invoke the deserialization routine, it's 317 | preferable if the provided boundaries are treated like an "end of file" marker 318 | for the deserializer. FastStreams provides an easy way to achieve this without 319 | extra copies and memory allocations through the `withReadableRange` facility. 320 | Here is a typical usage: 321 | 322 | ```nim 323 | proc decodeFrame(s: AsyncInputStream, DecodedType: type): Option[DecodedType] = 324 | if not s.readable(4): 325 | return 326 | 327 | let lengthPrefix = toInt32 s.read(4) 328 | if s.readable(lengthPrefix): 329 | s.withReadableRange(lengthPrefix, range): 330 | range.readValue(Json, DecodedType) 331 | ``` 332 | 333 | Please note that the above example uses the [nim-serialization library](https://github.com/status-im/nim-serialization/) 334 | 335 | Simply, inside the `withReadableRange` block, `range` becomes a stream for 336 | which `s.readable` will return `false` as soon as the Json parser has consumed 337 | the specified number of bytes. 338 | 339 | Furthermore, `withReadableRange` guarantees that all stream operations within 340 | the block will be non-blocking, so it will transform the `AsyncInputStream` 341 | into a regular `InputStream`. Depending on the complexity of the stream 342 | processing functions, this will often lead to significant performance gains. 343 | 344 | ### `OutputStream` and `AsyncOutputStream` 345 | 346 | An `OutputStream` manages a particular output device. The library offers out 347 | of the box the following output stream types: 348 | 349 | * `writeFileOutput` 350 | 351 | For writing files through the familiar `fwrite` API from the C run-time. 352 | 353 | * `memoryOutput` 354 | 355 | For building a `string` or a `seq[byte]` result. 356 | 357 | * `unsafeMemoryOutput` 358 | 359 | For writing to an arbitrary existing buffer.
360 | You are responsible for ensuring that the backing buffer won't be invalidated 361 | while the stream is being used. 362 | 363 | * `pipeOutput` (async) 364 | 365 | For arbitrary communication between a produced and a consumer. 366 | 367 | * `chronosOutput` (async) 368 | 369 | Enabled by importing `faststreams/chronos_adapters`.
370 | It can represent any Chronos `Transport` as an input stream. 371 | 372 | * `asyncSocketOutput` (async) 373 | 374 | Enabled by importing `faststreams/std_adapters`.
375 | Allows using Nim's standard library `AsyncSocket` type as an output stream. 376 | 377 | You can extend the library with new `OutputStream` types without modifying it. 378 | Please see the inline code documentation of `OutputStreamVTable` for more details. 379 | 380 | All of the above APIs are possible constructors for creating an `OutputStream`. 381 | The stream instances will manage their resources through destructors, but you 382 | might want to `close` them explicitly in async context or when you need to 383 | handle the possible errors from the closing operation. 384 | 385 | Here is an example usage: 386 | 387 | ```nim 388 | import 389 | faststreams/outputs 390 | 391 | type 392 | ABC = object 393 | a: int 394 | b: char 395 | c: string 396 | 397 | var stream = memoryOutput() 398 | stream.writeNimRepr(ABC(a: 1, b: 'b', c: "str")) 399 | var repr = stream.getOutput(string) 400 | ``` 401 | 402 | The `writeNimRepr` in the above example is not part of the library, but 403 | let's see how it can be implemented: 404 | 405 | ```nim 406 | import 407 | typetraits, faststreams/outputs 408 | 409 | proc writeNimRepr*(stream: OutputStream, str: string) = 410 | stream.write '"' 411 | 412 | for c in str: 413 | if c == '"': 414 | stream.write ['\'', '"'] 415 | else: 416 | stream.write c 417 | 418 | stream.write '"' 419 | 420 | proc writeNimRepr*(stream: OutputStream, x: char) = 421 | stream.write ['\'', x, '\''] 422 | 423 | proc writeNimRepr*(stream: OutputStream, x: int) = 424 | stream.write $x # Making this more optimal has been left 425 | # as an exercise for the reader 426 | 427 | proc writeNimRepr*[T](stream: OutputStream, obj: T) = 428 | stream.write typetraits.name(T) 429 | stream.write '(' 430 | 431 | var firstField = true 432 | for name, val in fieldPairs(obj): 433 | if not firstField: 434 | stream.write ", " 435 | 436 | stream.write name 437 | stream.write ": " 438 | stream.writeNimRepr val 439 | 440 | firstField = false 441 | 442 | stream.write ')' 443 | ``` 444 | 445 | When the stream is created, its output buffers will be initialized with a 446 | single page of `pageSize` bytes (specified at stream creation). Calls to 447 | `write` will just populate this page until it becomes full and only then 448 | it would be sent to the output device. 449 | 450 | As the example demonstrates, a `memoryOutput` will continue buffering 451 | pages until they can be finally concatenated and returned in `stream.getOutput`. 452 | If the output fits within a single page, it will be efficiently moved to 453 | the `getOutput` result. When the output size is known upfront you can ensure 454 | that this optimization is used by calling `stream.ensureRunway` before any 455 | writes, but please note that the library is free to ignore this hint in async 456 | context or if a maximum memory usage policy is specified. 457 | 458 | In a non-memory stream, any writes larger than a page or issued through the 459 | `writeNow` API will be sent to the output device immediately. 460 | 461 | Please note that even in async context, `write` will complete immediately. 462 | To handle back-pressure properly, use `stream.flush` or `stream.waitForConsumer` 463 | which will ensure that the buffered data is drained to a specified number of 464 | bytes before continuing. The rationale here is that introducing an interruption 465 | point at every `write` produces less optimal code, but if this is desired you 466 | can use the `stream.writeAndWait` API. 467 | 468 | If you have existing algorithms that output data to an `openArray`, you can use 469 | the `stream.getWritableBytes` API to continue using them without introducing any 470 | intermediate buffers. 471 | 472 | #### Delayed Writes 473 | 474 | Many protocols and formats employ fixed-size and variable-size length prefixes 475 | that have been traditionally difficult to handle because they require you to 476 | either measure the size of the content before writing it to the stream, or 477 | even worse, serialize it to a memory buffer in order to determine its size. 478 | 479 | FastStreams supports handling such length prefixes with a zero-copy mechanism 480 | that doesn't require additional memory allocations. `stream.delayFixedSizeWrite` 481 | and `stream.delayVarSizeWrite` are APIs that return a `WriteCursor` object that 482 | can be used to implement a delayed write to the stream. After obtaining the 483 | write cursor you can take a note of the current `pos` in the stream and then 484 | continue issuing `stream.write` operations normally. After all of the content 485 | is written, you obtain `pos` again to determine the final value of the length 486 | prefix. Throughout the whole time, you are free to call `write` on the cursor 487 | to populate the "hole" left in the stream with bytes, but at the end you must 488 | call `finalize` to unlock the stream for flushing. You can also perform the 489 | finalization in one step with `finalWrite` (the one-step approach is mandatory 490 | for variable-size prefixes). 491 | 492 | ### `Pipeline` 493 | 494 | (This section is a stub and it will be expanded with more details in the future) 495 | 496 | A `Pipeline` represents a chain of transformations that should be applied to a 497 | stream. It starts with an `InputStream` followed by one or more transformation 498 | steps and ending with a result. 499 | 500 | Each transformation step is a function of the kind: 501 | 502 | ```nim 503 | type PipelineStep* = proc (i: InputStream, o: OutputStream) 504 | {.gcsafe, raises: [CatchableError].} 505 | ``` 506 | 507 | A result obtaining operation is a function of the kind: 508 | 509 | ```nim 510 | type PipelineResultProc*[T] = proc (i: InputStream): T 511 | {.gcsafe, raises: [CatchableError].} 512 | ``` 513 | 514 | Please note that `stream.getOutput` is an example of such a function. 515 | 516 | Pipelines executed in place with `executePipeline` API. If the first input 517 | source is async, then the whole pipeline with be executing asynchronously which 518 | can result in a much lower memory usage. 519 | 520 | The pipeline transformation steps are usually employing the `fsMultiSync` 521 | pragma to make them usable in both synchronous and asynchronous scenarios. 522 | 523 | Please note that the above higher-level APIs are just about simplifying the 524 | instantiation of multiple `Pipe` objects that can be used to hook input and 525 | output streams in arbitrary ways. 526 | 527 | A stream multiplexer for example is likely to rely on the lower-level `Pipe` 528 | objects and the underlying `PageBuffers` directly. 529 | 530 | ## License 531 | 532 | Licensed and distributed under either of 533 | 534 | * MIT license: [LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT 535 | 536 | or 537 | 538 | * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0) 539 | 540 | at your option. This file may not be copied, modified, or distributed except according to those terms. 541 | -------------------------------------------------------------------------------- /faststreams/buffers.nim: -------------------------------------------------------------------------------- 1 | import 2 | std/[deques, options], 3 | stew/ptrops, 4 | async_backend 5 | 6 | export 7 | deques 8 | 9 | type 10 | PageSpan* = object 11 | ## View into memory area backed by a Page, allowing efficient access. 12 | ## 13 | ## Unlike openArray, lifetime must be managed manually 14 | ## Similar to UncheckedArray, range checking is generally not performed. 15 | ## 16 | ## The end address points to the memory immediately after the last entry, 17 | ## meaning that when startAddr == endAddr, the span is empty. 18 | ## endAddr is used instead of length so that when advancing, only a single 19 | ## pointer needs to be updated. 20 | startAddr*, endAddr*: ptr byte 21 | 22 | Page* = object 23 | ## A Page is a contiguous fixed-size memory area for managing a stream of 24 | ## bytes. 25 | ## 26 | ## Each page is divided into input and output sequences - the output 27 | ## sequence is the part prepared for writing while the input is the 28 | ## part already written and waiting to be read. 29 | ## 30 | ## As data is written, the output sequence is moved to the input sequence 31 | ## by adjusting the `writtenTo` counter. Similarly, as data is read from the 32 | ## input sequence, `consumedTo` is updated to reflect the number of bytes read. 33 | ## 34 | ## A page may also be created to reserve a part of the output sequence for 35 | ## delayed writing, such as when a length prefix is written after producing 36 | ## the data. Such a page will have `reservedTo` set, and as long as such a 37 | ## marker exists, the following pages will not be made available for 38 | ## consuming. 39 | ## 40 | ## Data is typically read and written in batches represented by a PageSpan, 41 | ## where each batch can be bulk-processed efficiently. 42 | consumedTo*: int 43 | ## Number of bytes consumed from the input sequence 44 | writtenTo*: Natural 45 | ## Number of bytes written to the input sequence 46 | reservedTo*: Natural 47 | ## Number of bytes reserved for future writing 48 | store*: ref seq[byte] 49 | ## Memory backing the page - allocated once and never resized to maintain 50 | ## pointer stability. Multiple pages may share the same store, which 51 | ## specially happens when reserving parts of a page for delayed writing. 52 | 53 | PageRef* = ref Page 54 | 55 | PageBuffers* = ref object 56 | ## PageBuffers is a memory management structure designed for efficient 57 | ## buffering of streaming data. It divides the buffer into pages (blocks of 58 | ## memory), which are managed in a queue. 59 | ## 60 | ## This approach is suitable in scenarios where data arrives or is consumed 61 | ## in chunks of varying size, such as when decoding a network stream or file 62 | ## into structured data. 63 | ## 64 | ## Pages are kept in a deque. New pages are prepared as needed when writing, 65 | ## and fully consumed pages are removed after reading, except for the last 66 | ## one that is recycled for the next write. 67 | ## 68 | ## Pages can also be reserved to delay the writing of a prefix while data 69 | ## is being buffered. In such cases, writing can continue but reading will 70 | ## be blocked until the reservation is committed. 71 | pageSize*: Natural 72 | ## Default size of freshly allocated pages - pages are typically of at 73 | ## least this size but may also be larger 74 | 75 | maxBufferedBytes*: Natural 76 | ## TODO: currently unusued 77 | 78 | queue*: Deque[PageRef] 79 | 80 | when fsAsyncSupport: 81 | waitingReader*: Future[void] 82 | waitingWriter*: Future[void] 83 | 84 | eofReached*: bool 85 | 86 | const 87 | nimPageSize* = 4096 88 | nimAllocatorMetadataSize* = 32 89 | # TODO: Get this legally from the Nim allocator. 90 | # The goal is to make perfect page-aligned allocations 91 | # that go through a fast O(0) path in the allocator. 92 | defaultPageSize* = 4096 - nimAllocatorMetadataSize 93 | maxStackUsage* = 16384 94 | 95 | when not declared(newSeqUninit): # nim 2.2+ 96 | template newSeqUninit[T: byte](len: int): seq[byte] = 97 | newSeqUninitialized[byte](len) 98 | 99 | when debugHelpers: 100 | proc describeBuffers*(context: static string, buffers: PageBuffers) = 101 | debugEcho context, " :: buffers" 102 | for page in buffers.queue: 103 | debugEcho " page ", page.store[][page.consumedTo ..< 104 | min(page.consumedTo + 16, page.writtenTo)] 105 | debugEcho " len = ", page.store[].len 106 | debugEcho " start = ", page.consumedTo 107 | debugEcho " written to = ", page.writtenTo 108 | 109 | func contents*(buffers: PageBuffers): seq[byte] = 110 | for page in buffers.queue: 111 | result.add page.store[][page.consumedTo ..< page.writtenTo - 1] 112 | else: 113 | template describeBuffers*(context: static string, buffers: PageBuffers) = 114 | discard 115 | 116 | func openArrayToPair*(a: var openArray[byte]): (ptr byte, Natural) = 117 | if a.len > 0: 118 | (addr a[0], Natural(a.len)) 119 | else: 120 | (nil, Natural(0)) 121 | 122 | template allocRef[T: not ref](x: T): ref T = 123 | let res = new type(x) 124 | res[] = x 125 | res 126 | 127 | func nextAlignedSize*(minSize, pageSize: Natural): Natural = 128 | if pageSize == 0: 129 | minSize 130 | else: 131 | max(((minSize + pageSize - 1) div pageSize) * pageSize, pageSize) 132 | 133 | func len*(span: PageSpan): Natural {.inline.} = 134 | distance(span.startAddr, span.endAddr) 135 | 136 | func `$`*(span: PageSpan): string = 137 | # avoid repr(ptr byte) since it can crash if the pointee is gone 138 | "[startAddr: " & repr(pointer(span.startAddr)) & ", len: " & $span.len & "]" 139 | 140 | func atEnd*(span: PageSpan): bool {.inline.} = 141 | span.startAddr == span.endAddr 142 | 143 | template hasRunway*(span: var PageSpan): bool = 144 | not span.atEnd() 145 | 146 | # BEWARE! These templates violate the double evaluation 147 | # safety measures in order to produce better inlined 148 | # code - `var` helps prevent misuse 149 | template advance*(span: var PageSpan, numberOfBytes: Natural = 1) = 150 | span.startAddr = offset(span.startAddr, numberOfBytes) 151 | 152 | template split*(span: var PageSpan, pos: Natural): PageSpan = 153 | let other = PageSpan(startAddr: span.startAddr, endAddr: offset(span.startAddr, pos)) 154 | span.advance(pos) 155 | other 156 | 157 | template read*(span: var PageSpan): byte = 158 | let b = span.startAddr[] 159 | span.advance(1) 160 | b 161 | 162 | func read*(span: var PageSpan, val: var openArray[byte]) {.inline.} = 163 | if val.len > 0: # avoid accessing addr val[0] when it's empty 164 | copyMem(addr val[0], span.startAddr, val.len) 165 | span.advance(val.len) 166 | 167 | func write*(span: var PageSpan, val: openArray[byte]) {.inline.} = 168 | if val.len > 0: # avoid accessing addr val[0] when it's empty 169 | copyMem(span.startAddr, unsafeAddr val[0], val.len) 170 | span.advance(val.len) 171 | 172 | template write*(span: var PageSpan, val: byte) = 173 | span.startAddr[] = val 174 | span.startAddr = offset(span.startAddr, 1) 175 | 176 | template data*(span: var PageSpan): var openArray[byte] = 177 | makeOpenArray(span.startAddr, span.len()) 178 | 179 | template allocationStart*(page: PageRef): ptr byte = 180 | baseAddr page.store[] 181 | 182 | func readableStart*(page: PageRef): ptr byte {.inline.} = 183 | offset(page.allocationStart(), page.consumedTo) 184 | 185 | func readableEnd*(page: PageRef): ptr byte {.inline.} = 186 | offset(page.allocationStart(), page.writtenTo) 187 | 188 | func reservedEnd*(page: PageRef): ptr byte {.inline.} = 189 | offset(page.allocationStart(), page.reservedTo) 190 | 191 | template writableStart*(page: PageRef): ptr byte = 192 | readableEnd(page) 193 | 194 | func allocationEnd*(page: PageRef): ptr byte {.inline.} = 195 | offset(page.allocationStart(), page.store[].len) 196 | 197 | func reserved*(page: PageRef): bool {.inline.} = 198 | page.reservedTo > 0 199 | 200 | func len*(page: PageRef): Natural {.inline.} = 201 | ## The number of bytes that can be read from this page, ie what would be 202 | ## returned by `consume`. 203 | page.writtenTo - page.consumedTo 204 | 205 | func capacity*(page: PageRef): Natural {.inline.} = 206 | ## The amount of bytes that can be written to this page, ie what would be 207 | ## returned by `prepare`. 208 | page.store[].len - page.writtenTo 209 | 210 | func recycledCapacity(page: PageRef, isSingle: bool): Natural {.inline.} = 211 | if isSingle and page.consumedTo == page.writtenTo: 212 | page.store[].len 213 | else: 214 | page.store[].len - page.writtenTo 215 | 216 | func recycle(page: PageRef, isSingle: bool): Natural = 217 | # To recycle, we must have committed all reservations and consumed all data 218 | # leading up to this buffer 219 | if isSingle and page.consumedTo == page.writtenTo: 220 | page.consumedTo = 0 221 | page.writtenTo = 0 222 | page.store[].len 223 | else: 224 | page.store[].len - page.writtenTo 225 | 226 | template data*(pageParam: PageRef): openArray[byte] = 227 | ## Currnet input sequence, or what would be returned by `consume` 228 | let page = pageParam 229 | var baseAddr = cast[ptr UncheckedArray[byte]](allocationStart(page)) 230 | toOpenArray(baseAddr, page.consumedTo, page.writtenTo - 1) 231 | 232 | func prepare*(page: PageRef): PageSpan = 233 | ## Return a span representing the output sequence of this page, ie the 234 | ## of space available for writing. 235 | ## 236 | ## After writing data to the span, `commit` should be called with the number 237 | ## of bytes written (or the advanced span). 238 | fsAssert not page.reserved 239 | let baseAddr = allocationStart(page) 240 | PageSpan( 241 | startAddr: offset(baseAddr, page.writtenTo), 242 | endAddr: offset(baseAddr, page.store[].len), 243 | ) 244 | 245 | func prepare*(page: PageRef, len: Natural): PageSpan = 246 | ## Return a span representing the output sequence of this page, ie the 247 | ## of space available for writing, limited to `len` bytes. 248 | ## 249 | ## After writing data to the span, `commit` should be called with the number 250 | ## of bytes written (or the advanced span). 251 | ## 252 | ## `len` must not be greater than `page.capacity()`. 253 | fsAssert not page.reserved 254 | fsAssert len <= page.capacity() 255 | 256 | let 257 | baseAddr = allocationStart(page) 258 | startAddr = offset(baseAddr, page.writtenTo) 259 | endAddr = offset(startAddr, len) 260 | PageSpan(startAddr: startAddr, endAddr: endAddr) 261 | 262 | func contains*(page: PageRef, address: ptr byte): bool {.inline.} = 263 | address >= page.writableStart() and address <= page.allocationEnd() 264 | 265 | func commit*(page: PageRef) {.inline.} = 266 | ## Mark the span returned by `prepare` as committed, allowing it to be 267 | ## accessed by `consume` 268 | page.reservedTo = 0 269 | page.writtenTo = page.store[].len() 270 | 271 | func commit*(page: PageRef, len: Natural) {.inline.} = 272 | ## Mark `len` prepared bytes as committed, allowing them to be accessed by 273 | ## `consume`. `len` may be fewer bytes than were returned by `prepare`. 274 | fsAssert len <= page.capacity() 275 | page.reservedTo = 0 276 | page.writtenTo += len 277 | 278 | func consume*(page: PageRef, maxLen: Natural): PageSpan = 279 | ## Consume up to `maxLen` bytes committed to this page returning the 280 | ## corresponding span. May return a span shorter than `maxLen`. 281 | ## 282 | ## The span remains valid until the next consume call. 283 | let 284 | startAddr = page.readableStart() 285 | bytes = min(page.len, maxLen) 286 | endAddr = offset(startAddr, bytes) 287 | 288 | page.consumedTo += bytes 289 | 290 | PageSpan(startAddr: startAddr, endAddr: endAddr) 291 | 292 | func consume*(page: PageRef): PageSpan = 293 | ## Consume all bytes committed to this page returning the corresponding span. 294 | ## 295 | ## The span remains valid until the next consume call. 296 | 297 | # Although page.consumedTo == page.writtenTo after this operation, we don't 298 | # recycle the page just yet since `unconsume` might give some of the bytes back 299 | page.consume(page.len) 300 | 301 | func unconsume(page: PageRef, len: int) = 302 | ## Return bytes from the last `consume` call to the Page, making them available 303 | ## for reading again. 304 | fsAssert len <= page.consumedTo, "cannot unconsume more bytes than were consumed" 305 | page.consumedTo -= len 306 | 307 | func init*(_: type PageRef, store: ref seq[byte], pos: int): PageRef = 308 | fsAssert store != nil and store[].len > 0 309 | fsAssert pos <= store[].len 310 | PageRef(store: store, consumedTo: pos, writtenTo: pos) 311 | 312 | func init*(_: type PageRef, pageSize: Natural): PageRef = 313 | fsAssert pageSize > 0 314 | PageRef(store: allocRef newSeqUninit[byte](pageSize)) 315 | 316 | func nextAlignedSize*(buffers: PageBuffers, len: Natural): Natural = 317 | nextAlignedSize(len, buffers.pageSize) 318 | 319 | func capacity*(buffers: PageBuffers): int = 320 | if buffers.queue.len > 0: 321 | buffers.queue.peekLast.recycledCapacity(buffers.queue.len == 1) 322 | else: 323 | 0 324 | 325 | func prepare*(buffers: PageBuffers, minLen: Natural = 1): PageSpan = 326 | ## Return a contiguous span of at least `minLen` bytes, switching to a 327 | ## new page if need be but otherwise returning the largest possible contiguous 328 | ## memory area available for writing. 329 | ## 330 | ## The span returned by `prepare` remains valid until the next call to either 331 | ## `reserve` or `commit` and is guaranteed to contain at least `capacity` or 332 | ## `minLen` bytes, whichever is larger. 333 | ## 334 | ## Calling prepare with `minLen` > 1 may result in space being wasted when the 335 | ## desired allocation does not not fit in the current page. Use `prepare()` to 336 | ## avoid this situation or make the initial call with your best guess of the 337 | ## total maximum memory that will be needed. 338 | ## 339 | ## Calling `prepare` multiple times without a `commit` in between will result 340 | ## in the same span being returned. 341 | 342 | # When there's only one page that is empty, we can be assume there are no 343 | # reservations and it is therefore safe to update `writtenTo`. 344 | if buffers.queue.len() == 0 or 345 | buffers.queue.peekLast().recycle(buffers.queue.len() == 1) < minLen: 346 | # Create a new buffer for the desired runway, ending the current buffer 347 | # potentially without using its entire space because existing write 348 | # cursors may point to the memory inside it. 349 | # In git history, one can find code that moves data from the existing page 350 | # data to the new buffer - this would be a good idea if it wasn't for the 351 | # fact that it would invalidate pointers to the previous buffer. 352 | buffers.queue.addLast PageRef.init(buffers.nextAlignedSize(minLen)) 353 | 354 | buffers.queue.peekLast().prepare() 355 | 356 | func reserve*(buffers: PageBuffers, len: Natural): PageSpan = 357 | ## Reserve a span in a page that is frozen for reading until the span is 358 | ## committed. The reserved span must be committed even if it's not written to, 359 | ## to release subsequent writes for reading. 360 | ## 361 | ## `reserve` is used to create a writeable area whose size and contents are 362 | ## unknown at the time of reserve but whose maximum size is bounded and 363 | ## reasonable. 364 | ## 365 | ## Calling `reserve` multiple times will result in a new span being returned 366 | ## every time - each reserved span must be committed separately. 367 | ## 368 | ## If len is 0, no reservation is made. 369 | if len == 0: 370 | # If the reservation is empty, we'd end up with overlapping pages in the 371 | # buffer which would make finding the span (to commit it) tricky 372 | return PageSpan() 373 | 374 | if buffers.queue.len() == 0 or 375 | buffers.queue.peekLast().recycle(buffers.queue.len() == 1) < len: 376 | buffers.queue.addLast PageRef.init(nextAlignedSize(len, buffers.pageSize)) 377 | 378 | let page = buffers.queue.peekLast() 379 | page.reservedTo = page.writtenTo + len 380 | 381 | # The next `prepare` / `reserve` will go into a fresh part 382 | buffers.queue.addLast PageRef.init(page.store, page.reservedTo) 383 | 384 | let 385 | startAddr = page.writableStart 386 | endAddr = offset(startAddr, len) # Same as reservedTo 387 | 388 | PageSpan(startAddr: startAddr, endAddr: endAddr) 389 | 390 | func commit*(buffers: PageBuffers, len: Natural) = 391 | ## Commit `len` bytes from a span previously given by the last call to `prepare`. 392 | fsAssert buffers.queue.len > 0, "Must call prepare with at least `len` bytes" 393 | buffers.queue.peekLast().commit(len) 394 | 395 | func commit*(buffers: PageBuffers, span: PageSpan) = 396 | ## Commit a span previously return by `prepare` or `reserve`, committing all 397 | ## bytes that were written to the span. 398 | ## 399 | ## Spans received from `reserve` may be committed in any order - the data will 400 | ## be made available to `consume` in `reserve` order as soon as all preceding 401 | ## reservations have been committed. 402 | fsAssert span.endAddr >= span.startAddr, "Buffer overrun when writing span" 403 | 404 | if span.startAddr == nil: # zero-length reservations 405 | return 406 | 407 | var span = span 408 | 409 | for ridx in 1..buffers.queue.len(): 410 | let page = buffers.queue[^ridx] 411 | 412 | if span.startAddr in page: 413 | if ridx < buffers.queue.len(): 414 | # A non-reserved page may share end address with the reserved address of 415 | # the preceding page if the reservation exactly covers the end of a page 416 | # and no allocation has been done yet 417 | let page2 = buffers.queue[^(ridx + 1)] 418 | if page2.reservedTo > 0 and span.endAddr == page2.reservedEnd(): 419 | page2.commit(distance(page2.writableStart(), span.startAddr)) 420 | return 421 | 422 | page.commit(distance(page.writableStart(), span.startAddr)) 423 | return 424 | 425 | fsAssert false, "Could not find page to commit" 426 | 427 | func consumable*(buffers: PageBuffers): int = 428 | ## Total number of bytes ready to be consumed - it may take several calls to 429 | ## `consume` to get all of them! 430 | var len = 0 431 | for b in buffers.queue: 432 | len += b.len 433 | 434 | if b.reserved(): 435 | break 436 | len 437 | 438 | func consume*(buffers: PageBuffers, maxLen = int.high()): PageSpan = 439 | ## Return a span representing up to `maxLen` bytes from a single page. The 440 | ## return span is valid until the next call to `consume`, `prepare` or `reserve`. 441 | ## 442 | ## Calling code should make no assumptions about the number of `consume` calls 443 | ## needed to consume a corresponding amount of commits - ie commits may be 444 | ## split and consolidated as optimizations and features are implemented. 445 | while buffers.queue.len > 0: 446 | let page = buffers.queue.peekFirst() 447 | if page.len() > 0: 448 | return page.consume(maxLen) 449 | 450 | if page.reserved(): # The unconsumed parts of the page have been reserved 451 | break 452 | 453 | if buffers.queue.len() == 1: # recycle the last page 454 | break 455 | 456 | discard buffers.queue.popFirst() 457 | 458 | PageSpan() # No ready pages 459 | 460 | iterator consumeAll*(buffers: PageBuffers): PageSpan = 461 | ## Iterate over all spans consuming them in the process. If the iteration is 462 | ## interrupted (for example by a `break` or exception), the last visited span 463 | ## will still be treated as consumed - use `unconsume` to undo. 464 | var span: PageSpan 465 | while true: 466 | span = buffers.consume() 467 | if span.atEnd: 468 | break 469 | 470 | yield span 471 | 472 | func unconsume*(buffers: PageBuffers, bytes: Natural) = 473 | ## Return bytes that were given by the previous call to `consume` 474 | fsAssert buffers.queue.len > 0 475 | buffers.queue.peekFirst.unconsume(bytes) 476 | 477 | func write*(buffers: PageBuffers, val: openArray[byte]) = 478 | if val.len > 0: 479 | var written = 0 480 | if buffers.capacity() > 0: # Use up whatever space is left in the current page 481 | var span = buffers.prepare() 482 | written = min(span.len, val.len) 483 | 484 | span.write(val.toOpenArray(0, written - 1)) 485 | buffers.commit(written) 486 | 487 | if written < val.len: # Put the rest in a fresh page 488 | let remaining = val.len - written 489 | var span = buffers.prepare(remaining) 490 | span.write(val.toOpenArray(written, val.len - 1)) 491 | buffers.commit(remaining) 492 | 493 | func write*(buffers: PageBuffers, val: byte) = 494 | buffers.write([val]) 495 | 496 | func init*(_: type PageBuffers, pageSize: Natural, maxBufferedBytes: Natural = 0): PageBuffers = 497 | fsAssert pageSize > 0 498 | PageBuffers(pageSize: pageSize, maxBufferedBytes: maxBufferedBytes) 499 | 500 | template pageBytes*(page: PageRef): var openArray[byte] {.deprecated: "data".} = 501 | page.data() 502 | 503 | template pageChars*(page: PageRef): var openArray[char] {.deprecated: "data".} = 504 | var baseAddr = cast[ptr UncheckedArray[char]](allocationStart(page)) 505 | toOpenArray(baseAddr, page.consumedTo, page.writtenTo - 1) 506 | 507 | func pageLen*(page: PageRef): Natural {.deprecated: "len".} = 508 | page.len() 509 | 510 | func writableSpan*(page: PageRef): PageSpan {.deprecated: "prepare".} = 511 | page.prepare() 512 | 513 | func fullSpan*(page: PageRef): PageSpan {.deprecated: "data or prepare".} = 514 | let baseAddr = page.allocationStart 515 | PageSpan(startAddr: baseAddr, endAddr: offset(baseAddr, page.store[].len)) 516 | 517 | template bumpPointer*(span: var PageSpan, numberOfBytes: Natural = 1) {.deprecated: "advance".}= 518 | span.advance(numberOfBytes) 519 | template writeByte*(span: var PageSpan, val: byte) {.deprecated: "write".} = 520 | span.write(val) 521 | 522 | func obtainReadableSpan*(buffers: PageBuffers, maxLen: Option[Natural]): PageSpan {.deprecated: "consume".} = 523 | buffers.consume(maxLen.get(int.high())) 524 | 525 | func returnReadableSpan*(buffers: PageBuffers, bytes: Natural) {.deprecated: "unconsume".} = 526 | fsAssert buffers.queue.len > 0 527 | buffers.queue.peekFirst.consumedTo -= bytes 528 | 529 | func initPageBuffers*(pageSize: Natural, 530 | maxBufferedBytes: Natural = 0): PageBuffers {.deprecated: "init".} = 531 | if pageSize == 0: 532 | nil 533 | else: 534 | PageBuffers.init(pageSize, maxBufferedBytes) 535 | 536 | func trackWrittenToEnd*(buffers: PageBuffers) {.deprecated: "commit".} = 537 | if buffers != nil and buffers.queue.len > 0: 538 | buffers.commit(buffers.queue.peekLast.capacity()) 539 | 540 | func trackWrittenTo*(buffers: PageBuffers, spanHeadPos: ptr byte) {.deprecated: "commit".} = 541 | # Compatibility hack relying on commit ignoring `endAddr` for now 542 | if buffers != nil and spanHeadPos != nil: 543 | buffers.commit(PageSpan(startAddr: spanHeadPos)) 544 | 545 | template allocWritablePage*(pageSize: Natural, writtenToParam: Natural = 0): auto {.deprecated: "PageRef.init".} = 546 | PageRef(store: allocRef newSeqUninit[byte](pageSize), 547 | writtenTo: writtenToParam) 548 | 549 | func addWritablePage*(buffers: PageBuffers, pageSize: Natural): PageRef {.deprecated: "prepare".} = 550 | trackWrittenToEnd(buffers) 551 | result = allocWritablePage(pageSize) 552 | buffers.queue.addLast result 553 | 554 | func getWritablePage*(buffers: PageBuffers, 555 | preferredSize: Natural): PageRef {.deprecated: "prepare".} = 556 | if buffers.queue.len > 0: 557 | let lastPage = buffers.queue.peekLast 558 | if lastPage.writtenTo < lastPage.store[].len: 559 | return lastPage 560 | 561 | return addWritablePage(buffers, preferredSize) 562 | 563 | func addWritablePage*(buffers: PageBuffers): PageRef {.deprecated: "prepare".} = 564 | buffers.addWritablePage(buffers.pageSize) 565 | 566 | template getWritableSpan*(buffers: PageBuffers): PageSpan {.deprecated: "prepare".} = 567 | let page = getWritablePage(buffers, buffers.pageSize) 568 | writableSpan(page) 569 | 570 | func appendUnbufferedWrite*(buffers: PageBuffers, 571 | src: pointer, srcLen: Natural) {.deprecated: "write".} = 572 | if srcLen > 0: 573 | buffers.write(makeOpenArray(cast[ptr byte](src), srcLen)) 574 | 575 | func ensureRunway*(buffers: PageBuffers, 576 | currentHeadPos: var PageSpan, 577 | neededRunway: Natural) {.deprecated: "prepare".} = 578 | # End writing to the current buffer (if any) and create a new one of the given 579 | # length 580 | if currentHeadPos.startAddr != nil: 581 | buffers.commit(currentHeadPos) 582 | currentHeadPos = buffers.prepare(neededRunway) 583 | 584 | let page = 585 | if currentHeadPos.startAddr == nil: 586 | # This is a brand new stream, just like we recomend. 587 | buffers.addWritablePage(neededRunway) 588 | else: 589 | # Create a new buffer for the desired runway, ending the current buffer 590 | # potentially without using its entire space because existing write 591 | # cursors may point to the memory inside it. 592 | # In git history, one can find code that moves data from the existing page 593 | # data to the new buffer - this is not a bad idea in general but breaks 594 | # when write cursors are in play 595 | trackWrittenTo(buffers, currentHeadPos.startAddr) 596 | let page = allocWritablePage(neededRunway) 597 | buffers.queue.addLast page 598 | page 599 | 600 | currentHeadPos = page.fullSpan 601 | 602 | template len*(buffers: PageBuffers): Natural = 603 | buffers.queue.len 604 | 605 | func totalBufferedBytes*(buffers: PageBuffers): Natural {.deprecated: "consumable".} = 606 | for i in 0 ..< buffers.queue.len: 607 | result += buffers.queue[i].pageLen 608 | 609 | func canAcceptWrite*(buffers: PageBuffers, writeSize: Natural): bool {.deprecated: "Unimplemented".} = 610 | true or # TODO Remove this line 611 | buffers.queue.len == 0 or 612 | buffers.maxBufferedBytes == 0 or 613 | buffers.totalBufferedBytes < buffers.maxBufferedBytes 614 | 615 | template popFirst*(buffers: PageBuffers): PageRef = 616 | buffers.queue.popFirst 617 | 618 | template `[]`*(buffers: PageBuffers, idx: Natural): PageRef = 619 | buffers.queue[idx] 620 | 621 | func splitLastPageAt*(buffers: PageBuffers, address: ptr byte) {.deprecated: "reserve".} = 622 | var 623 | topPage = buffers.queue.peekLast 624 | splitPosition = distance(topPage.allocationStart, address) 625 | newPage = PageRef( 626 | store: topPage.store, 627 | consumedTo: splitPosition, 628 | writtenTo: splitPosition) 629 | 630 | topPage.writtenTo = splitPosition 631 | buffers.queue.addLast newPage 632 | 633 | iterator consumePages*(buffers: PageBuffers): PageRef {.deprecated: "consumeAll".} = 634 | fsAssert buffers != nil 635 | 636 | while buffers.queue.len > 0: 637 | var page = peekFirst(buffers.queue) 638 | if page.len > 0: 639 | # TODO: what if the body throws an exception? 640 | # Should we do anything with the consumed page? 641 | yield page 642 | 643 | if page.reserved(): 644 | discard page.consume() 645 | break # Stop at the first active reservation 646 | 647 | if buffers.len == 1: 648 | # This was the last page - recycle it 649 | discard page.consume() 650 | break 651 | 652 | discard buffers.queue.popFirst 653 | 654 | iterator consumePageBuffers*(buffers: PageBuffers): (ptr byte, Natural) {.deprecated: "consumeAll".} = 655 | for page in consumePages(buffers): 656 | yield (page.readableStart, page.len()) 657 | 658 | template charsToBytes*( 659 | chars: openArray[char] 660 | ): untyped {.deprecated: "multiple evaluation!".} = 661 | chars.toOpenArrayByte(0, chars.len - 1) 662 | 663 | type 664 | ReadFlag* = enum 665 | partialReadIsEof 666 | zeroReadIsNotEof 667 | 668 | ReadFlags* = set[ReadFlag] 669 | 670 | template implementSingleRead*(buffersParam: PageBuffers, 671 | dstParam: pointer, 672 | dstLenParam: Natural, 673 | flags: static ReadFlags, 674 | readStartVar, readLenVar, 675 | readBlock: untyped): Natural = 676 | var 677 | buffers = buffersParam 678 | readStartVar = dstParam 679 | readLenVar = dstLenParam 680 | bytesRead: Natural 681 | 682 | if readStartVar != nil: 683 | bytesRead = readBlock 684 | else: 685 | var span = buffers.prepare(buffers.nextAlignedSize(readLenVar)) 686 | 687 | readStartVar = span.startAddr 688 | readLenVar = span.len 689 | 690 | # readBlock may raise meaning that the prepared span might be get recycled 691 | # in a future read, even if readBlock wrote some data in it - we have no 692 | # way of knowing how much though! 693 | bytesRead = readBlock 694 | buffers.commit(bytesRead) 695 | 696 | if (bytesRead == 0 and zeroReadIsNotEof notin flags) or 697 | (partialReadIsEof in flags and bytesRead < readLenVar): 698 | buffers.eofReached = true 699 | 700 | bytesRead 701 | -------------------------------------------------------------------------------- /faststreams/outputs.nim: -------------------------------------------------------------------------------- 1 | ## Please note that the use of unbuffered streams comes with a number 2 | ## of restrictions: 3 | ## 4 | ## * Delayed writes are not supported. 5 | ## * Output consuming operations such as `getOutput`, `consumeOutputs` and 6 | ## `consumeContiguousOutput` should not be used with them. 7 | ## * They cannot participate as intermediate steps in pipelines. 8 | 9 | import 10 | deques, typetraits, 11 | stew/[ptrops, strings], 12 | buffers, async_backend 13 | 14 | export 15 | buffers, CloseBehavior 16 | 17 | {.pragma: iocall, nimcall, gcsafe, raises: [IOError].} 18 | 19 | when not declared(newSeqUninit): # nim 2.2+ 20 | template newSeqUninit[T: byte](len: int): seq[byte] = 21 | newSeqUninitialized[byte](len) 22 | 23 | when fsAsyncSupport: 24 | # Circular type refs prevent more targeted `when` 25 | type 26 | OutputStream* = ref object of RootObj 27 | vtable*: ptr OutputStreamVTable # This is nil for any memory output 28 | buffers*: PageBuffers # This is nil for unsafe memory outputs 29 | span*: PageSpan 30 | spanEndPos*: Natural 31 | extCursorsCount: int 32 | closeFut: Future[void] # This is nil before `close` is called 33 | when debugHelpers: 34 | name*: string 35 | 36 | AsyncOutputStream* {.borrow: `.`.} = distinct OutputStream 37 | 38 | WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural) {.iocall.} 39 | WriteAsyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural): Future[void] {.iocall.} 40 | FlushSyncProc* = proc (s: OutputStream) {.iocall.} 41 | FlushAsyncProc* = proc (s: OutputStream): Future[void] {.iocall.} 42 | CloseSyncProc* = proc (s: OutputStream) {.iocall.} 43 | CloseAsyncProc* = proc (s: OutputStream): Future[void] {.iocall.} 44 | 45 | OutputStreamVTable* = object 46 | writeSync*: WriteSyncProc 47 | writeAsync*: WriteAsyncProc 48 | flushSync*: FlushSyncProc 49 | flushAsync*: FlushAsyncProc 50 | closeSync*: CloseSyncProc 51 | closeAsync*: CloseAsyncProc 52 | 53 | MaybeAsyncOutputStream* = OutputStream | AsyncOutputStream 54 | 55 | else: 56 | type 57 | OutputStream* = ref object of RootObj 58 | vtable*: ptr OutputStreamVTable # This is nil for any memory output 59 | buffers*: PageBuffers # This is nil for unsafe memory outputs 60 | span*: PageSpan 61 | spanEndPos*: Natural 62 | extCursorsCount: int 63 | when debugHelpers: 64 | name*: string 65 | 66 | WriteSyncProc* = proc (s: OutputStream, src: pointer, srcLen: Natural) {.iocall.} 67 | FlushSyncProc* = proc (s: OutputStream) {.iocall.} 68 | CloseSyncProc* = proc (s: OutputStream) {.iocall.} 69 | 70 | OutputStreamVTable* = object 71 | writeSync*: WriteSyncProc 72 | flushSync*: FlushSyncProc 73 | closeSync*: CloseSyncProc 74 | 75 | MaybeAsyncOutputStream* = OutputStream 76 | 77 | type 78 | WriteCursor* = object 79 | span: PageSpan 80 | spill: PageSpan 81 | stream: OutputStream 82 | 83 | LayeredOutputStream* = ref object of OutputStream 84 | destination*: OutputStream 85 | allowWaitFor*: bool 86 | 87 | OutputStreamHandle* = object 88 | s*: OutputStream 89 | 90 | VarSizeWriteCursor* = distinct WriteCursor 91 | 92 | FileOutputStream = ref object of OutputStream 93 | file: File 94 | allowAsyncOps: bool 95 | 96 | VmOutputStream = ref object of OutputStream 97 | data: seq[byte] 98 | 99 | template Sync*(s: OutputStream): OutputStream = s 100 | 101 | when fsAsyncSupport: 102 | template Async*(s: OutputStream): AsyncOutputStream = AsyncOutputStream(s) 103 | 104 | template Sync*(s: AsyncOutputStream): OutputStream = OutputStream(s) 105 | template Async*(s: AsyncOutputStream): AsyncOutputStream = s 106 | 107 | proc disconnectOutputDevice(s: OutputStream) = 108 | if s.vtable != nil: 109 | when fsAsyncSupport: 110 | if s.vtable.closeAsync != nil: 111 | s.closeFut = s.vtable.closeAsync(s) 112 | elif s.vtable.closeSync != nil: 113 | s.vtable.closeSync(s) 114 | else: 115 | if s.vtable.closeSync != nil: 116 | s.vtable.closeSync(s) 117 | 118 | s.vtable = nil 119 | 120 | when fsAsyncSupport: 121 | template disconnectOutputDevice(s: AsyncOutputStream) = 122 | disconnectOutputDevice OutputStream(s) 123 | 124 | template prepareSpan(s: OutputStream, minLen: int) = 125 | s.span = s.buffers.prepare(s.buffers.nextAlignedSize(minLen)) 126 | 127 | # Count the full span so that when span.len decreases as data gets written, 128 | # we can compute the total number of written bytes without having to 129 | # update another counter at each write 130 | s.spanEndPos += s.span.len 131 | 132 | template commitSpan(s: OutputStream, span: PageSpan) = 133 | # Adjust position to discount the uncommitted bytes, now that the span is 134 | # being reset 135 | s.spanEndPos -= span.len 136 | s.buffers.commit(span) 137 | span.reset() 138 | 139 | template flushImpl(s: OutputStream, awaiter, writeOp, flushOp: untyped) = 140 | fsAssert s.extCursorsCount == 0 141 | if s.vtable != nil: 142 | if s.buffers != nil: 143 | s.commitSpan(s.span) 144 | awaiter s.vtable.writeOp(s, nil, 0) 145 | 146 | if s.vtable.flushOp != nil: 147 | awaiter s.vtable.flushOp(s) 148 | 149 | proc flush*(s: OutputStream) = 150 | flushImpl(s, noAwait, writeSync, flushSync) 151 | 152 | proc close*(s: OutputStream, 153 | behavior = dontWaitAsyncClose) 154 | {.raises: [IOError].} = 155 | if s == nil: 156 | return 157 | 158 | flush s 159 | disconnectOutputDevice(s) 160 | when fsAsyncSupport: 161 | if s.closeFut != nil: 162 | fsTranslateErrors "Stream closing failed": 163 | if behavior == waitAsyncClose: 164 | waitFor s.closeFut 165 | else: 166 | asyncCheck s.closeFut 167 | 168 | template closeNoWait*(sp: MaybeAsyncOutputStream) = 169 | ## Close the stream without waiting even if's async. 170 | ## This operation will use `asyncCheck` internally to detect unhandled 171 | ## errors from the closing operation. 172 | close(InputStream(s), dontWaitAsyncClose) 173 | 174 | # TODO 175 | # The destructors are currently disabled because they seem to cause 176 | # mysterious segmentation faults related to corrupted GC internal 177 | # data structures. 178 | #[ 179 | proc `=destroy`*(h: var OutputStreamHandle) {.raises: [].} = 180 | if h.s != nil: 181 | if h.s.vtable != nil and h.s.vtable.closeSync != nil: 182 | try: 183 | h.s.vtable.closeSync(h.s) 184 | except IOError: 185 | # Since this is a destructor, there is not much we can do here. 186 | # If the user wanted to handle the error, they would have called 187 | # `close` manually. 188 | discard # TODO 189 | # h.s = nil 190 | ]# 191 | 192 | converter implicitDeref*(h: OutputStreamHandle): OutputStream = 193 | h.s 194 | 195 | template makeHandle*(sp: OutputStream): OutputStreamHandle = 196 | let s = sp 197 | OutputStreamHandle(s: s) 198 | 199 | proc memoryOutput*(pageSize = defaultPageSize): OutputStreamHandle = 200 | when nimvm: 201 | makeHandle VmOutputStream(data: @[]) 202 | else: 203 | fsAssert pageSize > 0 204 | # We are not creating an initial output page, because `ensureRunway` 205 | # can determine the most appropriate size. 206 | makeHandle OutputStream(buffers: PageBuffers.init(pageSize)) 207 | 208 | proc unsafeMemoryOutput*(buffer: pointer, len: Natural): OutputStreamHandle = 209 | let buffer = cast[ptr byte](buffer) 210 | 211 | makeHandle OutputStream( 212 | span: PageSpan(startAddr: buffer, endAddr: offset(buffer, len)), 213 | spanEndPos: len) 214 | 215 | proc ensureRunway*(s: OutputStream, neededRunway: Natural) = 216 | ## The hint provided in `ensureRunway` overrides any previous 217 | ## hint specified at stream creation with `pageSize`. 218 | let runway = s.span.len 219 | 220 | if neededRunway > runway: 221 | # If you use an unsafe memory output, you must ensure that 222 | # it will have a large enough size to hold the data you are 223 | # feeding to it. 224 | fsAssert s.buffers != nil, "Unsafe memory output of insufficient size" 225 | s.commitSpan(s.span) 226 | s.prepareSpan(neededRunway) 227 | 228 | when fsAsyncSupport: 229 | template ensureRunway*(s: AsyncOutputStream, neededRunway: Natural) = 230 | ensureRunway OutputStream(s), neededRunway 231 | 232 | template implementWrites*(buffersParam: PageBuffers, 233 | srcParam: pointer, 234 | srcLenParam: Natural, 235 | dstDesc: static string, 236 | writeStartVar, writeLenVar, 237 | writeBlock: untyped) = 238 | let 239 | buffers = buffersParam 240 | writeStartVar = srcParam 241 | writeLenVar = srcLenParam 242 | 243 | template raiseError = 244 | raise newException(IOError, "Failed to write all bytes to " & dstDesc) 245 | 246 | if buffers != nil: 247 | for span in s.buffers.consumeAll(): 248 | let 249 | writeStartVar = span.startAddr 250 | writeLenVar = span.len 251 | bytesWritten = writeBlock 252 | # TODO: Can we repair the buffers here? 253 | if bytesWritten != writeLenVar: raiseError() 254 | 255 | if writeLenVar > 0: 256 | fsAssert writeStartVar != nil 257 | let bytesWritten = writeBlock 258 | if bytesWritten != writeLenVar: raiseError() 259 | 260 | proc writeFileSync(s: OutputStream, src: pointer, srcLen: Natural) 261 | {.iocall.} = 262 | var file = FileOutputStream(s).file 263 | 264 | implementWrites(s.buffers, src, srcLen, "FILE", 265 | writeStartAddr, writeLen): 266 | file.writeBuffer(writeStartAddr, writeLen) 267 | 268 | proc flushFileSync(s: OutputStream) 269 | {.iocall.} = 270 | flushFile FileOutputStream(s).file 271 | 272 | proc closeFileSync(s: OutputStream) 273 | {.iocall.} = 274 | close FileOutputStream(s).file 275 | 276 | when fsAsyncSupport: 277 | proc writeFileAsync(s: OutputStream, src: pointer, srcLen: Natural): Future[void] 278 | {.iocall.} = 279 | fsAssert FileOutputStream(s).allowAsyncOps 280 | writeFileSync(s, src, srcLen) 281 | result = newFuture[void]() 282 | fsTranslateErrors "Unexpected exception from merely completing a future": 283 | result.complete() 284 | 285 | proc flushFileAsync(s: OutputStream): Future[void] 286 | {.iocall.} = 287 | fsAssert FileOutputStream(s).allowAsyncOps 288 | flushFile FileOutputStream(s).file 289 | result = newFuture[void]() 290 | fsTranslateErrors "Unexpected exception from merely completing a future": 291 | result.complete() 292 | 293 | const fileOutputVTable = when fsAsyncSupport: 294 | OutputStreamVTable( 295 | writeSync: writeFileSync, 296 | writeAsync: writeFileAsync, 297 | flushSync: flushFileSync, 298 | flushAsync: flushFileAsync, 299 | closeSync: closeFileSync) 300 | else: 301 | OutputStreamVTable( 302 | writeSync: writeFileSync, 303 | flushSync: flushFileSync, 304 | closeSync: closeFileSync) 305 | 306 | template vtableAddr*(vtable: OutputStreamVTable): ptr OutputStreamVTable = 307 | # https://github.com/nim-lang/Nim/issues/22389 308 | when (NimMajor, NimMinor, NimPatch) >= (2, 0, 12): 309 | addr vtable 310 | else: 311 | let vtable2 {.global.} = vtable 312 | {.noSideEffect.}: 313 | unsafeAddr vtable2 314 | 315 | proc fileOutput*(f: File, 316 | pageSize = defaultPageSize, 317 | allowAsyncOps = false): OutputStreamHandle 318 | {.raises: [IOError].} = 319 | makeHandle FileOutputStream( 320 | vtable: vtableAddr fileOutputVTable, 321 | buffers: if pageSize > 0: PageBuffers.init(pageSize) else: nil, 322 | file: f, 323 | allowAsyncOps: allowAsyncOps) 324 | 325 | proc fileOutput*(filename: string, 326 | fileMode: FileMode = fmWrite, 327 | pageSize = defaultPageSize, 328 | allowAsyncOps = false): OutputStreamHandle 329 | {.raises: [IOError].} = 330 | fileOutput(open(filename, fileMode), pageSize, allowAsyncOps) 331 | 332 | proc pos*(s: OutputStream): int = 333 | s.spanEndPos - s.span.len 334 | 335 | when fsAsyncSupport: 336 | template pos*(s: AsyncOutputStream): int = 337 | pos OutputStream(s) 338 | 339 | proc getBuffers*(s: OutputStream): PageBuffers = 340 | fsAssert s.buffers != nil 341 | s.commitSpan(s.span) 342 | 343 | s.buffers 344 | 345 | proc recycleBuffers*(s: OutputStream, buffers: PageBuffers) = 346 | if buffers != nil: 347 | s.buffers = buffers 348 | 349 | let len = buffers.queue.len 350 | if len > 0: 351 | if len > 1: 352 | buffers.queue.shrink(fromLast = len - 1) 353 | 354 | let bufferPage = buffers.queue[0] 355 | bufferPage.writtenTo = 0 356 | bufferPage.consumedTo = 0 357 | 358 | s.span = bufferPage.prepare() 359 | s.spanEndPos = s.span.len 360 | return 361 | else: 362 | s.buffers = PageBuffers.init(defaultPageSize) 363 | 364 | s.span = default(PageSpan) 365 | s.spanEndPos = 0 366 | 367 | # Pre-conditions for `drainAllBuffers(Sync/Async)` 368 | # * The cursor has reached the current span end 369 | # * We are working with a vtable-enabled stream 370 | # 371 | # Post-conditions: 372 | # * All completed pages are written 373 | 374 | proc drainAllBuffersSync(s: OutputStream, buf: pointer, bufSize: Natural) = 375 | s.vtable.writeSync(s, buf, bufSize) 376 | 377 | when fsAsyncSupport: 378 | proc drainAllBuffersAsync(s: OutputStream, buf: pointer, bufSize: Natural) {.async.} = 379 | fsAwait s.vtable.writeAsync(s, buf, bufSize) 380 | 381 | proc createCursor(s: OutputStream, span, spill: PageSpan): WriteCursor = 382 | inc s.extCursorsCount 383 | 384 | WriteCursor(stream: s, span: span, spill: spill) 385 | 386 | proc delayFixedSizeWrite*(s: OutputStream, cursorSize: Natural): WriteCursor = 387 | let runway = s.span.len 388 | 389 | if s.buffers == nil: 390 | fsAssert cursorSize <= runway 391 | # Without buffers, we'll simply mark the part of the span as written and 392 | # move on 393 | createCursor(s, s.span.split(cursorSize), PageSpan()) 394 | else: 395 | # Commit what's already been written to the local span, in case a flush 396 | # happens 397 | s.commitSpan(s.span) 398 | 399 | let capacity = s.buffers.capacity() 400 | if cursorSize <= capacity or capacity == 0: 401 | # A single page buffer will be enough to cover the cursor 402 | var span = s.buffers.reserve(cursorSize) 403 | s.spanEndPos += span.len 404 | 405 | createCursor(s, span, PageSpan()) 406 | else: 407 | # There is some capacity in the page buffer but not enough for the full 408 | # cursor - create a split cursor that references two memory areas 409 | var 410 | span = s.buffers.reserve(capacity) 411 | spill = s.buffers.reserve(cursorSize - capacity) 412 | s.spanEndPos += span.len + spill.len 413 | 414 | 415 | createCursor(s, span, spill) 416 | 417 | proc delayVarSizeWrite*(s: OutputStream, maxSize: Natural): VarSizeWriteCursor = 418 | ## Please note that using variable sized writes are not supported 419 | ## for unbuffered streams and unsafe memory inputs. 420 | fsAssert s.buffers != nil 421 | 422 | VarSizeWriteCursor s.delayFixedSizeWrite(maxSize) 423 | 424 | proc getWritableBytesOnANewPage(s: OutputStream, spanSize: Natural): ptr byte = 425 | fsAssert s.buffers != nil 426 | s.commitSpan(s.span) 427 | 428 | s.prepareSpan(spanSize) 429 | 430 | s.span.startAddr 431 | 432 | template getWritableBytes*(sp: OutputStream, spanSizeParam: Natural): var openArray[byte] = 433 | ## Returns a contiguous range of memory that the caller is free to populate fully 434 | ## or partially. The caller indicates how many bytes were written to the openArray 435 | ## by calling `advance(numberOfBytes)` once or multiple times. Advancing the stream 436 | ## past the allocated size is considered a defect. The typical usage pattern of this 437 | ## API looks as follows: 438 | ## 439 | ## stream.advance(myComponent.writeBlock(stream.getWritableBytes(maxBlockSize))) 440 | ## 441 | ## In the example, `writeBlock` would be a function returning the number of bytes 442 | ## written to the openArray. 443 | ## 444 | ## While it's not illegal to issue other writing operations to the stream during 445 | ## the `getWritetableBytes` -> `advance` sequence, doing this is not recommended 446 | ## because it will result in overwriting the same range of bytes. 447 | let 448 | s = sp 449 | spanSize = spanSizeParam 450 | runway = s.span.len 451 | startAddr = if spanSize <= runway: 452 | s.span.startAddr 453 | else: 454 | getWritableBytesOnANewPage(s, spanSize) 455 | 456 | makeOpenArray(startAddr, spanSize) 457 | 458 | proc advance*(s: OutputStream, bytesWrittenToWritableSpan: Natural) = 459 | ## Advance the stream write cursor. 460 | ## Typically used after a previous call to `getWritableBytes`. 461 | fsAssert bytesWrittenToWritableSpan <= s.span.len 462 | s.span.advance(bytesWrittenToWritableSpan) 463 | 464 | template writeToNewSpanImpl(s: OutputStream, b: byte, awaiter, writeOp, drainOp: untyped) = 465 | if s.buffers == nil: 466 | fsAssert s.vtable != nil # This is an unsafe memory output and we've reached 467 | # the end of the buffer which is range violation defect 468 | fsAssert s.vtable.writeOp != nil 469 | awaiter s.vtable.writeOp(s, unsafeAddr b, 1) 470 | elif s.vtable == nil or s.extCursorsCount > 0: 471 | # This is the main cursor of a stream, but we are either not 472 | # ready to flush due to outstanding delayed writes or this is 473 | # just a memory output stream. In both cases, we just need to 474 | # allocate more memory and continue writing: 475 | s.commitSpan(s.span) 476 | s.prepareSpan(s.buffers.pageSize) 477 | write(s.span, b) 478 | else: 479 | s.commitSpan(s.span) 480 | awaiter drainOp(s, nil, 0) 481 | s.prepareSpan(s.buffers.pageSize) 482 | write(s.span, b) 483 | 484 | proc writeToNewSpan(s: OutputStream, b: byte) = 485 | writeToNewSpanImpl(s, b, noAwait, writeSync, drainAllBuffersSync) 486 | 487 | template write*(sp: OutputStream, b: byte) = 488 | when nimvm: 489 | VmOutputStream(sp).data.add(b) 490 | else: 491 | let s = sp 492 | if hasRunway(s.span): 493 | write(s.span, b) 494 | else: 495 | writeToNewSpan(s, b) 496 | 497 | when fsAsyncSupport: 498 | proc write*(sp: AsyncOutputStream, b: byte) = 499 | let s = OutputStream sp 500 | if atEnd(s.span): 501 | s.commitSpan(s.span) 502 | s.prepareSpan() 503 | 504 | writeByte(s.span, b) 505 | 506 | template writeAndWait*(sp: AsyncOutputStream, b: byte) = 507 | let s = OutputStream sp 508 | if hasRunway(s.span): 509 | writeByte(s.span, b) 510 | else: 511 | writeToNewSpanImpl(s, b, fsAwait, writeAsync, drainAllBuffersAsync) 512 | 513 | template write*(s: AsyncOutputStream, x: char) = 514 | write s, byte(x) 515 | 516 | template write*(s: OutputStream|var WriteCursor, x: char) = 517 | bind write 518 | write s, byte(x) 519 | 520 | proc writeToANewPage(s: OutputStream, bytes: openArray[byte]) = 521 | let runway = s.span.len 522 | fsAssert bytes.len > runway 523 | 524 | if runway > 0: 525 | s.span.write(bytes.toOpenArray(0, runway - 1)) 526 | 527 | s.commitSpan(s.span) 528 | s.prepareSpan(bytes.len - runway) 529 | 530 | s.span.write(bytes.toOpenArray(runway, bytes.len - 1)) 531 | 532 | template writeBytesImpl(s: OutputStream, 533 | bytes: openArray[byte], 534 | drainOp: untyped) = 535 | let inputLen = bytes.len 536 | if inputLen == 0: return 537 | 538 | # We have a short inlinable function handling the case when the input is 539 | # short enough to fit in the current page. We'll keep buffering until the 540 | # page is full: 541 | let runway = s.span.len 542 | 543 | if inputLen <= runway: 544 | s.span.write(bytes) 545 | elif s.vtable == nil or s.extCursorsCount > 0: 546 | # We are not ready to flush, so we must create pending pages. 547 | # We'll try to create them as large as possible: 548 | 549 | s.writeToANewPage(bytes) 550 | else: 551 | s.commitSpan(s.span) 552 | drainOp 553 | 554 | proc write*(s: OutputStream, bytes: openArray[byte]) = 555 | when nimvm: 556 | VmOutputStream(s).data.add(bytes) 557 | else: 558 | writeBytesImpl(s, bytes): 559 | drainAllBuffersSync(s, baseAddr(bytes), bytes.len) 560 | 561 | proc write*(s: OutputStream, chars: openArray[char]) = 562 | write s, chars.toOpenArrayByte(0, chars.high()) 563 | 564 | proc write*(s: MaybeAsyncOutputStream, value: string) {.inline.} = 565 | write s, value.toOpenArrayByte(0, value.len - 1) 566 | 567 | proc write*(s: OutputStream, value: cstring) = 568 | for c in value: 569 | write s, c 570 | 571 | template memCopyToBytes(value: auto): untyped = 572 | type T = type(value) 573 | static: assert supportsCopyMem(T) 574 | let valueAddr = unsafeAddr value 575 | makeOpenArray(cast[ptr byte](valueAddr), sizeof(T)) 576 | 577 | proc writeMemCopy*(s: OutputStream, value: auto) = 578 | write s, memCopyToBytes(value) 579 | 580 | when fsAsyncSupport: 581 | proc writeBytesAsyncImpl(sp: OutputStream, 582 | bytes: openArray[byte]): Future[void] = 583 | let s = sp 584 | writeBytesImpl(s, bytes): 585 | return s.vtable.writeAsync(s, baseAddr(bytes), bytes.len) 586 | 587 | proc writeBytesAsyncImpl(s: OutputStream, 588 | chars: openArray[char]): Future[void] = 589 | writeBytesAsyncImpl s, chars.toOpenArrayByte(0, chars.high()) 590 | 591 | proc writeBytesAsyncImpl(s: OutputStream, 592 | str: string): Future[void] = 593 | writeBytesAsyncImpl s, str.toOpenArrayByte(0, str.high()) 594 | 595 | template writeAndWait*(s: OutputStream, value: untyped) = 596 | write s, value 597 | 598 | when fsAsyncSupport: 599 | template writeAndWait*(sp: AsyncOutputStream, value: untyped) = 600 | bind writeBytesAsyncImpl 601 | 602 | let 603 | s = OutputStream sp 604 | f = writeBytesAsyncImpl(s, value) 605 | 606 | if f != nil: 607 | fsAwait(f) 608 | 609 | template writeMemCopyAndWait*(sp: AsyncOutputStream, value: auto) = 610 | writeAndWait(sp, memCopyToBytes(value)) 611 | 612 | proc write*(c: var WriteCursor, bytes: openArray[byte]) = 613 | var remaining = bytes.len 614 | if remaining > 0: 615 | let written = min(remaining, c.span.len) 616 | c.span.write(bytes.toOpenArray(0, written - 1)) 617 | 618 | if c.span.len == 0 and c.stream.buffers != nil: 619 | c.stream.commitSpan(c.span) 620 | 621 | if written < remaining: 622 | # c.span is full - commit it and move to the spill 623 | # Reaching this point implies we have buffers backing the span 624 | remaining -= written 625 | 626 | c.span = c.spill 627 | c.spill.reset() 628 | 629 | c.span.write(bytes.toOpenArray(written, bytes.len - 1)) 630 | 631 | if c.span.len == 0 and c.stream.buffers != nil: 632 | c.stream.commitSpan(c.span) 633 | 634 | proc write*(c: var WriteCursor, b: byte) = 635 | write(c, [b]) 636 | 637 | proc write*(c: var WriteCursor, chars: openArray[char]) = 638 | write(c, chars.toOpenArrayByte(0, chars.len - 1)) 639 | 640 | proc writeMemCopy*[T](c: var WriteCursor, value: T) = 641 | write(c, memCopyToBytes(value)) 642 | 643 | proc write*(c: var WriteCursor, str: string) = 644 | write(c, str.toOpenArrayByte(0, str.len - 1)) 645 | 646 | proc finalize*(c: var WriteCursor) = 647 | fsAssert c.stream.extCursorsCount > 0 648 | 649 | dec c.stream.extCursorsCount 650 | 651 | proc finalWrite*(c: var WriteCursor, data: openArray[byte]) = 652 | c.write(data) 653 | finalize c 654 | 655 | proc write*(c: var VarSizeWriteCursor, b: byte) {.borrow.} 656 | proc write*(c: var VarSizeWriteCursor, bytes: openArray[byte]) {.borrow.} 657 | proc write*(c: var VarSizeWriteCursor, chars: openArray[char]) {.borrow.} 658 | proc writeMemCopy*[T](c: var VarSizeWriteCursor, value: T) = 659 | writeMemCopy(WriteCursor(c), value) 660 | 661 | proc write*(c: var VarSizeWriteCursor, str: string) {.borrow.} 662 | 663 | proc finalize*(c: var VarSizeWriteCursor) = 664 | fsAssert WriteCursor(c).stream.extCursorsCount > 0 665 | 666 | dec WriteCursor(c).stream.extCursorsCount 667 | 668 | if WriteCursor(c).span.startAddr != nil: 669 | WriteCursor(c).stream.commitSpan(WriteCursor(c).span) 670 | 671 | if WriteCursor(c).spill.startAddr != nil: 672 | WriteCursor(c).stream.commitSpan(WriteCursor(c).spill) 673 | 674 | proc finalWrite*(c: var VarSizeWriteCursor, data: openArray[byte]) = 675 | c.write(data) 676 | finalize c 677 | 678 | type 679 | OutputConsumingProc = proc (data: openArray[byte]) 680 | {.gcsafe, raises: [].} 681 | 682 | proc consumeOutputsImpl(s: OutputStream, consumer: OutputConsumingProc) = 683 | fsAssert s.extCursorsCount == 0 and s.buffers != nil 684 | s.commitSpan(s.span) 685 | 686 | for span in s.buffers.consumeAll(): 687 | var span = span # var for template 688 | consumer(span.data()) 689 | 690 | s.spanEndPos = 0 691 | 692 | template consumeOutputs*(s: OutputStream, bytesVar, body: untyped) = 693 | ## Please note that calling `consumeOutputs` on an unbuffered stream 694 | ## or an unsafe memory stream is considered a Defect. 695 | ## 696 | ## Before consuming the outputs, all outstanding delayed writes must 697 | ## be finalized. 698 | proc consumer(bytesVar: openArray[byte]) {.gensym, gcsafe, raises: [].} = 699 | body 700 | 701 | consumeOutputsImpl(s, consumer) 702 | 703 | proc consumeContiguousOutputImpl(s: OutputStream, consumer: OutputConsumingProc) = 704 | fsAssert s.extCursorsCount == 0 and s.buffers != nil 705 | s.commitSpan(s.span) 706 | 707 | if s.buffers.queue.len == 1: 708 | var span = s.buffers.consume() 709 | consumer(span.data()) 710 | else: 711 | var bytes = newSeqUninit[byte](s.pos) 712 | var pos = 0 713 | 714 | if bytes.len > 0: 715 | for span in s.buffers.consumeAll(): 716 | copyMem(addr bytes[pos], span.startAddr, span.len ) 717 | pos += span.len 718 | 719 | consumer(bytes) 720 | 721 | s.spanEndPos = 0 722 | 723 | template consumeContiguousOutput*(s: OutputStream, bytesVar, body: untyped) = 724 | ## Please note that calling `consumeContiguousOutput` on an unbuffered stream 725 | ## or an unsafe memory stream is considered a Defect. 726 | ## 727 | ## Before consuming the output, all outstanding delayed writes must 728 | ## be finalized. 729 | ## 730 | proc consumer(bytesVar: openArray[byte]) {.gensym, gcsafe, raises: [].} = 731 | body 732 | 733 | consumeContiguousOutputImpl(s, consumer) 734 | 735 | template toVMString(x: openArray[byte]): string = 736 | var z = newString(x.len) 737 | for i, c in x: 738 | z[i] = char(c) 739 | z 740 | 741 | proc getOutput*(s: OutputStream, T: type string): string = 742 | ## Consume data written so far to the in-memory page buffer - this operation 743 | ## is meaningful only for `memoryOutput` and leaves the page buffer empty. 744 | ## 745 | ## Please note that calling `getOutput` on an unbuffered stream 746 | ## or an unsafe memory stream is considered a Defect. 747 | ## 748 | ## Before consuming the output, all outstanding delayed writes must be finalized. 749 | ## 750 | when nimvm: 751 | return toVMString(VmOutputStream(s).data) 752 | else: 753 | fsAssert s.extCursorsCount == 0 and s.buffers != nil 754 | s.commitSpan(s.span) 755 | 756 | result = newStringOfCap(s.pos) 757 | s.spanEndPos = 0 758 | 759 | for span in s.buffers.consumeAll(): 760 | when compiles(span.data().toOpenArrayChar(0, len - 1)): 761 | result.add span.data().toOpenArrayChar(0, len - 1) 762 | else: 763 | let p = cast[ptr char](span.startAddr) 764 | result.add makeOpenArray(p, span.len) 765 | 766 | proc getOutput*(s: OutputStream, T: type seq[byte]): seq[byte] = 767 | ## Consume data written so far to the in-memory page buffer - this operation 768 | ## is meaningful only for `memoryOutput` and leaves the page buffer empty. 769 | ## 770 | ## Please note that calling `getOutput` on an unbuffered stream 771 | ## or an unsafe memory stream is considered a Defect. 772 | ## 773 | ## Before consuming the output, all outstanding delayed writes must be finalized. 774 | when nimvm: 775 | return VmOutputStream(s).data 776 | else: 777 | fsAssert s.extCursorsCount == 0 and s.buffers != nil 778 | s.commitSpan(s.span) 779 | 780 | if s.buffers.queue.len == 1: 781 | let page = s.buffers.queue[0] 782 | if page.consumedTo == 0: 783 | # "move" the buffer to the caller - swap works for all nim versions and gcs 784 | result.swap page.store[] 785 | result.setLen page.writtenTo 786 | # We clear the buffers, so the stream will be in pristine state. 787 | # The next write is going to create a fresh new starting page. 788 | s.buffers.queue.clear() 789 | s.spanEndPos = 0 790 | return 791 | 792 | result = newSeqUninit[byte](s.pos) 793 | s.spanEndPos = 0 794 | 795 | var pos = 0 796 | for span in s.buffers.consumeAll(): 797 | copyMem(addr result[pos], span.startAddr, span.len) 798 | pos += span.len 799 | 800 | template getOutput*(s: OutputStream): seq[byte] = 801 | s.getOutput(seq[byte]) 802 | 803 | when fsAsyncSupport: 804 | template getOutput*(s: AsyncOutputStream): seq[byte] = 805 | getOutput OutputStream(s) 806 | 807 | template getOutput*(s: AsyncOutputStream, T: type): untyped = 808 | getOutput OutputStream(s), T 809 | 810 | when fsAsyncSupport: 811 | template flush*(sp: AsyncOutputStream) = 812 | let s = OutputStream sp 813 | flushImpl(s, fsAwait, writeAsync, flushAsync) 814 | 815 | proc flushAsync*(s: AsyncOutputStream) {.async.} = 816 | flush s 817 | 818 | proc close*(sp: AsyncOutputStream) = 819 | let s = OutputStream sp 820 | if s != nil: 821 | flush(Async s) 822 | disconnectOutputDevice(s) 823 | if s.closeFut != nil: 824 | fsAwait s.closeFut 825 | 826 | proc closeAsync*(s: AsyncOutputStream) {.async.} = 827 | close s 828 | -------------------------------------------------------------------------------- /faststreams/inputs.nim: -------------------------------------------------------------------------------- 1 | import 2 | os, memfiles, options, 3 | stew/[ptrops], 4 | async_backend, buffers 5 | 6 | export 7 | options, CloseBehavior 8 | 9 | {.pragma: iocall, nimcall, gcsafe, raises: [IOError].} 10 | 11 | when fsAsyncSupport: 12 | # Circular type refs prevent more targeted `when` 13 | type 14 | InputStream* = ref object of RootObj 15 | vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs 16 | buffers*: PageBuffers # This is nil for unsafe memory inputs 17 | span*: PageSpan 18 | spanEndPos*: Natural 19 | maxBufferedBytes*: Option[Natural] 20 | closeFut*: Future[void] # This is nil before `close` is called 21 | when debugHelpers: 22 | name*: string 23 | 24 | AsyncInputStream* {.borrow: `.`.} = distinct InputStream 25 | 26 | ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural {.iocall.} 27 | ReadAsyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Future[Natural] {.iocall.} 28 | CloseSyncProc* = proc (s: InputStream) {.iocall.} 29 | CloseAsyncProc* = proc (s: InputStream): Future[void] {.iocall.} 30 | GetLenSyncProc* = proc (s: InputStream): Option[Natural] {.iocall.} 31 | 32 | InputStreamVTable* = object 33 | readSync*: ReadSyncProc 34 | closeSync*: CloseSyncProc 35 | getLenSync*: GetLenSyncProc 36 | readAsync*: ReadAsyncProc 37 | closeAsync*: CloseAsyncProc 38 | 39 | MaybeAsyncInputStream* = InputStream | AsyncInputStream 40 | 41 | else: 42 | type 43 | InputStream* = ref object of RootObj 44 | vtable*: ptr InputStreamVTable # This is nil for unsafe memory inputs 45 | buffers*: PageBuffers # This is nil for unsafe memory inputs 46 | span*: PageSpan 47 | spanEndPos*: Natural 48 | maxBufferedBytes*: Option[Natural] 49 | # When using withReadableRange, the amount of bytes we're allowed to 50 | # obtain from buffers (in addition to what's in the span) 51 | 52 | when debugHelpers: 53 | name*: string 54 | 55 | ReadSyncProc* = proc (s: InputStream, dst: pointer, dstLen: Natural): Natural {.iocall.} 56 | CloseSyncProc* = proc (s: InputStream) {.iocall.} 57 | GetLenSyncProc* = proc (s: InputStream): Option[Natural] {.iocall.} 58 | 59 | InputStreamVTable* = object 60 | readSync*: ReadSyncProc 61 | closeSync*: CloseSyncProc 62 | getLenSync*: GetLenSyncProc 63 | 64 | MaybeAsyncInputStream* = InputStream 65 | 66 | type 67 | LayeredInputStream* = ref object of InputStream 68 | source*: InputStream 69 | allowWaitFor*: bool 70 | 71 | InputStreamHandle* = object 72 | s*: InputStream 73 | 74 | MemFileInputStream = ref object of InputStream 75 | file: MemFile 76 | 77 | FileInputStream = ref object of InputStream 78 | file: File 79 | 80 | VmInputStream = ref object of InputStream 81 | data: seq[byte] 82 | pos: int 83 | 84 | template Sync*(s: InputStream): InputStream = s 85 | 86 | when fsAsyncSupport: 87 | template Async*(s: InputStream): AsyncInputStream = AsyncInputStream(s) 88 | 89 | template Sync*(s: AsyncInputStream): InputStream = InputStream(s) 90 | template Async*(s: AsyncInputStream): AsyncInputStream = s 91 | 92 | proc disconnectInputDevice(s: InputStream) {.raises: [IOError].} = 93 | # TODO 94 | # Document the behavior that closeAsync is preferred 95 | if s.vtable != nil: 96 | when fsAsyncSupport: 97 | if s.vtable.closeAsync != nil: 98 | s.closeFut = s.vtable.closeAsync(s) 99 | elif s.vtable.closeSync != nil: 100 | s.vtable.closeSync(s) 101 | else: 102 | if s.vtable.closeSync != nil: 103 | s.vtable.closeSync(s) 104 | s.vtable = nil 105 | 106 | when fsAsyncSupport: 107 | template disconnectInputDevice(s: AsyncInputStream) = 108 | disconnectInputDevice InputStream(s) 109 | 110 | func preventFurtherReading(s: InputStream) = 111 | s.vtable = nil 112 | when nimvm: 113 | discard 114 | else: 115 | # TODO https://github.com/nim-lang/Nim/issues/25066 116 | fsAssert s.span.startAddr <= s.span.endAddr, "Buffer overrun in previous read!" 117 | s.span.endAddr = s.span.startAddr 118 | 119 | when fsAsyncSupport: 120 | template preventFurtherReading(s: AsyncInputStream) = 121 | preventFurtherReading InputStream(s) 122 | 123 | template makeHandle*(sp: InputStream): InputStreamHandle = 124 | InputStreamHandle(s: sp) 125 | 126 | proc close(s: VmInputStream) = 127 | if s == nil: 128 | return 129 | s.pos = s.data.len 130 | 131 | proc close*(s: InputStream, 132 | behavior = dontWaitAsyncClose) 133 | {.raises: [IOError].} = 134 | ## Closes the stream. Any resources associated with the stream 135 | ## will be released and no further reading will be possible. 136 | ## 137 | ## If the underlying input device requires asynchronous closing 138 | ## and `behavior` is set to `waitAsyncClose`, this proc will use 139 | ## `waitFor` to block until the async operation completes. 140 | when nimvm: 141 | close(VmInputStream(s)) 142 | else: 143 | if s == nil: 144 | return 145 | 146 | s.disconnectInputDevice() 147 | s.preventFurtherReading() 148 | when fsAsyncSupport: 149 | if s.closeFut != nil: 150 | fsTranslateErrors "Stream closing failed": 151 | if behavior == waitAsyncClose: 152 | waitFor s.closeFut 153 | else: 154 | asyncCheck s.closeFut 155 | 156 | when fsAsyncSupport: 157 | template close*(sp: AsyncInputStream) = 158 | ## Starts the asychronous closing of the stream and returns a future that 159 | ## tracks the closing operation. 160 | let s = InputStream sp 161 | if s != nil: 162 | disconnectInputDevice(s) 163 | preventFurtherReading(s) 164 | if s.closeFut != nil: 165 | fsAwait s.closeFut 166 | 167 | template closeNoWait*(sp: MaybeAsyncInputStream) = 168 | ## Close the stream without waiting even if's async. 169 | ## This operation will use `asyncCheck` internally to detect unhandled 170 | ## errors from the closing operation. 171 | close(InputStream(s), dontWaitAsyncClose) 172 | 173 | # TODO 174 | # The destructors are currently disabled because they seem to cause 175 | # mysterious segmentation faults related to corrupted GC internal 176 | # data structures. 177 | #[ 178 | proc `=destroy`*(h: var InputStreamHandle) {.raises: [].} = 179 | if h.s != nil: 180 | if h.s.vtable != nil and h.s.vtable.closeSync != nil: 181 | try: 182 | h.s.vtable.closeSync(h.s) 183 | except IOError: 184 | # Since this is a destructor, there is not much we can do here. 185 | # If the user wanted to handle the error, they would have called 186 | # `close` manually. 187 | discard # TODO 188 | # TODO ATTENTION! 189 | # Uncommenting the following line will lead to a GC heap corruption. 190 | # Most likely this leads to Nim collecting some object prematurely. 191 | # h.s = nil 192 | # We work-around the problem through more indirect incapacitatation 193 | # of the stream object: 194 | h.s.preventFurtherReading() 195 | ]# 196 | 197 | converter implicitDeref*(h: InputStreamHandle): InputStream = 198 | ## Any `InputStreamHandle` value can be implicitly converted to an 199 | ## `InputStream` or an `AsyncInputStream` value. 200 | h.s 201 | 202 | template vtableAddr*(vtable: InputStreamVTable): ptr InputStreamVTable = 203 | # https://github.com/nim-lang/Nim/issues/22389 204 | when (NimMajor, NimMinor, NimPatch) >= (2, 0, 12): 205 | addr vtable 206 | else: 207 | let vtable2 {.global.} = vtable 208 | {.noSideEffect.}: 209 | unsafeAddr vtable2 210 | 211 | const memFileInputVTable = InputStreamVTable( 212 | closeSync: proc (s: InputStream) = 213 | try: 214 | close MemFileInputStream(s).file 215 | except OSError as err: 216 | raise newException(IOError, "Failed to close file", err) 217 | , 218 | getLenSync: func (s: InputStream): Option[Natural] = 219 | some s.span.len 220 | ) 221 | 222 | proc memFileInput*(filename: string, mappedSize = -1, offset = 0): InputStreamHandle 223 | {.raises: [IOError].} = 224 | ## Creates an input stream for reading the contents of a memory-mapped file. 225 | ## 226 | ## Using this API will provide better performance than `fileInput`, 227 | ## but this comes at a cost of higher address space usage which may 228 | ## be problematic when working with extremely large files. 229 | ## 230 | ## All parameters are forwarded to Nim's memfiles.open function: 231 | ## 232 | ## ``filename`` 233 | ## The name of the file to read. 234 | ## 235 | ## ``mappedSize`` and ``offset`` 236 | ## can be used to map only a slice of the file. 237 | ## 238 | ## ``offset`` must be multiples of the PAGE SIZE of your OS 239 | ## (usually 4K or 8K, but is unique to your OS) 240 | 241 | # Nim's memfiles module will fail to map an empty file, 242 | # but we don't consider this a problem. The stream will 243 | # be in non-readable state from the start. 244 | try: 245 | let fileSize = getFileSize(filename) 246 | if fileSize == 0: 247 | return makeHandle InputStream() 248 | 249 | let 250 | memFile = memfiles.open(filename, 251 | mode = fmRead, 252 | mappedSize = mappedSize, 253 | offset = offset) 254 | head = cast[ptr byte](memFile.mem) 255 | mappedSize = memFile.size 256 | 257 | makeHandle MemFileInputStream( 258 | vtable: vtableAddr memFileInputVTable, 259 | span: PageSpan( 260 | startAddr: head, 261 | endAddr: offset(head, mappedSize)), 262 | spanEndPos: mappedSize, 263 | file: memFile) 264 | except OSError as err: 265 | raise newException(IOError, err.msg, err) 266 | 267 | func getNewSpan(s: InputStream) = 268 | fsAssert s.buffers != nil 269 | fsAssert s.span.startAddr <= s.span.endAddr, "Buffer overrun in previous read!" 270 | 271 | s.span = s.buffers.consume(s.maxBufferedBytes.get(int.high)) 272 | s.spanEndPos += s.span.len 273 | if s.maxBufferedBytes.isSome(): 274 | s.maxBufferedBytes.get() -= s.span.len 275 | 276 | func getNewSpanOrDieTrying(s: InputStream) = 277 | getNewSpan s 278 | fsAssert s.span.hasRunway 279 | 280 | func readableNow*(s: InputStream): bool = 281 | when nimvm: 282 | VmInputStream(s).pos < VmInputStream(s).data.len 283 | else: 284 | if s.span.hasRunway: return true 285 | getNewSpan s 286 | s.span.hasRunway 287 | 288 | when fsAsyncSupport: 289 | template readableNow*(s: AsyncInputStream): bool = 290 | readableNow InputStream(s) 291 | 292 | proc readOnce*(sp: AsyncInputStream): Future[Natural] {.async.} = 293 | let s = InputStream(sp) 294 | fsAssert s.buffers != nil and s.vtable != nil 295 | 296 | result = fsAwait s.vtable.readAsync(s, nil, 0) 297 | 298 | if s.buffers.eofReached: 299 | disconnectInputDevice(s) 300 | 301 | if result > 0 and s.span.len == 0: 302 | getNewSpan s 303 | 304 | proc timeoutToNextByteImpl(s: AsyncInputStream, 305 | deadline: Future): Future[bool] {.async.} = 306 | let readFut = s.readOnce 307 | fsAwait readFut or deadline 308 | if not readFut.finished: 309 | readFut.cancel() 310 | return true 311 | else: 312 | return false 313 | 314 | template timeoutToNextByte*(sp: AsyncInputStream, deadline: Future): bool = 315 | let s = sp 316 | if readableNow(s): 317 | true 318 | else: 319 | fsAwait timeoutToNextByteImpl(s, deadline) 320 | 321 | template timeoutToNextByte*(sp: AsyncInputStream, timeout: Duration): bool = 322 | let s = sp 323 | if readableNow(s): 324 | true 325 | else: 326 | fsAwait timeoutToNextByteImpl(s, sleepAsync(timeout)) 327 | 328 | proc closeAsync*(s: AsyncInputStream) {.async.} = 329 | close s 330 | 331 | template totalUnconsumedBytes*(s: AsyncInputStream): Natural = 332 | ## Alias for InputStream.totalUnconsumedBytes 333 | totalUnconsumedBytes InputStream(s) 334 | 335 | func getBestContiguousRunway(s: InputStream): Natural = 336 | result = s.span.len 337 | if result == 0 and s.buffers != nil: 338 | getNewSpan s 339 | result = s.span.len 340 | 341 | func totalUnconsumedBytes(s: VmInputStream): Natural = 342 | s.data.len - s.pos 343 | 344 | func totalUnconsumedBytes*(s: InputStream): Natural = 345 | ## Returns the number of bytes that are currently sitting within the stream 346 | ## buffers and that can be consumed with `read` or `advance`. 347 | when nimvm: 348 | totalUnconsumedBytes(VmInputStream(s)) 349 | else: 350 | let 351 | localRunway = s.span.len 352 | runwayInBuffers = 353 | if s.maxBufferedBytes.isSome(): 354 | s.maxBufferedBytes.get() 355 | elif s.buffers != nil: 356 | s.buffers.consumable() 357 | else: 358 | 0 359 | 360 | localRunway + runwayInBuffers 361 | 362 | proc prepareReadableRange(s: InputStream, rangeLen: Natural): auto = 363 | let 364 | vtable = s.vtable 365 | maxBufferedBytes = s.maxBufferedBytes 366 | 367 | runway = s.span.len 368 | endBytes = 369 | if rangeLen <= runway: 370 | s.span.endAddr = offset(s.span.startAddr, rangeLen) 371 | 372 | if s.buffers != nil: 373 | s.buffers.unconsume(runway - rangeLen) 374 | s.maxBufferedBytes = some Natural 0 375 | 0 # The bytes we removed from the local span are in the buffers already 376 | else: 377 | runway - rangeLen 378 | else: 379 | assert s.buffers != nil, "need buffers to cover the non-span part" 380 | s.maxBufferedBytes = some Natural (rangeLen - runway) 381 | 0 382 | 383 | s.vtable = nil 384 | (vtable: vtable, maxBufferedBytes: maxBufferedBytes, endBytes: endBytes) 385 | 386 | proc restoreReadableRange(s: InputStream, state: auto) = 387 | s.vtable = state.vtable 388 | s.maxBufferedBytes = state.maxBufferedBytes 389 | s.span.endAddr = offset(s.span.endAddr, state.endBytes) 390 | 391 | template withReadableRange*(sp: MaybeAsyncInputStream, 392 | rangeLen: Natural, 393 | rangeStreamVarName, blk: untyped) = 394 | let 395 | s = InputStream sp 396 | state = prepareReadableRange(s, rangeLen) 397 | try: 398 | let rangeStreamVarName {.inject.} = s 399 | blk 400 | finally: 401 | s.restoreReadableRange(state) 402 | 403 | const fileInputVTable = InputStreamVTable( 404 | readSync: proc (s: InputStream, dst: pointer, dstLen: Natural): Natural 405 | {.iocall.} = 406 | let file = FileInputStream(s).file 407 | fsAssert s.span.len == 0, "writing to buffer invalidates `consume`" 408 | implementSingleRead(s.buffers, dst, dstLen, 409 | {partialReadIsEof}, 410 | readStartAddr, readLen): 411 | file.readBuffer(readStartAddr, readLen) 412 | , 413 | getLenSync: proc (s: InputStream): Option[Natural] 414 | {.iocall.} = 415 | let 416 | s = FileInputStream(s) 417 | runway = s.totalUnconsumedBytes 418 | 419 | let preservedPos = getFilePos(s.file) 420 | setFilePos(s.file, 0, fspEnd) 421 | let endPos = getFilePos(s.file) 422 | setFilePos(s.file, preservedPos) 423 | 424 | some Natural(endPos - preservedPos + runway) 425 | , 426 | closeSync: proc (s: InputStream) 427 | {.iocall.} = 428 | try: 429 | close FileInputStream(s).file 430 | except OSError as err: 431 | raise newException(IOError, "Failed to close file", err) 432 | ) 433 | 434 | proc fileInput*(file: File, 435 | offset = 0, 436 | pageSize = defaultPageSize): InputStreamHandle 437 | {.raises: [IOError, OSError].} = 438 | ## Creates an input stream for reading the contents of a file 439 | ## through Nim's `io` module. 440 | ## 441 | ## Parameters: 442 | ## 443 | ## ``file`` 444 | ## The file to read. 445 | ## 446 | ## ``offset`` 447 | ## Initial position in the file where reading should start. 448 | ## 449 | 450 | if offset != 0: 451 | setFilePos(file, offset) 452 | 453 | makeHandle FileInputStream( 454 | vtable: vtableAddr fileInputVTable, 455 | buffers: if pageSize > 0: PageBuffers.init(pageSize) else: nil, 456 | file: file) 457 | 458 | proc fileInput*(filename: string, 459 | offset = 0, 460 | pageSize = defaultPageSize): InputStreamHandle 461 | {.raises: [IOError, OSError].} = 462 | ## Creates an input stream for reading the contents of a file 463 | ## through Nim's `io` module. 464 | ## 465 | ## Parameters: 466 | ## 467 | ## ``filename`` 468 | ## The name of the file to read. 469 | ## 470 | ## ``offset`` 471 | ## Initial position in the file where reading should start. 472 | ## 473 | let file = system.open(filename, fmRead) 474 | return fileInput(file, offset, pageSize) 475 | 476 | func unsafeMemoryInput*(mem: openArray[byte]): InputStreamHandle = 477 | ## Unsafe memory input that relies on `mem` to remain available for the 478 | ## duration of the usage of the input stream. 479 | ## 480 | ## One particular high-risk scenario is when using `--mm:refc` - when `mem` 481 | ## refers to a local garbage-collected type like `seq`, the GC might claim 482 | ## the instance even though scope-wise, it looks like it should stay alive. 483 | ## 484 | ## See also https://github.com/nim-lang/Nim/issues/25080 485 | when nimvm: 486 | makeHandle VmInputStream(data: @mem, pos: 0) 487 | else: 488 | let head = cast[ptr byte](mem) 489 | 490 | makeHandle InputStream( 491 | span: PageSpan( 492 | startAddr: head, 493 | endAddr: offset(head, mem.len)), 494 | spanEndPos: mem.len) 495 | 496 | func unsafeMemoryInput*(str: string): InputStreamHandle = 497 | unsafeMemoryInput str.toOpenArrayByte(0, str.len - 1) 498 | 499 | proc len(s: VmInputStream): Option[Natural] = 500 | doAssert s.data.len - s.pos >= 0 501 | some(Natural(s.data.len - s.pos)) 502 | 503 | proc len*(s: InputStream): Option[Natural] {.raises: [IOError].} = 504 | when nimvm: 505 | len(VmInputStream(s)) 506 | else: 507 | if s.vtable == nil: 508 | some s.totalUnconsumedBytes 509 | elif s.vtable.getLenSync != nil: 510 | s.vtable.getLenSync(s) 511 | else: 512 | none Natural 513 | 514 | when fsAsyncSupport: 515 | template len*(s: AsyncInputStream): Option[Natural] = 516 | len InputStream(s) 517 | 518 | func memoryInput*(buffers: PageBuffers): InputStreamHandle = 519 | makeHandle InputStream(buffers: buffers) 520 | 521 | func memoryInput*(data: openArray[byte]): InputStreamHandle = 522 | when nimvm: 523 | makeHandle VmInputStream(data: @data, pos: 0) 524 | else: 525 | let stream = if data.len > 0: 526 | let buffers = PageBuffers.init(data.len) 527 | buffers.write(data) 528 | 529 | InputStream(buffers: buffers) 530 | else: 531 | InputStream() 532 | 533 | makeHandle stream 534 | 535 | func memoryInput*(data: openArray[char]): InputStreamHandle = 536 | memoryInput data.toOpenArrayByte(0, data.high()) 537 | 538 | func resetBuffers*(s: InputStream, buffers: PageBuffers) = 539 | # This should be used only on safe memory input streams 540 | fsAssert s.vtable == nil and s.buffers != nil and buffers.len > 0 541 | s.spanEndPos = 0 542 | s.buffers = buffers 543 | getNewSpan s 544 | 545 | proc continueAfterRead(s: InputStream, bytesRead: Natural): bool = 546 | # Please note that this is extracted into a proc only to reduce the code 547 | # that ends up inlined into async procs by `bufferMoreDataImpl`. 548 | # The inlining itself is required to support the await-free operation of 549 | # the `readable` APIs. 550 | 551 | # The read might have been incomplete which signals the EOF of the stream. 552 | # If this is the case, we disconnect the input device which prevents any 553 | # further attempts to read from it: 554 | if s.buffers.eofReached: 555 | disconnectInputDevice(s) 556 | 557 | if bytesRead > 0: 558 | getNewSpan s 559 | true 560 | else: 561 | false 562 | 563 | template bufferMoreDataImpl(s, awaiter, readOp: untyped): bool = 564 | # This template is always called when the current page has been 565 | # completely exhausted. It should produce `true` if more data was 566 | # successfully buffered, so reading can continue. 567 | # 568 | # The vtable will be `nil` for a memory stream and `vtable.readOp` 569 | # will be `nil` for a memFile. If we've reached here, this is the 570 | # end of the memory buffer, so we can signal EOF: 571 | if s.buffers == nil: 572 | false 573 | else: 574 | # There might be additional pages in our buffer queue. If so, we 575 | # just jump to the next one: 576 | getNewSpan s 577 | if hasRunway(s.span): 578 | true 579 | elif s.vtable != nil and s.vtable.readOp != nil: 580 | # We ask our input device to populate our page queue with newly 581 | # read pages. The state of the queue afterwards will tell us if 582 | # the read was successful. In `continueAfterRead`, we examine if 583 | # EOF was reached, but please note that some data might have been 584 | # read anyway: 585 | continueAfterRead(s, awaiter s.vtable.readOp(s, nil, 0)) 586 | else: 587 | false 588 | 589 | proc bufferMoreDataSync(s: InputStream): bool = 590 | # This proc exists only to avoid inlining of the code of 591 | # `bufferMoreDataImpl` into `readable` (which in turn is 592 | # a template inlined in the user code). 593 | bufferMoreDataImpl(s, noAwait, readSync) 594 | 595 | template readable*(sp: InputStream): bool = 596 | ## Checks whether reading more data from the stream is possible. 597 | ## 598 | ## If there is any unconsumed data in the stream buffers, the 599 | ## operation returns `true` immediately. You can call `read` 600 | ## or `peek` afterwards to consume or examine the next byte 601 | ## in the stream. 602 | ## 603 | ## If the stream buffers are empty, the operation may block 604 | ## until more data becomes available. The end of the stream 605 | ## may be reached at this point, which will be indicated by 606 | ## a `false` return value. Any attempt to call `read` or 607 | ## `peek` afterwards is considered a `Defect`. 608 | ## 609 | ## Please note that this API is intended for stream consumers 610 | ## who need to consume the data one byte at a time. A typical 611 | ## usage will be the following: 612 | ## 613 | ## ```nim 614 | ## while stream.readable: 615 | ## case stream.peek.char 616 | ## of '"': 617 | ## parseString(stream) 618 | ## of '0'..'9': 619 | ## parseNumber(stream) 620 | ## of '\': 621 | ## discard stream.read # skip the slash 622 | ## let escapedChar = stream.read 623 | ## ``` 624 | ## 625 | ## Even though the user code consumes the data one byte at a time, 626 | ## in the majority of cases this consist of simply incrementing a 627 | ## pointer within the stream buffers. Only when the stream buffers 628 | ## are exhausted, a new read operation will be executed throught 629 | ## the stream input device which may repopulate the buffers with 630 | ## fresh data. See `Stream Pages` for futher discussion of this. 631 | 632 | # This is a template, because we want the pointer check to be 633 | # inlined at the call sites. Only if it fails, we call into the 634 | # larger non-inlined proc: 635 | when nimvm: 636 | VmInputStream(sp).pos < VmInputStream(sp).data.len 637 | else: 638 | let s = sp 639 | hasRunway(s.span) or bufferMoreDataSync(s) 640 | 641 | when fsAsyncSupport: 642 | template readable*(sp: AsyncInputStream): bool = 643 | ## Async version of `readable`. 644 | ## The intended API usage is the same. Instead of blocking, an async 645 | ## stream will use `await` while waiting for more data. 646 | let s = InputStream sp 647 | if hasRunway(s.span): 648 | true 649 | else: 650 | bufferMoreDataImpl(s, fsAwait, readAsync) 651 | 652 | func continueAfterReadN(s: InputStream, 653 | runwayBeforeRead, bytesRead: Natural) = 654 | if runwayBeforeRead == 0 and bytesRead > 0: 655 | getNewSpan s 656 | 657 | template readableNImpl(s, n, awaiter, readOp: untyped): bool = 658 | let runway = totalUnconsumedBytes(s) 659 | if runway >= n: 660 | true 661 | elif s.buffers == nil or s.vtable == nil or s.vtable.readOp == nil: 662 | false 663 | else: 664 | var 665 | res = false 666 | bytesRead = Natural 0 667 | bytesDeficit = n - runway 668 | # Return current span to buffers 669 | if s.span.len > 0: 670 | s.buffers.unconsume(s.span.len) 671 | s.span.reset() 672 | 673 | while true: 674 | bytesRead += awaiter s.vtable.readOp(s, nil, bytesDeficit) 675 | 676 | if s.buffers.eofReached: 677 | disconnectInputDevice(s) 678 | res = bytesRead >= bytesDeficit 679 | break 680 | 681 | if bytesRead >= bytesDeficit: 682 | res = true 683 | break 684 | 685 | continueAfterReadN(s, runway, bytesRead) 686 | res 687 | 688 | proc readable*(s: InputStream, n: int): bool = 689 | ## Checks whether reading `n` bytes from the input stream is possible. 690 | ## 691 | ## If there is enough unconsumed data in the stream buffers, the 692 | ## operation will return `true` immediately. You can use `read`, 693 | ## `peek`, `read(n)` or `peek(n)` afterwards to consume up to the 694 | ## number of verified bytes. Please note that consuming more bytes 695 | ## will be considered a `Defect`. 696 | ## 697 | ## If the stream buffers do not contain enough data, the operation 698 | ## may block until more data becomes available. The end of the stream 699 | ## may be reached at this point, which will be indicated by a `false` 700 | ## return value. Please note that the stream might still contain some 701 | ## unconsumed bytes after `readable(n)` returned false. You can use 702 | ## `totalUnconsumedBytes` or a combination of `readable` and `read` 703 | ## to consume the remaining bytes if desired. 704 | ## 705 | ## If possible, prefer consuming the data one byte at a time. This 706 | ## ensures the most optimal usage of the stream buffers. Even after 707 | ## calling `readable(n)`, it's still preferrable to continue with 708 | ## `read` instead of `read(n)` because the later may require the 709 | ## resulting bytes to be copied to a freshly allocated sequence. 710 | ## 711 | ## In the situation where the consumed bytes need to be copied to 712 | ## an existing external buffer, `readInto` will provide the best 713 | ## performance instead. 714 | ## 715 | ## Just like `readable`, this operation will invoke reads on the 716 | ## stream input device only when necessary. See `Stream Pages` 717 | ## for futher discussion of this. 718 | when nimvm: 719 | VmInputStream(s).pos + n <= VmInputStream(s).data.len 720 | else: 721 | readableNImpl(s, n, noAwait, readSync) 722 | 723 | when fsAsyncSupport: 724 | template readable*(sp: AsyncInputStream, np: int): bool = 725 | ## Async version of `readable(n)`. 726 | ## The intended API usage is the same. Instead of blocking, an async 727 | ## stream will use `await` while waiting for more data. 728 | let 729 | s = InputStream sp 730 | n = np 731 | 732 | readableNImpl(s, n, fsAwait, readAsync) 733 | 734 | template peek(s: VmInputStream): byte = 735 | doAssert s.pos < s.data.len 736 | s.data[s.pos] 737 | 738 | template peek*(sp: InputStream): byte = 739 | when nimvm: 740 | peek(VmInputStream(sp)) 741 | else: 742 | let s = sp 743 | if hasRunway(s.span): 744 | s.span.startAddr[] 745 | else: 746 | getNewSpanOrDieTrying s 747 | s.span.startAddr[] 748 | 749 | when fsAsyncSupport: 750 | template peek*(s: AsyncInputStream): byte = 751 | peek InputStream(s) 752 | 753 | func readFromNewSpan(s: InputStream): byte = 754 | getNewSpanOrDieTrying s 755 | s.span.read() 756 | 757 | template read(s: VmInputStream): byte = 758 | doAssert s.pos < s.data.len 759 | inc s.pos 760 | s.data[s.pos-1] 761 | 762 | template read*(sp: InputStream): byte = 763 | when nimvm: 764 | read(VmInputStream(sp)) 765 | else: 766 | let s = sp 767 | if hasRunway(s.span): 768 | s.span.read() 769 | else: 770 | readFromNewSpan s 771 | 772 | when fsAsyncSupport: 773 | template read*(s: AsyncInputStream): byte = 774 | read InputStream(s) 775 | 776 | func peekAt(s: VmInputStream, pos: int): byte = 777 | doAssert s.pos + pos < s.data.len 778 | s.data[s.pos + pos] 779 | 780 | func peekAt*(s: InputStream, pos: int): byte {.inline.} = 781 | when nimvm: 782 | return peekAt(VmInputStream(s), pos) 783 | else: 784 | let runway = s.span.len 785 | if pos < runway: 786 | let peekHead = offset(s.span.startAddr, pos) 787 | return peekHead[] 788 | 789 | if s.buffers != nil: 790 | var p = pos - runway 791 | for page in s.buffers.queue: 792 | if p < page.len(): 793 | return page.data()[p] 794 | p -= page.len() 795 | 796 | fsAssert false, 797 | "peeking past readable position pos=" & $pos & " readable = " & $s.totalUnconsumedBytes() 798 | 799 | when fsAsyncSupport: 800 | template peekAt*(s: AsyncInputStream, pos: int): byte = 801 | peekAt InputStream(s), pos 802 | 803 | func advance(s: VmInputStream) = 804 | doAssert s.pos < s.data.len 805 | inc s.pos 806 | 807 | func advance*(s: InputStream) = 808 | when nimvm: 809 | advance(VmInputStream(s)) 810 | else: 811 | if s.span.atEnd: 812 | getNewSpanOrDieTrying s 813 | 814 | s.span.advance() 815 | 816 | func advance*(s: InputStream, n: Natural) = 817 | # TODO This is silly, implement it properly 818 | for i in 0 ..< n: 819 | advance s 820 | 821 | when fsAsyncSupport: 822 | template advance*(s: AsyncInputStream) = 823 | advance InputStream(s) 824 | 825 | template advance*(s: AsyncInputStream, n: Natural) = 826 | advance InputStream(s), n 827 | 828 | func drainBuffersInto*(s: InputStream, dstAddr: ptr byte, dstLen: Natural): Natural = 829 | var 830 | dst = dstAddr 831 | remainingBytes = dstLen 832 | runway = s.span.len 833 | 834 | if runway >= remainingBytes: 835 | # Fast path: there is more data in the span that is being requested 836 | s.span.read(dst.makeOpenArray(remainingBytes)) 837 | return dstLen 838 | 839 | if runway > 0: 840 | s.span.read(dst.makeOpenArray(runway)) 841 | dst = offset(dst, runway) 842 | remainingBytes -= runway 843 | 844 | if s.buffers != nil: 845 | while remainingBytes > 0: 846 | getNewSpan s 847 | runway = s.span.len 848 | 849 | if runway == 0: 850 | break 851 | 852 | let bytes = min(runway, remainingBytes) 853 | 854 | s.span.read(dst.makeOpenArray(bytes)) 855 | dst = offset(dst, bytes) 856 | remainingBytes -= bytes 857 | 858 | dstLen - remainingBytes 859 | 860 | template readIntoExImpl(s: InputStream, 861 | dst: ptr byte, dstLen: Natural, 862 | awaiter, readOp: untyped): Natural = 863 | let totalBytesDrained = drainBuffersInto(s, dst, dstLen) 864 | var bytesDeficit = (dstLen - totalBytesDrained) 865 | if bytesDeficit > 0 and s.vtable != nil: 866 | var adjustedDst = offset(dst, totalBytesDrained) 867 | 868 | while true: 869 | let newBytesRead = awaiter s.vtable.readOp(s, adjustedDst, bytesDeficit) 870 | 871 | s.spanEndPos += newBytesRead 872 | bytesDeficit -= newBytesRead 873 | 874 | if s.buffers.eofReached: 875 | disconnectInputDevice(s) 876 | break 877 | 878 | if bytesDeficit == 0: 879 | break 880 | 881 | adjustedDst = offset(adjustedDst, newBytesRead) 882 | 883 | dstLen - bytesDeficit 884 | 885 | proc readIntoEx(s: VmInputStream, dst: var openArray[byte]): int = 886 | result = 0 887 | for i in 0 ..< dst.len: 888 | if s.pos >= s.data.len: 889 | break 890 | dst[i] = s.data[s.pos] 891 | inc s.pos 892 | inc result 893 | 894 | proc readIntoEx*(s: InputStream, dst: var openArray[byte]): int = 895 | ## Read data into the destination buffer. 896 | ## 897 | ## Returns the number of bytes that were successfully 898 | ## written to the buffer. The function will return a 899 | ## number smaller than the buffer length only if EOF 900 | ## was reached before the buffer was fully populated. 901 | when nimvm: 902 | readIntoEx(VmInputStream(s), dst) 903 | else: 904 | if dst.len > 0: 905 | let dstAddr = baseAddr dst 906 | let dstLen = dst.len 907 | readIntoExImpl(s, dstAddr, dstLen, noAwait, readSync) 908 | else: 909 | 0 910 | 911 | proc readInto*(s: InputStream, target: var openArray[byte]): bool = 912 | ## Read data into the destination buffer. 913 | ## 914 | ## Returns `false` if EOF was reached before the buffer 915 | ## was fully populated. if you need precise information 916 | ## regarding the number of bytes read, see `readIntoEx`. 917 | s.readIntoEx(target) == target.len 918 | 919 | when fsAsyncSupport: 920 | template readIntoEx*(sp: AsyncInputStream, dst: var openArray[byte]): int = 921 | let s = InputStream(sp) 922 | # BEWARE! `openArrayToPair` here is needed to avoid 923 | # double evaluation of the `dst` expression: 924 | let (dstAddr, dstLen) = openArrayToPair(dst) 925 | readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) 926 | 927 | template readInto*(sp: AsyncInputStream, dst: var openArray[byte]): bool = 928 | ## Asynchronously read data into the destination buffer. 929 | ## 930 | ## Returns `false` if EOF was reached before the buffer 931 | ## was fully populated. if you need precise information 932 | ## regarding the number of bytes read, see `readIntoEx`. 933 | ## 934 | ## If there are enough bytes already buffered by the stream, 935 | ## the expression will complete immediately. 936 | ## Otherwise, it will await more bytes to become available. 937 | 938 | let s = InputStream(sp) 939 | # BEWARE! `openArrayToPair` here is needed to avoid 940 | # double evaluation of the `dst` expression: 941 | let (dstAddr, dstLen) = openArrayToPair(dst) 942 | readIntoExImpl(s, dstAddr, dstLen, fsAwait, readAsync) == dstLen 943 | 944 | type MemAllocType {.pure.} = enum 945 | StackMem, HeapMem 946 | 947 | template readNImpl(sp: InputStream, 948 | np: Natural, 949 | memAllocType: static MemAllocType): openArray[byte] = 950 | let 951 | s = sp 952 | n = np 953 | runway = getBestContiguousRunway(s) 954 | 955 | # Since Nim currently doesn't allow the `makeOpenArray` calls bellow 956 | # to appear in different branches of an if statement, the code must 957 | # be written in this branch-free linear fashion. The `dataCopy` seq 958 | # may remain empty in the case where we use stack memory or return 959 | # an `openArray` from the existing span. 960 | var startAddr: ptr byte 961 | 962 | # If the "var buffer" is in the `block`, the ARC and ORC memory managers free 963 | # it when the block scope ends. 964 | when memAllocType == MemAllocType.StackMem: 965 | var buffer: array[np + 1, byte] 966 | elif memAllocType == MemAllocType.HeapMem: 967 | var buffer: seq[byte] 968 | else: 969 | static: doAssert false 970 | 971 | block: 972 | if n > runway: 973 | when memAllocType == MemAllocType.HeapMem: 974 | buffer.setLen(n) 975 | startAddr = baseAddr buffer 976 | let drained {.used.} = drainBuffersInto(s, startAddr, n) 977 | fsAssert drained == n 978 | else: 979 | startAddr = s.span.startAddr 980 | s.span.advance(n) 981 | 982 | makeOpenArray(startAddr, n) 983 | 984 | template read(s: VmInputStream, n: Natural): openArray[byte] = 985 | doAssert s.pos + n - 1 <= s.data.len, "not enough data to read" 986 | toOpenArray(s.data, s.pos, s.pos + n - 1) 987 | 988 | template read*(sp: InputStream, np: static Natural): openArray[byte] = 989 | when nimvm: 990 | read(VmInputStream(sp), np) 991 | else: 992 | const n = np 993 | when n < maxStackUsage: 994 | readNImpl(sp, n, MemAllocType.StackMem) 995 | else: 996 | readNImpl(sp, n, MemAllocType.HeapMem) 997 | 998 | template read*(s: InputStream, n: Natural): openArray[byte] = 999 | when nimvm: 1000 | read(VmInputStream(s), n) 1001 | else: 1002 | readNImpl(s, n, MemAllocType.HeapMem) 1003 | 1004 | when fsAsyncSupport: 1005 | template read*(s: AsyncInputStream, n: Natural): openArray[byte] = 1006 | read InputStream(s), n 1007 | 1008 | func lookAheadMatch*(s: InputStream, data: openArray[byte]): bool = 1009 | for i in 0 ..< data.len: 1010 | if s.peekAt(i) != data[i]: 1011 | return false 1012 | 1013 | return true 1014 | 1015 | when fsAsyncSupport: 1016 | template lookAheadMatch*(s: AsyncInputStream, data: openArray[byte]): bool = 1017 | lookAheadMatch InputStream(s) 1018 | 1019 | proc next*(s: InputStream): Option[byte] = 1020 | if readable(s): 1021 | result = some read(s) 1022 | 1023 | when fsAsyncSupport: 1024 | template next*(sp: AsyncInputStream): Option[byte] = 1025 | let s = sp 1026 | if readable(s): 1027 | some read(s) 1028 | else: 1029 | none byte 1030 | 1031 | func pos*(s: InputStream): int {.inline.} = 1032 | when nimvm: 1033 | VmInputStream(s).pos 1034 | else: 1035 | s.spanEndPos - s.span.len 1036 | 1037 | when fsAsyncSupport: 1038 | template pos*(s: AsyncInputStream): int = 1039 | pos InputStream(s) 1040 | --------------------------------------------------------------------------------