├── .github └── workflows │ └── nim.yml ├── .gitignore ├── License ├── Readme.md ├── asyncIters.nimble ├── nimdoc.cfg ├── src ├── asyncIters.nim └── asyncIters │ ├── cfg.nim │ └── private │ ├── asyncIter.nim │ ├── awaitIter.nim │ └── utils.nim └── tests ├── .gitignore ├── nim.cfg ├── test.nim └── uSelectiveImport.nim /.github/workflows/nim.yml: -------------------------------------------------------------------------------- 1 | %YAML 1.1 2 | --- 3 | name: Process Nim 4 | on: 5 | push: 6 | branches: [master] 7 | paths: 8 | - .github/workflows/** 9 | - '*.nimble' 10 | - '**.nim' 11 | - '**.nims' 12 | - '**nim.cfg' 13 | - '**nimdoc.cfg' 14 | workflow_dispatch: 15 | permissions: 16 | contents: read 17 | concurrency: 18 | group: nim 19 | cancel-in-progress: true 20 | defaults: 21 | run: 22 | shell: bash 23 | 24 | jobs: 25 | nim: 26 | strategy: 27 | matrix: 28 | nim-version: ['1.4.0', '1.4.x', '1.6.0', '1.6.x', stable] 29 | name: Test on ${{ matrix.nim-version }} 30 | runs-on: ubuntu-latest 31 | 32 | steps: 33 | - name: Cache Nim toolchain 34 | id: nim-cache 35 | uses: actions/cache@v3 36 | with: 37 | key: ${{ runner.os }}-${{ runner.arch }}-Nim-${{ matrix.nim-version }} 38 | path: |- 39 | ~/.choosenim/toolchains/ 40 | ~/.choosenim/current 41 | ~/.nimble/bin/ 42 | 43 | - name: Set up Nim toolchain 44 | if: steps.nim-cache.outputs.cache-hit != 'true' 45 | uses: jiro4989/setup-nim-action@v1 46 | with: 47 | nim-version: ${{ matrix.nim-version }} 48 | repo-token: ${{ secrets.GITHUB_TOKEN }} 49 | 50 | - name: Adjust PATH 51 | run: echo ~/.nimble/bin >> "$GITHUB_PATH" 52 | 53 | - name: Check out the project 54 | uses: actions/checkout@v3 55 | 56 | - name: Run tests 57 | run: >- 58 | nimble -y --verbose test 59 | --verbosity=2 60 | --warning[GcUnsafe]=off 61 | --hint[GlobalVar]=off 62 | --hint[Link]=off 63 | --hint[MsgOrigin]=off 64 | 65 | - name: Generate API docs 66 | if: matrix.nim-version == 'stable' 67 | run: >- 68 | nimble doc src/asyncIters.nim 69 | --git.url="${{ github.server_url }}/${{ github.repository }}" 70 | --git.commit="${{ github.ref_name }}" 71 | 72 | - name: Set up Pages 73 | if: matrix.nim-version == 'stable' 74 | uses: actions/configure-pages@v3 75 | 76 | - name: Upload API docs 77 | if: matrix.nim-version == 'stable' 78 | uses: actions/upload-pages-artifact@v1 79 | with: 80 | path: htmldocs/ 81 | 82 | deploy-pages: 83 | name: Deploy Pages 84 | needs: nim 85 | permissions: 86 | id-token: write 87 | pages: write 88 | environment: 89 | name: github-pages 90 | url: ${{ steps.deployment.outputs.page_url }} 91 | runs-on: ubuntu-latest 92 | 93 | steps: 94 | - name: Deploy Pages 95 | id: deployment 96 | uses: actions/deploy-pages@v1 97 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /htmldocs 2 | -------------------------------------------------------------------------------- /License: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Nickolay Bukreyev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Async iterators for Nim 2 | 3 | ```nim 4 | from std/asyncdispatch import sleepAsync, waitFor 5 | import asyncIters # Imports `async`, `await`, and `std/asyncfutures` as well. 6 | 7 | func countUpAsync(a, b: int): AsyncIterator[int] = 8 | iterator countUpAsync: Future[int] {.asyncIter.} = 9 | for i in a .. b: 10 | echo "Generating..." 11 | await sleepAsync 50 # You can await. 12 | yieldAsync i # And you can yield. 13 | 14 | result = countUpAsync 15 | 16 | proc test {.async.} = 17 | for i in awaitIter countUpAsync(1, 5): 18 | echo "Received ", i 19 | await sleepAsync 150 20 | 21 | waitFor test() 22 | ``` 23 | 24 | [API documentation](https://sirnickolas.github.io/asyncIters-Nim/asyncIters) 25 | 26 | `yieldAsync` passes values back to the caller. Sadly, we could not use the obvious `yield` keyword 27 | because it is reserved in async procedures to mean, “wait for a future to finish but do not perform 28 | error handling.” 29 | 30 | `yieldAsyncFrom` allows to delegate iteration to another async iterator. It is semantically 31 | equivalent to 32 | `for x in awaitIter another: yieldAsync x` but is more efficient. Example: 33 | 34 | ```nim 35 | func countUpAsync(a, b: int; step = 1): auto = 36 | result = iterator: Future[int] {.asyncIter.} = 37 | for i in countUp(a, b, step): 38 | yieldAsync i 39 | 40 | func evensAndOdds(a, b: int): auto = 41 | let evens = countUpAsync(a, b, 2) 42 | let odds = countUpAsync(a + 1, b, 2) 43 | result = iterator: Future[int] {.asyncIter.} = 44 | yieldAsyncFrom evens 45 | yieldAsyncFrom odds 46 | 47 | proc test {.async.} = 48 | for x in awaitIter evensAndOdds(0, 9): 49 | echo x # => 0 2 4 6 8 1 3 5 7 9 50 | ``` 51 | 52 | 53 | ## `asyncIters` vs `asyncstreams` 54 | 55 | [`std/asyncstreams`][asyncstreams] may look similar to this library, but they solve different 56 | problems. Async procedures communicating via a `FutureStream` run as independently as possible. 57 | Sometimes this is the right thing, but sometimes you want finer control. For example, a consumer 58 | might decide to abort iteration, and it would like to stop the producer as well. Moreover, it is 59 | important to stop it immediately so that no extraneous data is produced. In this case, 60 | `FutureStream` is a bad solution. On the other hand, `asyncIters` were designed with this scenario 61 | in mind. 62 | 63 | [asyncstreams]: https://nim-lang.org/docs/asyncstreams.html 64 | 65 | 66 | ## Using with `chronos/asyncloop` 67 | 68 | This library is mainly compatible with [Chronos][], with a single exception. You cannot `return` 69 | from an `awaitIter` loop — it produces a compilation error. As a workaround, consider assigning 70 | to `result` and `break`ing from the loop. (Hint: you can wrap the whole body of your procedure 71 | in a [labeled][block-stmt] `block` statement and break out of it.) 72 | 73 | Upstream issue: [status-im/nim-chronos#368][]. 74 | 75 | And if you are using Chronos with Nim **1.x**, there’s one more gotcha to be aware of: 76 | 77 |
78 | You cannot use the pragma syntax with `asyncIter`. 79 | 80 | ```nim 81 | # These don't work. 82 | iterator myIter: Future[int] {.asyncIter.} = 83 | discard 84 | 85 | let myAnonIter = iterator: Future[int] {.asyncIter.} = 86 | discard 87 | 88 | # Use these instead: 89 | asyncIter: 90 | iterator myIter: Future[int] = 91 | discard 92 | 93 | let myAnonIter = asyncIter(iterator: Future[int] = 94 | discard 95 | ) 96 | ``` 97 | 98 | That was a compiler bug: [status-im/nim-chronos#367][]. 99 |
100 | 101 | [Chronos]: https://github.com/status-im/nim-chronos 102 | [status-im/nim-chronos#367]: https://github.com/status-im/nim-chronos/issues/367 103 | [status-im/nim-chronos#368]: https://github.com/status-im/nim-chronos/issues/368 104 | [block-stmt]: https://nim-lang.org/docs/manual.html#statements-and-expressions-block-statement 105 | 106 | 107 | ## How it works 108 | 109 | `asyncIter` transforms the iterator definition to an async proc (which, ironically, will be 110 | eventually transformed by `{.async.}` back to an iterator): 111 | 112 | ```nim 113 | iterator countToTen: Future[int] {.asyncIter.} = 114 | for i in 0 ..< 10: 115 | yieldAsync i 116 | 117 | # => 118 | 119 | proc countToTen(body: proc (item: int): Future[uint32] {.gcSafe.}): Future[uint32] {.async.} = 120 | for i in 0 ..< 10: 121 | if (let ret = await body i; ret != 0'u32): 122 | return ret 123 | ``` 124 | 125 | `awaitIter` transforms the loop to an async proc as well (loop variables become procedure’s 126 | parameters) and calls the provided iterator with it: 127 | 128 | ```nim 129 | for item in awaitIter countToTen: 130 | echo item 131 | 132 | # => 133 | 134 | proc asyncForBody(item: int): Future[uint32] {.async.} = 135 | echo item 136 | 137 | discard await countToTen asyncForBody 138 | ``` 139 | 140 | 141 | ### What are `Future[uint32]` for? 142 | 143 | For supporting `break` and `return`. A more complex example: 144 | 145 | ```nim 146 | block blk: 147 | for item in awaitIter countToTen: 148 | break 149 | break blk 150 | return item 151 | 152 | # => 153 | 154 | block blk: 155 | proc asyncForBody(item: int): Future[uint32] {.async.} = 156 | return 1'u32 # `break` 157 | return 3'u32 # `break blk` 158 | complete retFuture, item # It is the future of the *outer* proc. 159 | return 2'u32 # `return item` 160 | 161 | let ret = await countToTen asyncForBody 162 | # Recall that `countToTen` stops iteration upon receiving a non-zero. 163 | case ret: 164 | of 0'u32, 1'u32: 165 | discard 166 | of 2'u32: 167 | return nil # This is actually generated by `{.async.}`; we just reattach it here. 168 | else: 169 | break blk 170 | ``` 171 | 172 | 173 | ## Limitations 174 | 175 | 1. With regular Nim iterators, you supply arguments on each step: 176 | 177 | ```nim 178 | # Not async. 179 | iterator double(n: int): int {.closure.} = # `{.inline.}` works too. 180 | while true: 181 | yield n shl 1 182 | 183 | var prev = 0 184 | for cur in double prev + 1: 185 | echo cur 186 | if cur > 100: 187 | break 188 | prev = cur 189 | # => 2 6 14 30 62 126 190 | ``` 191 | 192 | Generators in Python and JavaScript (both sync and async) work the same: you can pass data both 193 | in and out. They just use a different syntax: 194 | 195 | ```py 196 | def double(n): 197 | while True: 198 | n = yield n << 1 199 | 200 | g = double(1) 201 | cur = next(g) 202 | while True: 203 | print(cur) 204 | if cur > 100: 205 | break 206 | cur = g.send(cur + 1) 207 | ``` 208 | 209 | Unfortunately, async iterators implemented in this library do not support such usage pattern. 210 | Parameterized iterators are not allowed. You can provide arguments only at the start, before 211 | iteration begins, by wrapping the iterator in a closure (see the synopsis for an example). 212 | I’d like to add this feature, but it requires reimplementing [`asyncdispatch.async`][asyncmacro] 213 | from scratch — that’s an interesting task, but not today, sorry. 214 | 215 | [asyncmacro]: https://github.com/nim-lang/Nim/blob/version-1-6/lib/pure/asyncmacro.nim 216 | 217 | 2. In regular `{.async.}` procedures, you must not invoke templates or macros that contain 218 | a `return` statement: 219 | 220 | ```nim 221 | template returnIfNegative(x: int) = 222 | if x < 0: 223 | return 224 | 225 | proc process(x: int) {.async.} = 226 | returnIfNegative x # WRONG. 227 | ``` 228 | 229 | With async iterators, this restriction goes further: 230 | 231 | 1. You must not *indirectly* (i.e., via a template) invoke `return`, `break`, or `continue` 232 | from inside an `awaitIter` loop body. 233 | 2. You must not *indirectly* access the `result` implicit variable from inside an `awaitIter` 234 | loop body. 235 | 236 | 3. `awaitIter` is always tied to a `for` loop. I.e., you cannot pull a single value from 237 | an iterator; you can only run through all values it is going to produce. However, `break`ing 238 | is allowed, as well as iterating multiple times, so you can work around it. 239 | 240 | 4. [`multisyncIter`][multisync] is not currently implemented. 241 | 242 | [multisync]: https://nim-lang.org/docs/asyncdispatch.html#multisync.m,untyped 243 | -------------------------------------------------------------------------------- /asyncIters.nimble: -------------------------------------------------------------------------------- 1 | version = "1.3.0" 2 | author = "Nickolay Bukreyev" 3 | description = "Async iterators. Able to both await futures and yield values" 4 | license = "MIT" 5 | 6 | srcDir = "src" 7 | 8 | requires( 9 | "nim >= 1.4.0", 10 | # "chronos", # Just for running tests with it. 11 | "letUtils >= 1.1.1 & < 2.0.0", 12 | ) 13 | -------------------------------------------------------------------------------- /nimdoc.cfg: -------------------------------------------------------------------------------- 1 | project 2 | path = src 3 | outDir = htmldocs 4 | 5 | doc.item.seeSrc = """ 6 |   Source 7 | """ 8 | -------------------------------------------------------------------------------- /src/asyncIters.nim: -------------------------------------------------------------------------------- 1 | ##[ 2 | This package implements asynchronous iterators. For more information about asynchronous procedures 3 | in general, see `std/asyncdispatch`_ documentation. 4 | 5 | .. _std/asyncdispatch: https://nim-lang.org/docs/asyncdispatch.html 6 | ]## 7 | runnableExamples: 8 | # `async`, `await`, and `std/asyncfutures` are imported as well. 9 | from std/asyncdispatch import sleepAsync, waitFor 10 | 11 | func countUpAsync(a, b: int): AsyncIterator[int] = 12 | iterator countUpAsync: Future[int] {.asyncIter.} = 13 | for i in a .. b: 14 | echo "Generating..." 15 | await sleepAsync 50 # You can await. 16 | yieldAsync i # And you can yield. 17 | 18 | result = countUpAsync 19 | 20 | proc test {.async.} = 21 | for i in awaitIter countUpAsync(1, 5): 22 | echo "Received ", i 23 | await sleepAsync 150 24 | 25 | waitFor test() 26 | 27 | from ./asyncIters/cfg import backend 28 | 29 | template exportWhenDeclared(symbol: untyped) {.used.} = 30 | when declared symbol: 31 | export symbol 32 | 33 | when backend == "asyncdispatch": 34 | from std/asyncdispatch import nil 35 | from std/asyncfutures import Future 36 | 37 | export asyncdispatch.async 38 | exportWhenDeclared asyncdispatch.await 39 | export asyncfutures except callSoon # `asyncdispatch` provides an alternative implementation. 40 | elif backend == "chronos": 41 | from chronos/asyncloop as chr import Future 42 | 43 | # Export the bare minimum necessary to compile async procedures with older `chronos` versions, 44 | # plus the shiny new `chronos/futures` API if it is available. 45 | export Future, chr.FutureBase, chr.async, chr.complete, chr.newFuture 46 | exportWhenDeclared chr.await 47 | exportWhenDeclared chr.futureContinue 48 | exportWhenDeclared chr.internalCheckComplete 49 | exportWhenDeclared chr.internalRead 50 | exportWhenDeclared chr.futures # https://github.com/status-im/nim-chronos/pull/405 51 | 52 | when defined nimdoc: 53 | {.push.} 54 | when (NimMajor, NimMinor) >= (1, 6): 55 | {.hint[DuplicateModuleImport]: off.} 56 | include ./asyncIters/private/asyncIter 57 | include ./asyncIters/private/awaitIter 58 | {.pop.} 59 | else: 60 | from ./asyncIters/private/asyncIter import nil 61 | from ./asyncIters/private/awaitIter import customAsyncIterator 62 | 63 | export asyncIter, awaitIter 64 | 65 | when declared Future: 66 | when (NimMajor, NimMinor) >= (1, 9): 67 | {.warning[AmbiguousLink]: off.} # `customAsyncIterator` 68 | template the(x): untyped = x # https://github.com/SirNickolas/asyncIters-Nim/issues/1 69 | type AsyncIterator*[T] = the customAsyncIterator(T, Future) 70 | ##[ 71 | Type of async iterators after they are processed. 72 | 73 | This type is not declared if you pass `-d=asyncBackend:none`:option: (or some unrecognized 74 | backend name) to the compiler. Known backends include `asyncdispatch`_ (used by default 75 | if not set explicitly) and `chronos`_. If you’d like to use `asyncIters` with a backend that 76 | did not exist at the moment of writing, you need to use `customAsyncIterator`_ and specify 77 | some `Future`_-like type. 78 | 79 | Note also that this is only a *suggested* iterator type. Nothing stops you from using 80 | a different one or even having multiple in the same program. 81 | 82 | .. _asyncdispatch: https://nim-lang.org/docs/asyncdispatch.html 83 | .. _chronos: https://github.com/status-im/nim-chronos 84 | .. _customAsyncIterator: #customAsyncIterator.t,typed,typed 85 | .. _Future: https://nim-lang.org/docs/asyncfutures.html 86 | ]## 87 | -------------------------------------------------------------------------------- /src/asyncIters/cfg.nim: -------------------------------------------------------------------------------- 1 | from std/strutils import normalize 2 | 3 | const 4 | asyncBackend {.strDefine.} = "asyncdispatch" 5 | backend* = asyncBackend.normalize 6 | ## The value of `-d=asyncBackend:...`:option: compile-time switch, processed 7 | ## by `strutils.normalize `_. 8 | -------------------------------------------------------------------------------- /src/asyncIters/private/asyncIter.nim: -------------------------------------------------------------------------------- 1 | import std/macros 2 | from ./utils import copyLineInfoTo, morphInto 3 | 4 | func checkReturnType(params: NimNode): NimNode = 5 | ## Extract the return type from iterator’s params and validate it’s some kind of `Future[T]`. 6 | result = params[0] 7 | if not `or`( 8 | result.kind == nnkBracketExpr and result.len == 2, 9 | result.kind in CallNodes and result.len == 3 and result[0].eqIdent"[]", 10 | ): 11 | error "async iterator must yield Future[T]", result or params 12 | 13 | func transformIterDef(iterDef: NimNode): NimNode = 14 | ## Turn an `iterator` into an async `proc`. 15 | let params = iterDef[3] 16 | if params.len != 1: 17 | error( 18 | "parameterized async iterators are currently unsupported." & 19 | " You can probably achieve what you are trying to by wrapping the iterator in a proc", 20 | params, 21 | ) 22 | let 23 | returnType = params.checkReturnType 24 | yieldType = returnType[^1] 25 | bodySym = genSym(nskParam, "body") 26 | itemSym = ident"item" # For friendlier error messages. 27 | iterBody = iterDef[6] 28 | returnType[^1] = bindSym"uint32" 29 | 30 | result = iterDef.morphInto nnkProcDef 31 | params.add (quote do: 32 | let `bodySym`: proc (`itemSym`: `yieldType`): `returnType` {.gcSafe.} 33 | )[0] 34 | result.addPragma ident"async" # An open symbol to allow custom `async` implementations. 35 | result[6] = quote: 36 | template yieldAsync(value: typed) {.used.} = 37 | if (let ret = await `bodySym` value; ret != 0'u32): 38 | return ret 39 | 40 | template yieldAsyncFrom(iter: typed) {.used.} = 41 | if (let ret = await iter `bodySym`; ret != 0'u32): 42 | return ret 43 | 44 | block: 45 | `iterBody` 46 | 47 | func transformIterList(node: NimNode): NimNode = 48 | ## Recursively process the statement list containing iterator definitions. 49 | node.expectKind {nnkIteratorDef, nnkPar, nnkStmtList} 50 | if node.kind == nnkIteratorDef: 51 | node.transformIterDef 52 | else: 53 | for i, child in node: 54 | node[i] = child.transformIterList 55 | node 56 | 57 | macro asyncIter*(iterDef: untyped): untyped = 58 | ##[ 59 | Define an async iterator. It can have `yieldAsync` and `yieldAsyncFrom` statements in its body. 60 | 61 | This macro can be applied to either individual iterator definitions (`{.asyncIter.}`) or entire 62 | sections of code containing them (`asyncIter:`). 63 | ]## 64 | result = iterDef.transformIterList 65 | when defined asyncIters_debugAsync: 66 | echo result.repr 67 | 68 | type Inaccessible = object 69 | 70 | # We must provide at least two overloads so that `yieldAsync` is treated as an open symbol 71 | # by default. Otherwise, users would have to `mixin yieldAsync` to access it from templates. 72 | template yieldAsync*(phantom: Inaccessible) 73 | {.error: "congratulations, you've found a way to invoke me".} = discard 74 | 75 | macro yieldAsync*(values: varargs[typed]): untyped 76 | {.deprecated: "enclose multiple values in parentheses to yield them as a tuple".} = 77 | ## Transfer control to the caller of the async iterator. If several values are passed, they 78 | ## are wrapped in a tuple. 79 | case values.len: 80 | of 0: error "need a value to yield", values 81 | of 1: error "yieldAsync outside an async iterator", values 82 | else: result = bindSym("yieldAsync", brOpen).newCall(values.morphInto nnkTupleConstr) 83 | -------------------------------------------------------------------------------- /src/asyncIters/private/awaitIter.nim: -------------------------------------------------------------------------------- 1 | import std/macros 2 | from std/strutils import nimIdentNormalize 3 | import std/tables 4 | from letUtils import asLet, asVar 5 | from ./utils import copyLineInfoTo, morphInto 6 | 7 | type SomeAsyncIterator[T; F] = proc (body: proc (item: T): F {.gcSafe.}): F {.gcSafe.} 8 | 9 | template customAsyncIterator*(T, fut: typed): type = 10 | ##[ 11 | Type of async iterators after they are processed. `T` is the type of values an iterator yields; 12 | `fut` is the future type constructor those values are wrapped with. The only requirement 13 | is that `fut` must be instantiable with one generic parameter (i.e., `fut[U]`). 14 | ]## 15 | SomeAsyncIterator[T, fut[uint32]] 16 | 17 | func safeSignature(node: NimNode): string = 18 | ## Return a string that uniquely identifies the node (which must be either `nnkIdent` 19 | ## or `nnkSym`). 20 | if node.kind == nnkIdent: 21 | node.strVal.nimIdentNormalize.asVar: &= '.' # Period prevents clashing with symbols. 22 | else: 23 | node.expectKind {nnkIdent, nnkSym} # For better error messages. 24 | node.signatureHash 25 | 26 | func prepareLoopVarAndBody(loopVars, body: NimNode): (NimNode, NimNode) = 27 | ## Extract loop variable from an `Arglist`. If there are several variables, create a tuple 28 | ## parameter and generate code that unpacks it. 29 | let tupleParam = genSym(nskParam, "item") 30 | let section = nnkLetSection.newNimNode 31 | if loopVars.len == 1: 32 | let loopVar = loopVars[0] 33 | if loopVar.kind != nnkVarTuple: # A single loop variable. 34 | return (loopVar, body) 35 | section.add loopVar.add tupleParam # A single destructuring statement. 36 | else: 37 | # Multiple loop variables (need to unpack a tuple). Nim does not currently support recursive 38 | # tuple destructuring so we cannot just morph `loopVars` into `nnkVarTuple`. 39 | let rootVarTuple = nnkVarTuple.newNimNode 40 | section.add rootVarTuple 41 | for loopVar in loopVars: 42 | rootVarTuple.add: 43 | if loopVar.kind != nnkVarTuple: 44 | loopVar 45 | else: 46 | genSym(ident = "tuple").asLet aux: 47 | section.add loopVar.add aux 48 | rootVarTuple.add newEmptyNode(), tupleParam 49 | (tupleParam, newStmtList(section, body)) 50 | 51 | type 52 | NamedBlock = object 53 | nestedDefs: int ## How many times blocks with this name have been declared on the lexical stack. 54 | breakStmt: NimNode ## `nnkBreakStmt` or `nil` if not allocated yet. 55 | magicCode: NimNode ## `nnkUInt32Lit` or `nil` if not allocated yet. 56 | 57 | Context = object 58 | zero, one, magicSym, magicCodeSym: NimNode ## Immutable fields. 59 | maxMagicCode: uint32 60 | hasPlainBreak: bool 61 | failedToReturnLit: bool 62 | knownNamedBlocks: Table[string, NamedBlock] 63 | # Fields for `return` support: 64 | resultSym: NimNode ## `nnkSym` or `nil` if not allocated yet. 65 | plainReturnMagicCode: NimNode ## `nnkUInt32Lit` or `nil` if not allocated yet. 66 | returnLitMagicCode: NimNode ## `nnkUInt32Lit` or `nil` if not allocated yet. 67 | returnLit: NimNode ## One of `nnkLiterals` or `nnkNone` if can return different values. 68 | deferredReturnLists: seq[NimNode] ## `seq[nnkStmtList]` 69 | forwardedReturnStmt: NimNode ## Typically, `nnkStmtList`; or `nil` if not occurred. 70 | 71 | using 72 | ctx: Context 73 | mctx: var Context 74 | 75 | template asyncLoopMagic(code: uint32) {.pragma.} 76 | ## A pragma used to mark `return` statements that have already been processed. 77 | 78 | template asyncLoopMagicCode(code: uint32): uint32 = code 79 | ## An identity template used to mark a value that is being returned. 80 | 81 | func initContext: Context = 82 | Context( 83 | zero: newLit 0'u32, 84 | one: newLit 1'u32, 85 | magicSym: bindSym"asyncLoopMagic", 86 | magicCodeSym: bindSym"asyncLoopMagicCode", 87 | maxMagicCode: 1, # 0 is reserved for `continue`; 1 is reserved for `break`. 88 | ) 89 | 90 | template getOrAllocate(lval, initializer: untyped): untyped = 91 | ## If `lval` is not nil, return it. Otherwise, evaluate `initializer` and assign to `lval`. 92 | lval.asVar tmp: 93 | if tmp.isNil: 94 | tmp = initializer 95 | lval = tmp 96 | 97 | func createResult(mctx): NimNode = 98 | mctx.resultSym.getOrAllocate genSym(nskTemplate, "result") # Must be named `result`. 99 | 100 | func newBareMagicReturn(ctx; val, prototype: NimNode): NimNode = 101 | # -> return asyncLoopMagicCode(val) 102 | nnkReturnStmt.newNimNode(prototype).add(ctx.magicCodeSym.newCall val) 103 | 104 | func wrapWithMagic(ctx; val, stmts: NimNode): NimNode = 105 | # -> {.asyncLoopMagic: val.}: stmts 106 | nnkPragmaBlock.newTree( 107 | nnkPragma.newTree nnkExprColonExpr.newTree(ctx.magicSym, val), 108 | stmts, 109 | ) 110 | 111 | func newMagicReturn(ctx; val, prototype: NimNode): NimNode = 112 | # -> {.asyncLoopMagic: val.}: return asyncLoopMagicCode(val) 113 | ctx.wrapWithMagic(val, ctx.newBareMagicReturn(val, prototype)) 114 | 115 | proc maybeEnterNamedBlock(mctx; node: NimNode): string = 116 | ## If `node` is a named `block` statement or expression, remember it in the context and return 117 | ## the signature of its name. Otherwise, do nothing and return an empty string. 118 | if node.kind in {nnkBlockStmt, nnkBlockExpr}: 119 | let name = node[0] 120 | if name.kind != nnkEmpty: 121 | return name.safeSignature.asLet signature: 122 | mctx.knownNamedBlocks.mgetOrPut(signature, NamedBlock()).nestedDefs += 1 123 | 124 | func maybeLeaveNamedBlock(mctx; signature: string): bool {.discardable.} = 125 | ## Unless `signature` is an empty string, unregister the named block it refers to from the context 126 | ## and return `true`. Otherwise, return `false`. 127 | result = signature.len != 0 128 | if result: 129 | mctx.knownNamedBlocks[signature].nestedDefs -= 1 130 | 131 | template withMaybeNamedBlock(mctx: Context; node: NimNode; body: untyped): bool = 132 | ## Evaluate `body` with proper bookkeeping. Return `true` iff `node` is a named block. 133 | let signature = mctx.maybeEnterNamedBlock node 134 | body 135 | mctx.maybeLeaveNamedBlock signature 136 | 137 | func maybeTransformMagicReturn(mctx; node: NimNode): bool = 138 | ## If `node` is an `{.asyncLoopMagic: ...'u32.}: ...` pragma block, process it and return `true`. 139 | ## Otherwise, return `false`. 140 | if node.kind == nnkPragmaBlock: 141 | for pragma in node[0]: 142 | if pragma.kind in {nnkExprColonExpr} + CallNodes and pragma[0] == mctx.magicSym: 143 | # We've found our `asyncLoopMagic` pragma in the loop body passed to us. That means we are 144 | # a nested loop - the pragma was put by an outer `awaitIter` invocation. 145 | if mctx.forwardedReturnStmt.isNil: 146 | let x = pragma[1].intVal.uint32 147 | if x > mctx.maxMagicCode: 148 | mctx.maxMagicCode = x 149 | # We assume that `async` transforms `return` statements uniformly (i.e., doesn't 150 | # special-case anything). Therefore, we can remember only one of its transformation 151 | # results and expect all others to look similarly. 152 | mctx.forwardedReturnStmt = node[1] 153 | # -> return asyncLoopMagicCode(...'u32) 154 | node[1] = mctx.newBareMagicReturn(pragma[1], prototype = node[1]) 155 | return true 156 | 157 | proc transformBreakStmt(mctx; brk: NimNode; interceptPlainBreak: bool): NimNode = 158 | let blockName = brk[0] 159 | if blockName.kind != nnkEmpty: 160 | # A labeled `break`. 161 | let blk = addr mctx.knownNamedBlocks.mgetOrPut(blockName.safeSignature, NamedBlock()) 162 | if blk.nestedDefs == 0: 163 | # This block is declared outside the loop body. 164 | if blk.magicCode.isNil: 165 | # This is the first time we break out of this block. 166 | blk.breakStmt = brk 167 | blk.magicCode = nnkUInt32Lit.newNimNode 168 | # -> {.asyncLoopMagic: ...'u32.}: return asyncLoopMagicCode(...'u32) 169 | return mctx.newMagicReturn(blk.magicCode, prototype = brk) 170 | elif interceptPlainBreak: 171 | # An unlabeled `break`. 172 | mctx.hasPlainBreak = true 173 | # -> {.asyncLoopMagic: 1'u32.}: return asyncLoopMagicCode(1'u32) 174 | return mctx.newMagicReturn(mctx.one, prototype = brk) 175 | brk 176 | 177 | func canHandleReturnValLazily(mctx; val: NimNode): bool = 178 | ## Return `true` if `val` equals every other return value seen before. 179 | if mctx.returnLitMagicCode.isNil: 180 | # This is the first `return val` statement we've encountered. 181 | mctx.returnLitMagicCode = nnkUInt32Lit.newNimNode 182 | mctx.returnLit = val # Remember it so that we can compare subsequent values against it. 183 | val.kind in nnkLiterals 184 | else: 185 | # Check if it is the same literal we've seen the first time. 186 | val == mctx.returnLit 187 | 188 | func processReturnVal(mctx; val, magicStmts: NimNode): NimNode = 189 | ## Process the value of a `return` statement, adding necessary statements to `magicStmts`, 190 | ## and return the magic code chosen for this statement. 191 | if val.kind != nnkEmpty: 192 | # A `return val` statement. 193 | if not mctx.failedToReturnLit: 194 | if mctx.canHandleReturnValLazily val: 195 | mctx.deferredReturnLists &= magicStmts 196 | return mctx.returnLitMagicCode 197 | 198 | # This is the first nontrivial `return val` statement we've encountered. 199 | mctx.failedToReturnLit = true 200 | # -> result = val 201 | magicStmts.add mctx.createResult.newAssignment val 202 | # A plain `return` statement. 203 | mctx.plainReturnMagicCode.getOrAllocate nnkUInt32Lit.newNimNode 204 | 205 | func transformReturnStmt(mctx; ret: NimNode): NimNode = 206 | result = nnkStmtList.newNimNode 207 | let val = mctx.processReturnVal(ret[0], magicStmts = result) 208 | # -> ...; {.asyncLoopMagic: val.}: return asyncLoopMagicCode(val) 209 | ret[0] = mctx.magicCodeSym.newCall val 210 | result.add mctx.wrapWithMagic(val, ret) 211 | 212 | proc transformBody(mctx; tree: NimNode; interceptBreakContinue: bool): bool = 213 | ## Recursively traverse the loop body and transform it. Return `true` iff current node should 214 | ## not be processed further. 215 | if tree.kind in RoutineNodes - {nnkTemplateDef} or mctx.maybeTransformMagicReturn tree: 216 | return true 217 | mctx.withMaybeNamedBlock tree: 218 | # We should stop intercepting `break` and `continue` when descending into the last child 219 | # of a nested loop. `block` statements are not treated specially since unlabeled `break` 220 | # inside a `block` is deprecated and will change its meaning to what we already do now. 221 | let loopBodyIndex = if tree.kind not_in {nnkForStmt, nnkWhileStmt}: -1 else: tree.len - 1 222 | for i, node in tree: 223 | # Recurse. 224 | if not mctx.transformBody(node, interceptBreakContinue and i != loopBodyIndex): 225 | # Not a routine definition, a magic return section, nor a named block. 226 | tree[i] = case node.kind: 227 | of nnkIdent: 228 | if not node.eqIdent"result": 229 | continue 230 | mctx.createResult 231 | of nnkBreakStmt: 232 | mctx.transformBreakStmt(node, interceptBreakContinue) 233 | of nnkContinueStmt: 234 | if not interceptBreakContinue: 235 | continue 236 | # -> {.asyncLoopMagic: 0'u32.}: return asyncLoopMagicCode(0'u32) 237 | mctx.newMagicReturn(mctx.zero, prototype = node) 238 | of nnkReturnStmt: 239 | mctx.transformReturnStmt node 240 | else: 241 | continue 242 | 243 | func assignMagicCode(mctx; test: NimNode): NimNode = 244 | ## Allocate a new `uint32` value and assign it to `test`, which must be an `nnkUInt32Lit`. 245 | ## Return an `nnkOfBranch` with `test` as its first child. 246 | let code = mctx.maxMagicCode + 1 247 | mctx.maxMagicCode = code 248 | test.intVal = code.BiggestInt 249 | nnkOfBranch.newTree test 250 | 251 | func replaceMagicCode(ctx; tree, repl: NimNode): NimNode = 252 | ## Traverse `tree` and replace `asyncLoopMagicCode(...)` with `repl`. 253 | let magic = ctx.magicCodeSym 254 | if tree.kind in CallNodes and tree[0] == magic: 255 | repl 256 | else: 257 | func recurse(tree: NimNode) = 258 | for i, child in tree: 259 | if child.kind in CallNodes and child[0] == magic: 260 | tree[i] = repl 261 | else: 262 | child.recurse 263 | 264 | tree.recurse 265 | tree 266 | 267 | func createCaseDispatcher(mctx; retVar: NimNode): NimNode = 268 | ## Create an **incomplete** `case` statement that handles the magic code returned from the loop 269 | ## body (expected to be stored in `retVar`). 270 | 271 | # -> case ret 272 | result = nnkCaseStmt.newTree(retVar, nnkOfBranch.newNimNode) # A branch for `0, 1`. 273 | 274 | if not mctx.plainReturnMagicCode.isNil: 275 | # -> of ...: return 276 | result.add mctx.assignMagicCode(mctx.plainReturnMagicCode).add do: 277 | nnkReturnStmt.newTree newEmptyNode() 278 | 279 | if not mctx.returnLitMagicCode.isNil: 280 | if not mctx.failedToReturnLit: 281 | # -> of ...: return returnLit 282 | result.add mctx.assignMagicCode(mctx.returnLitMagicCode).add do: 283 | nnkReturnStmt.newTree mctx.returnLit 284 | else: 285 | # We were hoping to return a literal first but abandoned that idea. All `return` statements 286 | # we've managed to generate under that optimistic assumption should use the same magic code 287 | # as statements generated afterwards. 288 | mctx.returnLitMagicCode.intVal = mctx.plainReturnMagicCode.intVal 289 | # Patch statements we've deferred. 290 | let resultSym = mctx.resultSym 291 | let initiallySeenLit = mctx.returnLit 292 | for deferred in mctx.deferredReturnLists: 293 | # -> result = initiallySeenLit 294 | deferred.insert 0, resultSym.newAssignment initiallySeenLit 295 | 296 | for blk in mctx.knownNamedBlocks.values: 297 | if not blk.magicCode.isNil: 298 | # -> of ...: break ... 299 | result.add mctx.assignMagicCode(blk.magicCode).add(blk.breakStmt) 300 | 301 | if not mctx.forwardedReturnStmt.isNil: 302 | # -> else: ... 303 | result.add nnkElse.newTree mctx.replaceMagicCode(mctx.forwardedReturnStmt, repl = retVar) 304 | 305 | func patchCaseDispatcher(ctx; caseStmt: NimNode): NimNode = 306 | ## Transform the `case` statement made by `createCaseDispatcher` into the most appropriate form. 307 | ## May return `nil` if a dispatcher is not needed at all. 308 | case caseStmt.len: 309 | of 2: 310 | nil # Can ignore the return code. 311 | of 3: 312 | # There is only one nontrivial branch. Can emit `if` instead of `case`. 313 | let cond = nnkInfix.newNimNode 314 | if ctx.hasPlainBreak: 315 | # -> 1'u32 < ... 316 | cond.add bindSym"<", ctx.one 317 | else: 318 | # -> 0'u32 != ... 319 | cond.add bindSym"!=", ctx.zero # `test, jnz` is better than `cmp, ja`. 320 | cond.add caseStmt[0] # `retVar` 321 | # -> if ...: ... 322 | newIfStmt (cond, caseStmt[2][^1]) 323 | else: 324 | # -> of 0'u32, 1'u32: discard 325 | let firstBranch = caseStmt[1] 326 | firstBranch.add ctx.zero 327 | if ctx.hasPlainBreak: 328 | firstBranch.add ctx.one 329 | firstBranch.add nnkDiscardStmt.newTree newEmptyNode() 330 | 331 | # If the last branch of a `case` is `of`, turn it into `else`. 332 | let lastBranch = caseStmt[^1] 333 | if lastBranch.kind == nnkOfBranch: 334 | caseStmt[^1] = nnkElse.newTree lastBranch[1] 335 | 336 | caseStmt 337 | 338 | func createDeclarations(ctx): NimNode = 339 | ## Generate declarations that must be visible to the loop body. 340 | if ctx.resultSym.isNil: 341 | nnkStmtList.newNimNode 342 | else: 343 | let resultSym = ctx.resultSym 344 | quote: 345 | # We have to capture the address to avoid the compilation error regarding memory safety. 346 | let resultAddr = addr result 347 | # Syntactical presence of `result` in the body does not guarantee it is actually accessed 348 | # so we need `{.used.}`. 349 | template `resultSym`: untyped {.used.} = resultAddr[] 350 | 351 | proc processBody(body: NimNode): tuple[decls, invoker, invocationWrapper: NimNode] = 352 | ##[ 353 | Transform the loop body and generate code that runs it. Return a tuple: 354 | #. `decls` (`nnkStmtList`) are declarations that must be injected prior to the body definition; 355 | #. `invoker` (`nnkStmtList`) is the code that runs the loop and does some postprocessing; 356 | #. `invocationWrapper` (a descendant of `invoker`) is where the body invocation should be added. 357 | ]## 358 | let 359 | retVar = genSym(ident = "ret") 360 | (caseStmt, ctx) = block: 361 | var mctx = initContext() 362 | discard mctx.transformBody(body, interceptBreakContinue = true) 363 | (mctx.createCaseDispatcher retVar, mctx) # The order of access to `mctx` is important. 364 | dispatcher = ctx.patchCaseDispatcher caseStmt 365 | result.decls = ctx.createDeclarations 366 | result.invoker = 367 | if dispatcher.isNil: 368 | # -> discard ... 369 | result.invocationWrapper = nnkDiscardStmt.newNimNode 370 | result.invocationWrapper 371 | else: 372 | # -> let ret = ... 373 | result.invocationWrapper = nnkIdentDefs.newTree(retVar, newEmptyNode()) 374 | newStmtList( 375 | nnkLetSection.newTree result.invocationWrapper, 376 | dispatcher, 377 | ) 378 | 379 | macro awaitEach(iter: SomeAsyncIterator; originalBody: untyped; loopVars: varargs[untyped]) = 380 | ## Transform the loop body into an asynchronous procedure and run it. 381 | let 382 | (futureType, yieldType) = block: 383 | let params = iter.getTypeImpl[0] 384 | (params[0], params[1][1][0][1][1]) 385 | (loopParam, body) = prepareLoopVarAndBody(loopVars, originalBody) 386 | generated = originalBody.processBody 387 | bodyProcSym = genSym(nskProc, "asyncForBody") # For better stack traces. 388 | result = generated.decls 389 | result.add( 390 | newProc( 391 | name = bodyProcSym, 392 | params = [futureType, newIdentDefs(loopParam, yieldType)], 393 | pragmas = nnkPragma.newTree ident"async", 394 | body = body, 395 | ), 396 | generated.invoker, 397 | ) 398 | # -> ... await(iter(asyncForBody)) 399 | generated.invocationWrapper.add: 400 | iter.copyLineInfoTo(ident"await").newCall(iter.newCall bodyProcSym) 401 | 402 | when defined asyncIters_debugAwait: 403 | echo result.repr 404 | 405 | macro awaitEach(iter: typed; loopBodyAndVars: varargs[untyped]) = 406 | ## An overload that emits a helpful error message when `iter` has incorrect type. 407 | error "awaitIter expects an async iterator, got " & iter.getTypeInst.repr, iter 408 | 409 | macro awaitIter*(loop: ForLoopStmt) = 410 | ## Iterate over an async iterator. Like regular `await`, this can only occur in procedures 411 | ## marked with `{.async.}` or `{.asyncIter.}`. 412 | let invocation = loop[^2] # `awaitIter(...)` 413 | invocation.expectLen 2 414 | # Rewrite the loop into `awaitEach` call. 415 | result = loop.copyLineInfoTo bindSym"awaitEach".newCall(invocation[1], loop[^1]) # Iterator, body. 416 | for i in 0 ..< loop.len - 2: # Loop variables. 417 | result.add loop[i] 418 | -------------------------------------------------------------------------------- /src/asyncIters/private/utils.nim: -------------------------------------------------------------------------------- 1 | ## Utility functions for working with Nim AST. 2 | 3 | import std/macros 4 | 5 | func copyLineInfoTo*(info, arg: NimNode): NimNode = 6 | arg.copyLineInfo info 7 | arg 8 | 9 | func morphInto*(prototype: NimNode; kind: NimNodeKind; indices: Slice[int]): NimNode = 10 | ## Create a new node of type `kind` and add `prototype[indices]` as its children. 11 | result = kind.newNimNode prototype 12 | for i in indices: 13 | result.add prototype[i] 14 | 15 | func morphInto*(prototype: NimNode; kind: NimNodeKind; start = 0): NimNode = 16 | ## Create a new node of type `kind` and add `prototype[start ..^ 1]` as its children. 17 | prototype.morphInto(kind, start ..< prototype.len) 18 | -------------------------------------------------------------------------------- /tests/.gitignore: -------------------------------------------------------------------------------- 1 | /test 2 | /test.exe 3 | -------------------------------------------------------------------------------- /tests/nim.cfg: -------------------------------------------------------------------------------- 1 | path = "../src" 2 | experimental = strictFuncs 3 | -------------------------------------------------------------------------------- /tests/test.nim: -------------------------------------------------------------------------------- 1 | import std/macros 2 | import std/unittest 3 | import asyncIters 4 | from asyncIters/cfg as asyncCfg import nil 5 | from ./uSelectiveImport import nil 6 | 7 | const chronosBackend = asyncCfg.backend == "chronos" 8 | 9 | when chronosBackend: 10 | from chronos/asyncloop import waitFor 11 | else: 12 | from std/asyncdispatch import waitFor 13 | 14 | template runAsync(body: untyped) = 15 | proc run {.genSym, async.} = body 16 | waitFor run() 17 | 18 | proc main = 19 | test "can declare an async iterator": 20 | when NimMajor >= 2 or not chronosBackend: # https://github.com/status-im/nim-chronos/issues/367 21 | iterator named0: Future[int] {.asyncIter, used.} = discard 22 | iterator named1(): Future[int] {.used, asyncIter.} = discard 23 | # Anonymous iterators (and procedures) produce unhelpful stack traces. But they are supported 24 | # if you prefer conciseness over ease of debugging. 25 | let unnamed0 {.used.} = iterator: Future[int] {.asyncIter.} = discard 26 | let unnamed1 {.used.} = iterator (): Future[int] {.asyncIter.} = discard 27 | let unnamed2 {.used.} = asyncIter(iterator: Future[int] = discard) 28 | let unnamed3 {.used.} = asyncIter: 29 | (iterator: Future[int] = discard) 30 | 31 | test "can declare an async iterator inside a template": 32 | template declareIter = 33 | asyncIter: 34 | iterator nop: Future[int] {.used.} = discard 35 | 36 | declareIter 37 | 38 | test "can iterate over an async iterator": 39 | asyncIter: 40 | iterator produce2: Future[int] = 41 | yieldAsync 2 42 | yieldAsync 2 43 | 44 | var n = 0 45 | runAsync: 46 | for x in awaitIter produce2: 47 | check x == 2 48 | n += 1 49 | check n == 2 50 | 51 | test "async iterator can close over variables in its scope": 52 | func countUpTo(n: int): AsyncIterator[int] = 53 | asyncIter: 54 | iterator countUpTo: Future[int] = 55 | for i in 0 ..< n: 56 | yieldAsync i 57 | 58 | result = countUpTo 59 | 60 | var sum = 0 61 | runAsync: 62 | for x in awaitIter countUpTo 5: 63 | sum += x 64 | check sum == 10 65 | 66 | func countUpAsync(a, b: int; step = 1): auto = 67 | ## A simple iterator for tests. Like `countUp`, but pretends to be asynchronous. 68 | result = asyncIter(iterator: Future[int] = 69 | for i in countUp(a, b, step): 70 | yieldAsync i 71 | ) 72 | 73 | test "can yield from another async iterator": 74 | func evensAndOdds(a, b: int): auto = 75 | let evens = countUpAsync(a, b, 2) 76 | let odds = countUpAsync(a + 1, b, 2) 77 | result = asyncIter(iterator: Future[int] = 78 | yieldAsyncFrom evens 79 | yieldAsyncFrom odds 80 | ) 81 | 82 | var data: seq[int] 83 | runAsync: 84 | for x in awaitIter evensAndOdds(0, 9): 85 | data &= x 86 | check data == [0, 2, 4, 6, 8, 1, 3, 5, 7, 9] 87 | 88 | test "`awaitIter` can unpack simple tuples": 89 | asyncIter: 90 | iterator indexedStrings: Future[(int, string)] = 91 | yieldAsync (1, "test") 92 | 93 | iterator indexedPositions: Future[(int, string, tuple[x, y: float])] = 94 | yieldAsync (1, "here", (2.0, 4.0)) 95 | 96 | var n = 0 97 | runAsync: 98 | for pair in awaitIter indexedStrings: 99 | check pair == (1, "test") 100 | n += 1 101 | for i, s in awaitIter indexedStrings: 102 | check i == 1 103 | check s == "test" 104 | n += 1 105 | for (i, s) in awaitIter indexedStrings: 106 | check i == 1 107 | check s == "test" 108 | n += 1 109 | for i, s, (x, y) in awaitIter indexedPositions: 110 | check i == 1 111 | check s == "here" 112 | check x == 2.0 113 | check y == 4.0 114 | n += 1 115 | check n == 4 116 | 117 | test "`awaitIter` can unpack a 1-element tuple": 118 | asyncIter: 119 | iterator wrap5: Future[(int, )] = 120 | check not compiles yieldAsync 5 121 | yieldAsync (5, ) 122 | 123 | var n = 0 124 | runAsync: 125 | for wrapped in awaitIter wrap5: 126 | check wrapped == (5, ) 127 | n += 1 128 | for (five) in awaitIter wrap5: 129 | check five == 5 130 | n += 1 131 | for (five, ) in awaitIter wrap5: 132 | check five == 5 133 | n += 1 134 | check n == 3 135 | 136 | test "can declare procedures inside a loop": 137 | var sum = 0 138 | runAsync: 139 | for i in awaitIter countUpAsync(1, 3): 140 | func sqr(x: int): int = return x * x # `return` is essential for this test. 141 | sum += i.sqr 142 | check sum == 1 + 4 + 9 143 | 144 | test "can yield from a template declared inside an async iterator": 145 | asyncIter: 146 | iterator produce1122: Future[int] = 147 | template yieldTwice(x: int) = 148 | yieldAsync x 149 | yieldAsync x 150 | 151 | yieldTwice 1 152 | yieldTwice 2 153 | 154 | var n = 0 155 | runAsync: 156 | for i in awaitIter produce1122: 157 | check i in 1 .. 2 158 | n += 1 159 | check n == 4 160 | 161 | test "can yield from a template declared outside an async iterator": 162 | template yieldTwice(x: int) = 163 | yieldAsync x 164 | yieldAsyncFrom countUpAsync(x, x) 165 | 166 | asyncIter: 167 | iterator produce1122: Future[int] = 168 | yieldTwice 1 169 | yieldTwice 2 170 | 171 | var n = 0 172 | runAsync: 173 | for i in awaitIter produce1122: 174 | check i in 1 .. 2 175 | n += 1 176 | check n == 4 177 | 178 | test "can redeclare `yieldAsync`": 179 | asyncIter: 180 | iterator nop: Future[int] = 181 | template yieldAsync(x: typed) = discard 182 | 183 | yieldAsync 1 184 | 185 | runAsync: 186 | for i in awaitIter nop: 187 | check false 188 | 189 | test "can continue from a template declared inside a loop": 190 | var n = 0 191 | runAsync: 192 | for i in awaitIter countUpAsync(1, 5): 193 | template continueIfSmall(x: int) = 194 | if x <= 2: 195 | continue 196 | 197 | continueIfSmall i 198 | n += 1 199 | check n == 3 200 | 201 | test "can continue a loop": 202 | var sum = 0 203 | runAsync: 204 | for i in awaitIter countUpAsync(1, 10): 205 | if (i and 0x3) == 0x0: 206 | continue 207 | sum += i 208 | sum += 100 209 | check sum == (1 + 2 + 3) + (5 + 6 + 7) + 9 + 10 + 100 210 | 211 | test "can break from a loop": 212 | var sum = 0 213 | runAsync: 214 | for i in awaitIter countUpAsync(1, 10): 215 | if (i and 0x3) == 0x0: 216 | break 217 | sum += i 218 | sum += 100 219 | check sum == 1 + 2 + 3 + 100 220 | 221 | test "can return from a loop": 222 | when chronosBackend: 223 | skip # https://github.com/status-im/nim-chronos/issues/368 224 | else: 225 | var sum = 0 226 | runAsync: 227 | for i in awaitIter countUpAsync(1, 10): 228 | if (i and 0x3) == 0x0: 229 | return # From `runAsync`. 230 | sum += i 231 | check false 232 | check sum == 1 + 2 + 3 233 | 234 | test "can return a value from a loop": 235 | when chronosBackend: 236 | skip # https://github.com/status-im/nim-chronos/issues/368 237 | else: 238 | var sum = 0 239 | 240 | proc run: Future[int] {.async.} = 241 | for i in awaitIter countUpAsync(1, 10): 242 | if (i and 0x3) == 0x0: 243 | return 13 244 | sum += i 245 | check false 246 | 247 | check waitFor(run()) == 13 248 | check sum == 1 + 2 + 3 249 | 250 | test "can access `result` inside a loop": 251 | var t = (result: 0) 252 | 253 | proc run: Future[int] {.async.} = 254 | for i in awaitIter countUpAsync(1, 3): 255 | t.result += i 256 | result -= i 257 | 258 | check waitFor(run()) == -(1 + 2 + 3) 259 | check t == (1 + 2 + 3, ) 260 | 261 | test "can pass `result` from a loop to a template": 262 | template add5(x: int) = x += 5 263 | 264 | proc run: Future[int] {.async.} = 265 | for i in awaitIter countUpAsync(1, 2): 266 | result.add5 267 | add5 result 268 | 269 | check waitFor(run()) == 20 270 | 271 | test "can nest a regular loop inside async one": 272 | var sum = 0 273 | 274 | proc run: Future[string] {.async.} = 275 | block procBody: 276 | for i in awaitIter countUpAsync(1, 10): 277 | for j in i .. 10: 278 | if j == 6: 279 | break 280 | elif i == 7 and j == 9: 281 | when chronosBackend: 282 | result = "ok" 283 | break procBody 284 | else: 285 | return "ok" 286 | sum += j 287 | continue 288 | sum += i * 100 289 | check false 290 | 291 | check waitFor(run()) == "ok" 292 | check sum == 2170 293 | 294 | test "can nest loops": 295 | var sum = 0 296 | 297 | proc run: Future[string] {.async.} = 298 | block procBody: 299 | for i in awaitIter countUpAsync(1, 10): 300 | for j in awaitIter countUpAsync(i, 10): 301 | if j == 6: 302 | break 303 | elif i == 7 and j == 9: 304 | when chronosBackend: 305 | result = "ok" 306 | break procBody 307 | else: 308 | return "ok" 309 | sum += j 310 | continue 311 | sum += i * 100 312 | check false 313 | 314 | check waitFor(run()) == "ok" 315 | check sum == 2170 316 | 317 | test "can return an implicit `result` from a nested loop": 318 | when chronosBackend: 319 | skip # https://github.com/status-im/nim-chronos/issues/368 320 | else: 321 | var sum = 0 322 | 323 | proc run: Future[string] {.async.} = 324 | result = "ok" 325 | for i in awaitIter countUpAsync(1, 10): 326 | for j in awaitIter countUpAsync(i, 10): 327 | if i == 3 and j == 5: 328 | return 329 | sum += j 330 | sum += i * 100 331 | check false 332 | 333 | check waitFor(run()) == "ok" 334 | check sum == 416 335 | 336 | test "can break from a named block": 337 | var sum = 0 338 | runAsync: 339 | block outer: 340 | for i in awaitIter countUpAsync(1, 10): 341 | block inner: 342 | for j in awaitIter countUpAsync(i, 10): 343 | if j == 6: 344 | break inner 345 | elif i == 7 and j == 9: 346 | break outer 347 | sum += j 348 | sum += i * 100 349 | check false 350 | check sum == 2170 351 | 352 | test "named blocks can shadow one another": 353 | var sum = 0 354 | runAsync: 355 | block blk: 356 | for i in awaitIter countUpAsync(1, 5): 357 | block blk: 358 | if i == 3: 359 | break blk 360 | sum += i 361 | if i == 4: 362 | break blk 363 | sum += i * 10 364 | check false 365 | check sum == 67 366 | 367 | test "can refer to a block before it is shadowed": 368 | var sum = 0 369 | runAsync: 370 | block blk: 371 | for i in awaitIter countUpAsync(1, 5): 372 | if i == 4: 373 | break blk 374 | block blk: 375 | if i == 3: 376 | break blk 377 | sum += i 378 | sum += i * 10 379 | check false 380 | check sum == 63 381 | 382 | test "unlabeled `break` does not interfere with `block`": 383 | var ok = false 384 | runAsync: 385 | for i in awaitIter countUpAsync(1, 5): 386 | block: 387 | break # Behaviour is different from Nim 1.x! 388 | check false 389 | for i in awaitIter countUpAsync(1, 5): 390 | block blk: 391 | break # Behaviour is different from Nim 1.x! 392 | check false 393 | ok = true 394 | check ok 395 | 396 | test "can have `proc` as a direct child of a loop": 397 | # Usually, an `nnkForStmt` wraps an `nnkStmtList`. We have to write a macro to attach 398 | # an `nnkProcDef` directly. 399 | macro construct(iter: typed) = 400 | result = quote: 401 | for i in awaitIter `iter`: 402 | proc p: string {.used.} = return "unused" # `return` is essential for this test. 403 | result[2].expectKind nnkStmtList 404 | result[2].expectLen 1 405 | result[2] = result[2][0] 406 | 407 | var ok = false 408 | runAsync: 409 | construct countUpAsync(1, 5) 410 | ok = true 411 | check ok 412 | 413 | test "can have `break` as a direct child of a `block`": 414 | # Usually, an `nnkBlockStmt` wraps an `nnkStmtList`. We have to write a macro to attach 415 | # an `nnkBreakStmt` directly. 416 | macro construct(n: int; iter: typed) = 417 | result = quote: 418 | for i in awaitIter `iter`: 419 | block blk: 420 | break blk 421 | `n` += 1 422 | let blk = result[2][0] 423 | blk.expectKind nnkBlockStmt 424 | blk[1].expectKind nnkStmtList 425 | blk[1].expectLen 1 426 | blk[1] = blk[1][0] 427 | 428 | var n = 0 429 | runAsync: 430 | construct n, countUpAsync(1, 5) 431 | check n == 5 432 | 433 | test "customAsyncIterator can work with arbitrary type constructors": 434 | type ArrayTypeCtor = object 435 | len: int 436 | 437 | template `[]`(ctor: ArrayTypeCtor; T: type): type = 438 | array[ctor.len, T] 439 | 440 | const arr10 = ArrayTypeCtor(len: 10) 441 | check (arr10[int] is array[10, int]) # Nim 1.4 requires parenthizing the expression. 442 | check (customAsyncIterator(int, arr10) is ( 443 | # An implementation detail. 444 | proc (body: proc (item: int): array[10, uint32]): array[10, uint32] 445 | )) 446 | 447 | test "can return different values from a nested loop": 448 | # This test requires a custom async backend: 449 | type Identity[T] = T 450 | template async {.pragma.} 451 | template await(x: typed): untyped = x 452 | 453 | asyncIter: 454 | iterator one: Identity[int] = 455 | yieldAsync 1 456 | 457 | proc run(flag: bool): string = 458 | for i in awaitIter one: 459 | check i == 1 460 | for j in awaitIter one: 461 | check j == 1 462 | if flag: 463 | return "yes" 464 | return "no" # Having two separate `return` statements is essential for this test. 465 | 466 | check run(true) == "yes" 467 | check run(false) == "no" 468 | 469 | main() 470 | -------------------------------------------------------------------------------- /tests/uSelectiveImport.nim: -------------------------------------------------------------------------------- 1 | # https://github.com/SirNickolas/asyncIters-Nim/issues/1 2 | from asyncIters import AsyncIterator 3 | 4 | {.used.} 5 | 6 | type A {.used.} = AsyncIterator[int] 7 | --------------------------------------------------------------------------------