├── .github └── workflows │ └── node.js.yml ├── .gitignore ├── .prettierrc.yaml ├── LICENSES ├── CC0-1.0.txt ├── LGPL-3.0-only.txt └── Unlicense.txt ├── README.md ├── bench.txt ├── benchmark ├── helpers │ ├── run_benchmark.js │ └── setup_test_functions.js └── index.js ├── copy-json-to-bipf-async.js ├── copy-json-to-bipf-offset.js ├── copy-json-to-bipf.js ├── example.js ├── files.js ├── flumelog.diff ├── flumelog.diff.license ├── index.js ├── operators.js ├── package.json ├── package.json.license ├── status.js └── test ├── add.js ├── bump-version.js ├── common.js ├── compaction.js ├── del.js ├── helpers.js ├── live-then-grow.js ├── live.js ├── lookup.js ├── operators.js ├── prefix.js ├── query.js ├── reindex.js ├── save-load.js ├── seq-index-not-uptodate.js └── slow-save.js /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | # 3 | # SPDX-License-Identifier: Unlicense 4 | 5 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 6 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 7 | 8 | name: CI 9 | 10 | on: 11 | push: 12 | branches: [master] 13 | pull_request: 14 | branches: [master] 15 | 16 | jobs: 17 | licenses: 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: REUSE Compliance Check 22 | uses: fsfe/reuse-action@v1 23 | 24 | test: 25 | runs-on: ubuntu-latest 26 | timeout-minutes: 10 27 | 28 | strategy: 29 | matrix: 30 | node-version: [16.x] 31 | 32 | steps: 33 | - uses: actions/checkout@v2 34 | - name: Use Node.js ${{ matrix.node-version }} 35 | uses: actions/setup-node@v1 36 | with: 37 | node-version: ${{ matrix.node-version }} 38 | - run: npm install 39 | - name: npm test 40 | run: DEBUG=jitdb npm test 41 | 42 | prepare_for_benchmarks: 43 | runs-on: ubuntu-latest 44 | timeout-minutes: 10 45 | outputs: 46 | matrix: ${{ steps.set-matrix.outputs.matrix }} 47 | steps: 48 | - uses: actions/checkout@v2 49 | - name: Use Node.js 16.x 50 | uses: actions/setup-node@v1 51 | with: 52 | node-version: 16.x 53 | - run: npm install 54 | - id: set-matrix 55 | run: | 56 | echo "getting benchmark matrix" 57 | BENCHMARKS=$(npm run --silent get-benchmark-matrix) 58 | echo "checking benchmark matrix" 59 | if [ -z "$BENCHMARKS" ]; then 60 | echo "Failed to generate benchmarks" 61 | exit 1 62 | else 63 | echo $BENCHMARKS 64 | echo "::set-output name=matrix::{\"benchmark\":$BENCHMARKS}" 65 | fi 66 | - name: Restore Benchmark Fixture Cache 67 | id: benchmark-fixture-cache 68 | uses: actions/cache@v2 69 | env: 70 | cache-name: cache-benchmark-fixture 71 | with: 72 | path: /tmp/jitdb-benchmark 73 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json','**/package.json','**/benchmark/**/*.js','**/.github/**/*') }} 74 | - name: Generate Benchmark Fixture 75 | if: steps.benchmark-fixture-cache.outputs.cache-hit != 'true' 76 | run: npm run benchmark-only-create 77 | - name: Upload Benchmark Fixture 78 | uses: actions/upload-artifact@v2 79 | with: 80 | name: benchmark-fixture 81 | path: /tmp/jitdb-benchmark 82 | retention-days: 1 83 | 84 | benchmark: 85 | needs: 86 | - test 87 | - prepare_for_benchmarks 88 | runs-on: ubuntu-latest 89 | strategy: 90 | fail-fast: false 91 | matrix: ${{ fromJson(needs.prepare_for_benchmarks.outputs.matrix) }} 92 | 93 | steps: 94 | - uses: actions/checkout@v2 95 | - name: Use Node.js 16.x 96 | uses: actions/setup-node@v1 97 | with: 98 | node-version: 16.x 99 | - run: npm install 100 | - name: Download Benchmark Fixture 101 | uses: actions/download-artifact@v2 102 | with: 103 | name: benchmark-fixture 104 | path: /tmp/jitdb-benchmark 105 | - name: Benchmark 106 | run: BENCHMARK_DURATION_MS=60000 CURRENT_BENCHMARK="${{matrix.benchmark}}" npm run benchmark 107 | - name: Upload Result 108 | uses: actions/upload-artifact@v2 109 | with: 110 | name: ${{matrix.benchmark}}-md 111 | path: /tmp/jitdb-benchmark/benchmark.md 112 | retention-days: 1 113 | 114 | benchmark_results: 115 | needs: benchmark 116 | runs-on: ubuntu-latest 117 | steps: 118 | - name: Download Results 119 | uses: actions/download-artifact@v2 120 | with: 121 | path: /tmp/artifacts 122 | - id: get-comment-body 123 | name: Gather results 124 | run: | 125 | body=$(cat /tmp/artifacts/*-md/* | awk '!x[$0]++') 126 | headLineCount=$(printf "$body" | grep -hn '\--' | cut -f1 -d:) 127 | totalLineCount=$(printf "$body" | wc -l | cut -f1 -d" ") 128 | headLines=$(printf "$body" | head -n $headLineCount) 129 | sortedTailLines=$(printf "$body" | tail -n $(($totalLineCount - $headLineCount + 1)) | sort) 130 | body=$(printf "$headLines\n$sortedTailLines") 131 | body="${body//'%'/'%25'}" 132 | body="${body//$'\n'/'%0A'}" 133 | body="${body//$'\r'/'%0D'}" 134 | echo ::set-output name=body::$body 135 | - name: Publish comment 136 | uses: mshick/add-pr-comment@v1 137 | continue-on-error: true 138 | with: 139 | message: ${{ steps.get-comment-body.outputs.body }} 140 | repo-token: ${{ secrets.GITHUB_TOKEN }} 141 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | # 3 | # SPDX-License-Identifier: Unlicense 4 | 5 | /node_modules 6 | /indexes 7 | /.vscode 8 | pnpm-lock.yaml 9 | *~ 10 | .nyc_output 11 | coverage -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | # SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros 3 | # 4 | # SPDX-License-Identifier: Unlicense 5 | 6 | semi: false 7 | singleQuote: true 8 | -------------------------------------------------------------------------------- /LICENSES/CC0-1.0.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /LICENSES/LGPL-3.0-only.txt: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | 6 | Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. 7 | 8 | This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 9 | 10 | 0. Additional Definitions. 11 | 12 | As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. 13 | 14 | "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. 15 | 16 | An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. 17 | 18 | A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". 19 | 20 | The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. 21 | 22 | The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 23 | 24 | 1. Exception to Section 3 of the GNU GPL. 25 | You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 26 | 27 | 2. Conveying Modified Versions. 28 | If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: 29 | 30 | a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or 31 | 32 | b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 33 | 34 | 3. Object Code Incorporating Material from Library Header Files. 35 | The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: 36 | 37 | a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. 38 | 39 | b) Accompany the object code with a copy of the GNU GPL and this license document. 40 | 41 | 4. Combined Works. 42 | You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: 43 | 44 | a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. 45 | 46 | b) Accompany the Combined Work with a copy of the GNU GPL and this license document. 47 | 48 | c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. 49 | 50 | d) Do one of the following: 51 | 52 | 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 53 | 54 | 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. 55 | 56 | e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 57 | 58 | 5. Combined Libraries. 59 | You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: 60 | 61 | a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. 62 | 63 | b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 64 | 65 | 6. Revised Versions of the GNU Lesser General Public License. 66 | The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. 67 | 68 | Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. 69 | 70 | If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall 71 | apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. 72 | -------------------------------------------------------------------------------- /LICENSES/Unlicense.txt: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. 4 | 5 | In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and 6 | successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 9 | 10 | For more information, please refer to 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # JITDB 8 | 9 | A database on top of [async-append-only-log] with automatic index generation and 10 | maintenance. 11 | 12 | The motivation for this database is that it should be: 13 | 14 | - fast 15 | - easy to understand 16 | - run in the browser and in node 17 | 18 | Async append only log takes care of persistance of the main log. It is 19 | expected to use [bipf] to encode data. On top of this, JITDB lazily 20 | creates and maintains indexes based on the way the data is queried. 21 | Meaning if you search for messages of type `post` an author `x` two 22 | indexes will be created the first time. One for type and one for 23 | author. Specific indexes will only updated when it is queried again. 24 | These indexes are tiny compared to normal [flume] indexes. An index of 25 | type `post` is 80kb. 26 | 27 | For this to be feasible it must be really fast to do a full log scan. 28 | It turns out that the combination of push streams and bipf makes 29 | streaming the full log not much slower than reading the file. Meaning 30 | a 350mb log can be scanned in a few seconds. 31 | 32 | While this is mainly aimed as a query engine, it is possible to base 33 | other indexes types on top of this, such as a reduce index on contact 34 | messages. 35 | 36 | ## API 37 | 38 | ### Setup 39 | 40 | Before using JITDB, you have to setup an instance of 41 | [async-append-only-log] located at a certain path. Then you can 42 | instantiate JITDB, and it requires a **path to the directory where the 43 | indexes** will live. 44 | 45 | ```js 46 | const Log = require('async-append-only-log') 47 | const JITDB = require('jitdb') 48 | 49 | const raf = Log('/home/me/path/to/async-log', { 50 | blockSize: 64 * 1024, 51 | }) 52 | const db = JITDB(raf, '/home/me/path/to/indexes') 53 | 54 | db.onReady(() => { 55 | // The db is ready to be queried 56 | }) 57 | ``` 58 | 59 | ### Operators 60 | 61 | JITDB comes with a set of composable "operators" that allow you to 62 | query the database. You can load these operators from 63 | `require('jitdb/operators')`. 64 | 65 | ```js 66 | const Log = require('async-append-only-log') 67 | const JITDB = require('jitdb') 68 | const {query, fromDB, where, slowEqual, toCallback} = require('jitdb/operators') 69 | 70 | const raf = Log('/home/me/path/to/async-log', { 71 | blockSize: 64 * 1024, 72 | }) 73 | const db = JITDB(raf, '/home/me/path/to/indexes') 74 | 75 | db.onReady(() => { 76 | query( 77 | fromDB(db), 78 | where(slowEqual('value.content.type', 'post')), 79 | toCallback((err, msgs) => { 80 | console.log(msgs) 81 | }) 82 | ) 83 | }) 84 | ``` 85 | 86 | The essential operators are `fromDB`, `query`, and `toCallback`. 87 | 88 | - **fromDB** specifies which JITDB instance we are interested in 89 | - **query** wraps all the operators, chaining them together 90 | - **where** wraps *descriptor operators* (see below) that narrow down the data 91 | - **toCallback** delivers the results of the query to a callback 92 | 93 | Then there are *descriptor operator* that help scope down the results to your 94 | desired set of messages: `and`, `or`, `not`, `equal`, `slowEqual`, and others. 95 | 96 | - `and(...args)` filters for messages that satisfy **all** `args` 97 | - `or(...args)` filters for messages that satisfy **at least one** of the `args` 98 | - `not(arg)` filters for messages that do not safisfy `arg` 99 | - `equal(seek, value, opts)` filters for messages where a `seek`ed _field_ 100 | matches a specific _value_: 101 | - `seek` is a function that takes a [bipf] buffer as input and uses 102 | `bipf.seekKey` to return a pointer to the _field_ 103 | - `value` is a string or buffer which is the value we want the _field_'s value to match 104 | - `opts` are additional configurations: 105 | - `indexType` is a name used to identify the index produced by this query 106 | - `prefix` boolean or number `32` that tells this query to use [prefix indexes](#prefix-indexes) 107 | - `slowEqual(objPath, value, opts)` is a more ergonomic (but slower) way of performing `equal`: 108 | - `objPath` a string in the shape `"foo.bar.baz"` which specifies the nested field `"baz"` inside `"bar"` inside `"foo"` 109 | - `value` is the same as `value` in the `equal` operator 110 | - `opts` same as the opts for `equal()` 111 | - `includes(seek, value, opts)` filters for messages where a `seek`ed _field_ 112 | is an array and includes a specific _value_ 113 | - `slowIncludes(objPath, value, opts)` is to `includes` what `slowEqual` is to 114 | `equal` 115 | - `predicate(seek, fn, opts)` filters for messages where a `seek`ed _field_ is 116 | passed to a predicate function `fn` and the `fn` returns true 117 | - `opts` are additional configurations such as `indexType` and `name`. You 118 | SHOULD pass `opts.name` as a simple string uniquely identifying the predicate, 119 | OR the `fn` function should be a named function 120 | - `slowPredicate(objPath, fn, opts)` is to `predicate` what `slowEqual` is to 121 | `equal` 122 | - `absent(seek, opts)` filters for messages where a `seek`ed _field_ does not 123 | exist in the message 124 | - `slowAbsent(objPath)` is to `absent` what `slowEqual` is to `equal` 125 | 126 | Some examples: 127 | 128 | **Get all messages of type `post`:** 129 | 130 | ```js 131 | query( 132 | fromDB(db), 133 | where(slowEqual('value.content.type', 'post')), 134 | toCallback((err, msgs) => { 135 | console.log('There are ' + msgs.length + ' messages of type "post"') 136 | }) 137 | ) 138 | ``` 139 | 140 | **Same as above but faster performance (recommended in production):** 141 | 142 | ```js 143 | query( 144 | fromDB(db), 145 | where(equal(seekType, 'post', { indexType: 'type' })), 146 | toCallback((err, msgs) => { 147 | console.log('There are ' + msgs.length + ' messages of type "post"') 148 | }) 149 | ) 150 | 151 | // The `seekType` function takes a buffer and uses `bipf` APIs to search for 152 | // the fields we want. 153 | const bValue = Buffer.from('value') // better for performance if defined outside 154 | const bContent = Buffer.from('content') 155 | const bType = Buffer.from('type') 156 | function seekType(buffer) { 157 | var p = 0 // p stands for "position" in the buffer, offset from start 158 | p = bipf.seekKey(buffer, p, bValue) 159 | if (p < 0) return 160 | p = bipf.seekKey(buffer, p, bContent) 161 | if (p < 0) return 162 | return bipf.seekKey(buffer, p, bType) 163 | } 164 | ``` 165 | 166 | **Get all messages of type `contact` from Alice or Bob:** 167 | 168 | ```js 169 | query( 170 | fromDB(db), 171 | where( 172 | and( 173 | slowEqual('value.content.type', 'contact'), 174 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)) 175 | ) 176 | ), 177 | toCallback((err, msgs) => { 178 | console.log('There are ' + msgs.length + ' messages') 179 | }) 180 | ) 181 | ``` 182 | 183 | **Same as above but faster performance (recommended in production):** 184 | 185 | ```js 186 | query( 187 | fromDB(db), 188 | where( 189 | and( 190 | equal(seekType, 'contact', 'type') 191 | or( 192 | equal(seekAuthor, aliceId, { indexType: 'author' }), 193 | equal(seekAuthor, bobId, { indexType: 'author' }) 194 | ) 195 | ) 196 | ), 197 | toCallback((err, msgs) => { 198 | console.log('There are ' + msgs.length + ' messages') 199 | }) 200 | ) 201 | 202 | // where seekAuthor is 203 | const bValue = Buffer.from('value') // better for performance if defined outside 204 | const bAuthor = Buffer.from('author') 205 | function seekAuthor(buffer) { 206 | var p = 0 207 | p = bipf.seekKey(buffer, p, bValue) 208 | if (p < 0) return 209 | return bipf.seekKey(buffer, p, bAuthor) 210 | } 211 | ``` 212 | 213 | #### Pagination 214 | 215 | If you use `toCallback`, it gives you all results in one go. If you 216 | want to get results in batches, you should use **`toPullStream`**, 217 | **`paginate`**, and optionally `startFrom` and `descending`. 218 | 219 | - **toPullStream** creates a [pull-stream] source to stream the results 220 | - **paginate** configures the size of each array sent to the pull-stream source 221 | - **startFrom** configures the beginning seq from where to start streaming 222 | - **descending** configures the pagination stream to order results 223 | from newest to oldest (otherwise the default order is oldest to 224 | newest) based on timestamp 225 | 226 | Example, **stream all messages of type `contact` from Alice or Bob in pages of size 10:** 227 | 228 | ```js 229 | const pull = require('pull-stream') 230 | 231 | const source = query( 232 | fromDB(db), 233 | where( 234 | and( 235 | slowEqual('value.content.type', 'contact') 236 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)), 237 | ), 238 | ), 239 | paginate(10), 240 | toPullStream() 241 | ) 242 | 243 | pull( 244 | source, 245 | pull.drain((msgs) => { 246 | console.log('next page') 247 | console.log(msgs) 248 | }) 249 | ) 250 | ``` 251 | 252 | **Stream all messages of type `contact` from Alice or Bob in pages of 253 | size 10, starting from the 15th message, sorted from newest to 254 | oldest:** 255 | 256 | ```js 257 | const pull = require('pull-stream') 258 | 259 | const source = query( 260 | fromDB(db), 261 | where( 262 | and( 263 | slowEqual('value.content.type', 'contact') 264 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)), 265 | ), 266 | ), 267 | paginate(10), 268 | startFrom(15), 269 | descending(), 270 | toPullStream() 271 | ) 272 | 273 | pull( 274 | source, 275 | pull.drain((msgs) => { 276 | console.log('next page:') 277 | console.log(msgs) 278 | }) 279 | ) 280 | ``` 281 | 282 | **Batching** with the operator `batch()` is similar to pagination in terms of 283 | performance, but the messages are delivered one-by-one to the final pull-stream, 284 | instead of as any array. Example: 285 | 286 | ```js 287 | const pull = require('pull-stream') 288 | 289 | const source = query( 290 | fromDB(db), 291 | where( 292 | and( 293 | slowEqual('value.content.type', 'contact') 294 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)), 295 | ), 296 | ), 297 | batch(10), // Note `batch` instead of `paginate` 298 | descending(), 299 | toPullStream() 300 | ) 301 | 302 | pull( 303 | source, 304 | // Note the below drain is `msg`, not `msgs` array: 305 | pull.drain((msg) => { 306 | console.log('next message:') 307 | console.log(msg) 308 | }) 309 | ) 310 | ``` 311 | 312 | #### async/await 313 | 314 | There are also operators that support getting the values using 315 | `await`. **`toPromise`** is like `toCallback`, delivering all results 316 | at once: 317 | 318 | ```js 319 | const msgs = await query( 320 | fromDB(db), 321 | where( 322 | and( 323 | slowEqual('value.content.type', 'contact') 324 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)), 325 | ), 326 | ), 327 | toPromise() 328 | ) 329 | 330 | console.log('There are ' + msgs.length + ' messages') 331 | ``` 332 | 333 | With pagination, **`toAsyncIter`** is like **`toPullStream`**, streaming the results in batches: 334 | 335 | ```js 336 | const results = query( 337 | fromDB(db), 338 | where( 339 | and( 340 | slowEqual('value.content.type', 'contact') 341 | or(slowEqual('value.author', aliceId), slowEqual('value.author', bobId)), 342 | ), 343 | ), 344 | paginate(10), 345 | startFrom(15), 346 | toAsyncIter() 347 | ) 348 | 349 | for await (let msgs of results) { 350 | console.log('next page:') 351 | console.log(msgs) 352 | } 353 | ``` 354 | 355 | #### Custom indexes and `deferred` operator 356 | 357 | There may be custom indexes external to JITDB, in which case you 358 | should convert the results from those indexes to `offsets()` or 359 | `seqs()` (read more about these in the low level API section). In 360 | those cases, the `OFFSETS` or `SEQS` are often received 361 | asynchronously. To support piping these async results in the `query` 362 | chain, we have the `deferred()` operator which postpones the fetching 363 | of results from your custom index, but allows you to compose 364 | operations nevertheless. 365 | 366 | ```js 367 | // operator 368 | deferred(task) 369 | ``` 370 | 371 | where `task` is any function of the format 372 | 373 | ```js 374 | function task(meta, cb[, onAbort]) 375 | ``` 376 | 377 | where `meta` is an object containing an instance of JITDB and other 378 | metadata, and `onAbort` is an optional function that you can pass an 379 | abort listener (i.e. `onAbort(() => { /* cancel my stuff */ })`). 380 | 381 | As an example, suppose you have a custom index that returns seqs 382 | `11`, `13` and `17`, and you want to include these results into your 383 | operator chain, to `AND` them with a specific author. Use `deferred` 384 | like this: 385 | 386 | ```js 387 | query( 388 | fromDB(db), 389 | deferred((meta, cb) => { 390 | // do something asynchronously, then deliver results to cb 391 | cb(null, seqs([11, 13, 17])) 392 | }), 393 | where(slowEqual('value.author', aliceId)), 394 | toCallback((err, results) => { 395 | console.log(results) 396 | }) 397 | ) 398 | ``` 399 | 400 | #### All operators 401 | 402 | This is a list of all the operators supported so far: 403 | 404 | ```js 405 | const { 406 | fromDB, 407 | query, 408 | where, 409 | and, 410 | or, 411 | not, 412 | equal, 413 | slowEqual, 414 | predicate, 415 | slowPredicate, 416 | absent, 417 | slowAbsent, 418 | includes, 419 | slowIncludes, 420 | gt, 421 | gte, 422 | lt, 423 | lte, 424 | deferred, 425 | liveSeqs, 426 | seqs, 427 | offsets, 428 | count, 429 | paginate, 430 | batch, 431 | startFrom, 432 | descending, 433 | asOffsets, 434 | sortByArrival, 435 | debug, 436 | toCallback, 437 | toPullStream, 438 | toPromise, 439 | toAsyncIter, 440 | } = require('jitdb/operators') 441 | ``` 442 | 443 | ## Prefix indexes 444 | 445 | Most indexes in JITDB are bitvectors, which are suitable for answering 446 | boolean queries such as "is this msg a post?" or "is this msg from 447 | author A?". For each of these queries, JITDB creates one file. 448 | 449 | This is fine for several cases, but some queries are not 450 | boolean. Queries on bitvectors such as "is this msg a reply to msg X?" 451 | can end up generating `N` files if the "msg X" can have N different 452 | values. The creation of indexes is this case becomes the overhead. 453 | 454 | **Prefix indexes** help in that case because they can answer 455 | non-boolean queries with multiple different values but using just one 456 | index file. For example, for N different values of "msg X", just one 457 | prefix index is enough for answering "is this msg a reply to msg X?". 458 | 459 | The way prefix indexes work is that for each message in the log, it 460 | picks the first 32 bits of a field in the message (hence 'prefix') and 461 | then compares your desired value with all of these prefixes. It 462 | doesn't store the whole value because that could turn out wasteful in 463 | storage and memory as the log scales (to 1 million or more 464 | messages). Storing just a prefix is not enough for uniqueness, though, 465 | as different values will have the same prefix, so queries on prefix 466 | indexes will create false positives, but JITDB does an additional 467 | check so in the resulting query, **you will not get false positives**. 468 | 469 | _Rule of thumb_: use prefix indexes in an EQUAL operation only when 470 | the target `value` of your EQUAL can dynamically assume many (more 471 | than a dozen) possible values. 472 | 473 | An additional option `useMap` can be provided that will store the 474 | prefix as a map instead of an array. The map can be seen as an 475 | inverted index that allows for faster queries at the cost of extra 476 | space. Maps don't store empty values meaning they are also a good fit 477 | for sparce indexes such as vote links. 478 | 479 | It is possible specifiy where in the target the prefix buffer should 480 | be constructed from using `prefixOffset`. This is useful for targets 481 | that starts with a common prefix such as % in order to increase the 482 | information amount. 483 | 484 | ## Low-level API 485 | 486 | First some terminology: offset refers to the byte position in the log 487 | of a message. Seq refers to the 0-based position of a message in the 488 | log. 489 | 490 | ### paginate(operation, seq, limit, descending, onlyOffset, sortBy, latestMsgKeyPrecompaction, cb) 491 | 492 | Query the database returning paginated results. If one or more indexes 493 | doesn't exist or are outdated, the indexes will be updated before the 494 | query is run. `onlyOffset` can be used to return offset instead of the 495 | actual messages. `sortBy` determines what timestamp to use for 496 | ordering messages. Can take values `declared` or `arrival`. `declared` 497 | refers to the timestamp for when a message was created, while 498 | `arrival` refers to when a message was added to the database. This can 499 | be important for messages from other peers that might arrive out of 500 | order compared when they were created. 501 | 502 | `latestMsgKeyPrecompaction` is used internally, and it's safe to just pass 503 | `null` as its value when you're calling `paginate`. 504 | 505 | The result is an object with the fields: 506 | 507 | - `data`: the actual messages 508 | - `total`: the total number of messages 509 | - `duration`: the number of ms the query took 510 | 511 | Operation can be of the following types: 512 | 513 | | type | data | 514 | | ------------- | -------------------------------------------- | 515 | | EQUAL | { seek, value, indexType, indexAll, prefix } | 516 | | GT,GTE,LT,LTE | { indexName, value } | 517 | | OFFSETS | { offsets } | 518 | | SEQS | { seqs } | 519 | | NOT | [operation] | 520 | | AND | [operation, operation] | 521 | | OR | [operation, operation] | 522 | 523 | `seek` is a function that takes a buffer from the database as input 524 | and returns an index in the buffer from where a value can be compared 525 | to the `value` given. `value` must be a bipf encoded value, usually 526 | the `equal` operator will take care of that. A field not being defined 527 | at a point in the buffer is equal to `undefined`. `prefix` enables the 528 | use of prefix indexes for this operation. `indexType` is used to group 529 | indexes of the same type. If `indexAll` is specified and no index of 530 | the type and value exists, then instead of only this index being 531 | created, missing indexes for all possible values given the seek 532 | pointer will be created. This can be particular useful for data where 533 | there number of different values are rather small, but still larger 534 | than a few. One example is author or feeds in SSB, a typical database 535 | of 1 million records will have roughly 700 authors. The biggest cost 536 | in creating the indexes is traversing the database, so creating all 537 | indexes in one go instead of several hundreds is a lot faster. 538 | 539 | For `GT`, `GTE`, `LT` and `LTE`, `indexName` can be either `sequence` 540 | or `timestamp`. 541 | 542 | `SEQS` and `OFFSETS` allow one to use seq and offset (respectively) 543 | positions into the log file as query operators. This is useful for 544 | interfacing with data indexed by something else than JITDB. Seqs 545 | are faster as they can be combined in queries directly. 546 | 547 | Example 548 | 549 | ``` 550 | { 551 | type: 'AND', 552 | data: [ 553 | { type: 'EQUAL', data: { seek: db.seekType, value: 'post', indexType: "type" } }, 554 | { type: 'EQUAL', data: { seek: db.seekAuthor, value: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519', indexType: "author" } } 555 | ] 556 | } 557 | ``` 558 | 559 | I considered adding an option to return raw buffers in order to do 560 | some after processing that you wouldn't create and index for, but the 561 | overhead of decoding the buffers is small enough that I don't think it 562 | makes sense. 563 | 564 | ### all(operation, seq, descending, onlyOffset, sortBy, cb) 565 | 566 | Similar to `paginate` except there is no `limit` argument and the result 567 | will be the messages directly. 568 | 569 | ### count(operation, seq, descending, cb) 570 | 571 | Similar to `all` except it does not fetch records from the log, it only responds with a number that tells the total amount of records matching the operation. 572 | 573 | ### prepare(operation, cb) 574 | 575 | Ensures that the indexes in the `operation` are up-to-date by creating or 576 | updating them, if necessary. This is not a query, as it won't return any 577 | results. When done, the callback `cb` is just called with the "duration" of 578 | index preparation as the second argument. 579 | 580 | ### lookup(operation, seq, cb) 581 | 582 | Given one `seq`, lookup its corresponding value on the index specified by 583 | `operation`, which is either an operation object or a string for the name of a 584 | core index, such as `'seq'` and `'timestamp'`. The callback `cb` is called with 585 | the index's value (at that `seq` position) in the 2nd arg. 586 | 587 | ### live(operation, cb) 588 | 589 | Will setup a pull stream and this in `cb`. The pull stream will emit 590 | new values as they are added to the underlying log. This is meant to 591 | run after `paginate` or `all`. 592 | 593 | Please note the index is _not_ updated when using this method and only 594 | one live seqs stream is supported. 595 | 596 | ### reindex(offset, cb) 597 | 598 | Reset all indexes to start from `offset`. This means that on the next 599 | query, messages from that offset and forward will be reindexed. 600 | 601 | ### onReady(cb) 602 | 603 | Will call when all existing indexes have been loaded. 604 | 605 | [flume]: https://github.com/flumedb/ 606 | [async-append-only-log]: https://github.com/ssb-ngi-pointer/async-append-only-log 607 | [bipf]: https://github.com/dominictarr/bipf/ 608 | [pull-stream]: https://github.com/pull-stream/pull-stream 609 | -------------------------------------------------------------------------------- /bench.txt: -------------------------------------------------------------------------------- 1 | 6 | 7 | async-flumelog (bipf): 8 | 9 | time: 8965ms, total items: 1414275 10 | get all: 67ms, total items: 1565 11 | get all posts from user: 9.059s 12 | results 1565 13 | get values and sort top 10: 3.495ms 14 | get last 10 posts from user: 7.235ms 15 | results 254016 16 | get values and sort top 50: 236.186ms 17 | get top 50 posts: 239.369ms 18 | time: 4323ms, total items: 1414275 19 | get all: 3866ms, total items: 301486 20 | contacts: 8.999s 21 | 22 | real 0m18,531s 23 | user 0m21,008s 24 | sys 0m1,724s 25 | 26 | flumelog-offset (bipf): 27 | 28 | time: 11285ms, total items: 1414454 29 | get all: 69ms, total items: 1565 30 | get all posts from user: 11.378s 31 | results 1565 32 | get values and sort top 10: 3.526ms 33 | get last 10 posts from user: 6.671ms 34 | results 254047 35 | get values and sort top 50: 267.493ms 36 | get top 50 posts: 270.632ms 37 | done 1414454 38 | time: 7081ms, total items: 1414454 39 | saving index:type_contact 40 | get all: 4368ms, total items: 301532 41 | contacts: 12.262s 42 | 43 | real 0m24,142s 44 | user 0m27,071s 45 | sys 0m2,083s 46 | -------------------------------------------------------------------------------- /benchmark/helpers/run_benchmark.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const prettyBytes = require('pretty-bytes') 6 | const gc = require('expose-gc/function') 7 | const nodemark = require('nodemark') 8 | 9 | const prettyBytesOptions = { 10 | maximumFractionDigits: 2 11 | } 12 | 13 | const heapToString = function() { 14 | const formattedMean = prettyBytes(this.mean, prettyBytesOptions) 15 | const [mean, units] = formattedMean.split(' ') 16 | const formattedError = `${Math.round(Math.abs(this.error * mean) * 100)/100} ${units}` 17 | return `${formattedMean} \xb1 ${formattedError}` 18 | } 19 | 20 | const statsToString = function() { 21 | const opsMs = this.ops.milliseconds(2) 22 | const opsError = Math.round(this.ops.error * opsMs * 100) / 100 23 | return `| ${this.name} | ${opsMs}ms \xb1 ${opsError}ms | ${this.heap} | ${this.ops.count} |\n` 24 | } 25 | 26 | function runBenchmark(benchmarkName, benchmarkFn, setupFn, callback) { 27 | let samples 28 | let oldMemory 29 | function calcMemUsage() { 30 | const newMemory = process.memoryUsage().heapUsed 31 | if (oldMemory === void 0) { 32 | oldMemory = newMemory 33 | } else { 34 | samples.push(newMemory - oldMemory) 35 | oldMemory = newMemory 36 | } 37 | } 38 | 39 | function onCycle(cb) { 40 | calcMemUsage() 41 | cb() 42 | } 43 | function onStart(cb) { 44 | samples = [] 45 | gc() 46 | cb() 47 | } 48 | function getTestStats(name, ops) { 49 | // Remove part before v8 optimizations, mimicking nodemark 50 | samples = samples.slice(samples.length - ops.count) 51 | let sum = 0 52 | let sumSquaredValues = 0 53 | for (const val of samples) { 54 | sum += val 55 | sumSquaredValues += val * val 56 | } 57 | const count = samples.length; 58 | const sumOfSquares = sumSquaredValues - sum * sum / count; 59 | const standardDeviation = Math.sqrt(sumOfSquares / (count - 1)); 60 | const criticalValue = 2 / Math.sqrt(count); 61 | const marginOfError = criticalValue * Math.sqrt(standardDeviation * standardDeviation / count); 62 | const mean = sum / count; 63 | const error = (marginOfError / Math.abs(mean)); 64 | const heap = { 65 | mean, 66 | error, 67 | toString: heapToString 68 | } 69 | return { 70 | name, 71 | ops, 72 | heap, 73 | toString: statsToString 74 | } 75 | } 76 | 77 | onStart(function(err) { 78 | if (err) return callback(err) 79 | nodemark( 80 | benchmarkFn, 81 | (cb) => { 82 | onCycle(function(err2) { 83 | if (err2) return cb(err2) 84 | else setupFn(function(err3) { 85 | if (err3) cb(err3) 86 | else { 87 | gc() 88 | cb() 89 | } 90 | }) 91 | }) 92 | }, 93 | Number(process.env.BENCHMARK_DURATION_MS || 3000) 94 | ).then(result => { 95 | callback(null, getTestStats(benchmarkName, result)) 96 | }).catch(e => { 97 | callback(e) 98 | }) 99 | }) 100 | } 101 | 102 | module.exports = runBenchmark -------------------------------------------------------------------------------- /benchmark/helpers/setup_test_functions.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const tape = require('tape') 6 | 7 | const skipCreate = 8 | process.argv[2] === 'noCreate' || 9 | !!process.env.GET_BENCHMARK_MATRIX || 10 | !!process.env.CURRENT_BENCHMARK 11 | exports.skipCreate = skipCreate 12 | const testList = [] 13 | if (process.env.GET_BENCHMARK_MATRIX) { 14 | process.on('exit', ({ exit }) => { 15 | console.log(JSON.stringify(testList)) 16 | if (exit) process.exit() 17 | }) 18 | } 19 | const fixture = (name, ...args) => { 20 | if (process.env.GET_BENCHMARK_MATRIX) { 21 | return 22 | } else if (process.env.FIXTURES_ONLY) { 23 | tape(name, ...args) 24 | } else if (process.env.CURRENT_BENCHMARK) { 25 | tape(name, (t) => { 26 | t.skip() 27 | t.end() 28 | }) 29 | } else { 30 | tape(name, ...args) 31 | } 32 | } 33 | exports.fixture = fixture 34 | const test = (name, ...args) => { 35 | if (process.env.GET_BENCHMARK_MATRIX) { 36 | testList.push(name) 37 | } else if (process.env.FIXTURES_ONLY) { 38 | tape(name, (t) => { 39 | t.skip() 40 | t.end() 41 | }) 42 | } else if (process.env.CURRENT_BENCHMARK) { 43 | if (name.startsWith(process.env.CURRENT_BENCHMARK)) { 44 | tape.only(name, ...args) 45 | } else { 46 | tape(name, ...args) 47 | } 48 | } else { 49 | tape(name, ...args) 50 | } 51 | } 52 | exports.test = test 53 | -------------------------------------------------------------------------------- /benchmark/index.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const fs = require('fs') 6 | const path = require('path') 7 | const pull = require('pull-stream') 8 | const Log = require('async-append-only-log') 9 | const generateFixture = require('ssb-fixtures') 10 | const rimraf = require('rimraf') 11 | const mkdirp = require('mkdirp') 12 | const multicb = require('multicb') 13 | const ssbKeys = require('ssb-keys') 14 | const TypedFastBitSet = require('typedfastbitset') 15 | const runBenchmark = require('./helpers/run_benchmark') 16 | const JITDB = require('../index') 17 | const { 18 | query, 19 | fromDB, 20 | where, 21 | and, 22 | or, 23 | equal, 24 | count, 25 | toCallback, 26 | toPullStream, 27 | startFrom, 28 | paginate, 29 | } = require('../operators') 30 | const { seekType, seekAuthor, seekVoteLink } = require('../test/helpers') 31 | const copy = require('../copy-json-to-bipf-async') 32 | const { skipCreate, fixture, test } = require("./helpers/setup_test_functions") 33 | 34 | const dir = '/tmp/jitdb-benchmark' 35 | const oldLogPath = path.join(dir, 'flume', 'log.offset') 36 | const newLogPath = path.join(dir, 'flume', 'log.bipf') 37 | const reportPath = path.join(dir, 'benchmark.md') 38 | const indexesDir = path.join(dir, 'indexes') 39 | 40 | /** 41 | * Wait for a file to exist and for writes to that file 42 | * to complete 43 | * 44 | * @param {string} filepath 45 | * @param {Function} cb 46 | */ 47 | const waitForFile = (filepath, cb) => { 48 | const maxTime = 5000 49 | const interval = 500 50 | let timeUsed = 0 51 | let fileSize = 0 52 | function fileReady() { 53 | fs.stat(filepath, (err, stats) => { 54 | if (err) { 55 | if (timeUsed < maxTime) { 56 | timeUsed += interval 57 | setTimeout(fileReady, interval) 58 | } else { 59 | cb(err) 60 | } 61 | } else if (stats.size > fileSize) { 62 | if (timeUsed < maxTime) { 63 | fileSize = stats.size 64 | timeUsed += interval 65 | setTimeout(fileReady, interval) 66 | } else { 67 | cb(new Error(`Timed out waiting for ${filepath} to finish writing`)) 68 | } 69 | } else { 70 | cb() 71 | } 72 | }) 73 | } 74 | setTimeout(fileReady, interval) 75 | } 76 | 77 | let alice 78 | let bob 79 | if (!skipCreate) { 80 | rimraf.sync(dir) 81 | mkdirp.sync(dir) 82 | 83 | const SEED = 'sloop' 84 | const MESSAGES = 100000 85 | const AUTHORS = 2000 86 | 87 | fixture('generate fixture with flumelog-offset', (t) => { 88 | generateFixture({ 89 | outputDir: dir, 90 | seed: SEED, 91 | messages: MESSAGES, 92 | authors: AUTHORS, 93 | slim: true, 94 | }).then(() => { 95 | t.pass(`seed = ${SEED}`) 96 | t.pass(`messages = ${MESSAGES}`) 97 | t.pass(`authors = ${AUTHORS}`) 98 | t.true(fs.existsSync(oldLogPath), 'log.offset was created') 99 | alice = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 100 | bob = ssbKeys.loadOrCreateSync(path.join(dir, 'secret-b')) 101 | fs.appendFileSync(reportPath, '## Benchmark results\n\n') 102 | fs.appendFileSync(reportPath, '| Part | Speed | Heap Change | Samples |\n|---|---|---|---|\n') 103 | t.end() 104 | }) 105 | }) 106 | 107 | fixture('move flumelog-offset to async-log', (t) => { 108 | copy(oldLogPath, newLogPath, (err) => { 109 | if (err) { 110 | t.fail(err) 111 | t.end() 112 | return 113 | } 114 | waitForFile( 115 | newLogPath, 116 | (err) => { 117 | if (err) { 118 | t.fail(err) 119 | t.end() 120 | } else { 121 | t.pass('log.bipf was created') 122 | t.end() 123 | } 124 | } 125 | ) 126 | }) 127 | }) 128 | } else { 129 | alice = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 130 | bob = ssbKeys.loadOrCreateSync(path.join(dir, 'secret-b')) 131 | } 132 | 133 | let raf 134 | let db 135 | 136 | const getJitdbReady = (cb) => { 137 | raf = Log(newLogPath, { blockSize: 64 * 1024 }) 138 | db = JITDB(raf, indexesDir) 139 | db.onReady((err) => { 140 | cb(err) 141 | }) 142 | } 143 | 144 | const closeLog = (cb) => { 145 | if (raf) { 146 | raf.close((err) => { 147 | if (err) cb(err) 148 | else rimraf(indexesDir, cb) 149 | }) 150 | } else { 151 | rimraf(indexesDir, cb) 152 | } 153 | } 154 | 155 | test('core indexes', (t) => { 156 | runBenchmark( 157 | 'Load core indexes', 158 | getJitdbReady, 159 | closeLog, 160 | (err, result) => { 161 | closeLog((err2) => { 162 | if (err || err2) { 163 | t.fail(err || err2) 164 | } else { 165 | fs.appendFileSync(reportPath, result.toString()) 166 | t.pass(result) 167 | } 168 | t.end() 169 | }) 170 | } 171 | ) 172 | }) 173 | 174 | const runHugeIndexQuery = (cb) => { 175 | query( 176 | fromDB(db), 177 | where(equal(seekType, 'post', { indexType: 'type' })), 178 | toCallback((err, msgs) => { 179 | if (err) { 180 | cb(err) 181 | } else if (msgs.length !== 23310) { 182 | cb(new Error('msgs.length is wrong: ' + msgs.length)) 183 | } 184 | cb() 185 | }) 186 | ) 187 | } 188 | 189 | test('query one huge index (first run)', (t) => { 190 | runBenchmark( 191 | 'Query 1 big index (1st run)', 192 | runHugeIndexQuery, 193 | (cb) => { 194 | closeLog((err) => { 195 | if (err) cb(err) 196 | else getJitdbReady(cb) 197 | }) 198 | }, 199 | (err, result) => { 200 | if (err) { 201 | t.fail(err) 202 | } else { 203 | fs.appendFileSync(reportPath, result.toString()) 204 | t.pass(result) 205 | } 206 | t.end() 207 | } 208 | ) 209 | }) 210 | 211 | test('query one huge index (second run)', (t) => { 212 | runBenchmark( 213 | 'Query 1 big index (2nd run)', 214 | runHugeIndexQuery, 215 | (cb) => { 216 | closeLog((err) => { 217 | if (err) cb(err) 218 | else getJitdbReady((err2) => { 219 | if (err2) cb(err2) 220 | else runHugeIndexQuery(cb) 221 | }) 222 | }) 223 | }, 224 | (err, result) => { 225 | if (err) { 226 | t.fail(err) 227 | } else { 228 | fs.appendFileSync(reportPath, result.toString()) 229 | t.pass(result) 230 | } 231 | t.end() 232 | } 233 | ) 234 | }) 235 | 236 | test('count one huge index (third run)', (t) => { 237 | runBenchmark( 238 | 'Count 1 big index (3rd run)', 239 | (cb) => { 240 | query( 241 | fromDB(db), 242 | where(equal(seekType, 'post', { indexType: 'type' })), 243 | count(), 244 | toCallback((err, total) => { 245 | if (err) { 246 | cb(err) 247 | } else if (total !== 23310) { 248 | cb(new Error('total is wrong: ' + total)) 249 | } 250 | cb() 251 | }) 252 | ) 253 | }, 254 | (cb) => { 255 | closeLog((err) => { 256 | if (err) cb(err) 257 | else getJitdbReady((err2) => { 258 | if (err2) cb(err2) 259 | else runHugeIndexQuery((err3) => { 260 | if (err3) cb(err3) 261 | else runHugeIndexQuery(cb) 262 | }) 263 | }) 264 | }) 265 | }, 266 | (err, result) => { 267 | if (err) { 268 | t.fail(err) 269 | } else { 270 | fs.appendFileSync(reportPath, result.toString()) 271 | t.pass(result) 272 | } 273 | t.end() 274 | } 275 | ) 276 | }) 277 | 278 | test('create an index twice concurrently', (t) => { 279 | let done 280 | runBenchmark( 281 | 'Create an index twice concurrently', 282 | (cb) => { 283 | query( 284 | fromDB(db), 285 | where(equal(seekType, 'about', { indexType: 'type' })), 286 | toCallback(done()) 287 | ) 288 | 289 | query( 290 | fromDB(db), 291 | where(equal(seekType, 'about', { indexType: 'type' })), 292 | toCallback(done()) 293 | ) 294 | 295 | done((err) => { 296 | if (err) cb(err) 297 | else cb() 298 | }) 299 | }, 300 | (cb) => { 301 | done = multicb({ pluck: 1 }) 302 | closeLog((err) => { 303 | if (err) cb(err) 304 | else getJitdbReady(cb) 305 | }) 306 | }, 307 | (err, result) => { 308 | if (err) { 309 | t.fail(err) 310 | } else { 311 | fs.appendFileSync(reportPath, result.toString()) 312 | t.pass(result) 313 | } 314 | t.end() 315 | } 316 | ) 317 | }) 318 | 319 | const runThreeIndexQuery = (cb) => { 320 | query( 321 | fromDB(db), 322 | where( 323 | or( 324 | equal(seekType, 'contact', { indexType: 'type' }), 325 | equal(seekAuthor, alice.id, { 326 | indexType: 'author', 327 | prefix: 32, 328 | prefixOffset: 1, 329 | }), 330 | equal(seekAuthor, bob.id, { 331 | indexType: 'author', 332 | prefix: 32, 333 | prefixOffset: 1, 334 | }) 335 | ) 336 | ), 337 | toCallback((err, msgs) => { 338 | if (err) cb(err) 339 | else if (msgs.length !== 24606) 340 | cb(new Error('msgs.length is wrong: ' + msgs.length)) 341 | else cb() 342 | }) 343 | ) 344 | } 345 | 346 | test('query three indexes (first run)', (t) => { 347 | runBenchmark( 348 | 'Query 3 indexes (1st run)', 349 | runThreeIndexQuery, 350 | (cb) => { 351 | closeLog((err) => { 352 | if (err) cb(err) 353 | else getJitdbReady(cb) 354 | }) 355 | }, 356 | (err, result) => { 357 | if (err) { 358 | t.fail(err) 359 | } else { 360 | fs.appendFileSync(reportPath, result.toString()) 361 | t.pass(result) 362 | } 363 | t.end() 364 | } 365 | ) 366 | }) 367 | 368 | test('query three indexes (second run)', (t) => { 369 | runBenchmark( 370 | 'Query 3 indexes (2nd run)', 371 | runThreeIndexQuery, 372 | (cb) => { 373 | closeLog((err) => { 374 | if (err) cb(err) 375 | else getJitdbReady((err) => { 376 | if (err) cb(err) 377 | else runThreeIndexQuery(cb) 378 | }) 379 | }) 380 | }, 381 | (err, result) => { 382 | if (err) { 383 | t.fail(err) 384 | } else { 385 | fs.appendFileSync(reportPath, result.toString()) 386 | t.pass(result) 387 | } 388 | t.end() 389 | } 390 | ) 391 | }) 392 | 393 | const useContactIndex = (cb) => { 394 | const filepath = path.join(indexesDir, 'type_contact.index') 395 | waitForFile( 396 | filepath, 397 | (err) => { 398 | if (err) cb(err) 399 | else { 400 | db.indexes['type_contact'] = { 401 | offset: 0, 402 | bitset: new TypedFastBitSet(), 403 | lazy: true, 404 | filepath, 405 | } 406 | cb() 407 | } 408 | } 409 | ) 410 | } 411 | 412 | test('load two indexes concurrently', (t) => { 413 | let done 414 | runBenchmark( 415 | 'Load two indexes concurrently', 416 | (cb) => { 417 | query( 418 | fromDB(db), 419 | where( 420 | or( 421 | equal(seekType, 'contact', { indexType: 'type' }), 422 | equal(seekAuthor, alice.id, { 423 | indexType: 'author', 424 | prefix: 32, 425 | prefixOffset: 1, 426 | }), 427 | equal(seekAuthor, bob.id, { 428 | indexType: 'author', 429 | prefix: 32, 430 | prefixOffset: 1, 431 | }) 432 | ) 433 | ), 434 | toCallback(done()) 435 | ) 436 | 437 | query( 438 | fromDB(db), 439 | where( 440 | or( 441 | equal(seekType, 'contact', { indexType: 'type' }), 442 | equal(seekAuthor, alice.id, { 443 | indexType: 'author', 444 | prefix: 32, 445 | prefixOffset: 1, 446 | }), 447 | equal(seekAuthor, bob.id, { 448 | indexType: 'author', 449 | prefix: 32, 450 | prefixOffset: 1, 451 | }) 452 | ) 453 | ), 454 | toCallback(done()) 455 | ) 456 | 457 | done((err) => { 458 | if (err) cb(err) 459 | else cb() 460 | }) 461 | }, 462 | (cb) => { 463 | closeLog((err) => { 464 | if (err) cb(err) 465 | else getJitdbReady((err) => { 466 | if (err) cb(err) 467 | else runThreeIndexQuery((err) => { 468 | if (err) cb(err) 469 | else { 470 | done = multicb({ pluck: 1 }) 471 | useContactIndex(cb) 472 | } 473 | }) 474 | }) 475 | }) 476 | }, 477 | (err, result) => { 478 | if (err) { 479 | t.fail(err) 480 | } else { 481 | fs.appendFileSync(reportPath, result.toString()) 482 | t.pass(result) 483 | } 484 | t.end() 485 | } 486 | ) 487 | }) 488 | 489 | test('paginate big index with small pageSize', (t) => { 490 | const TOTAL = 20000 491 | const PAGESIZE = 5 492 | const NUMPAGES = TOTAL / PAGESIZE 493 | runBenchmark( 494 | `Paginate ${TOTAL} msgs with pageSize=${PAGESIZE}`, 495 | (cb) => { 496 | let i = 0 497 | pull( 498 | query( 499 | fromDB(db), 500 | where(equal(seekType, 'post', { indexType: 'type' })), 501 | paginate(PAGESIZE), 502 | toPullStream() 503 | ), 504 | pull.take(NUMPAGES), 505 | pull.drain( 506 | (msgs) => { 507 | i++ 508 | }, 509 | (err) => { 510 | if (err) cb(err) 511 | else if (i !== NUMPAGES) cb(new Error('wrong number of pages read: ' + i)) 512 | else cb() 513 | } 514 | ) 515 | ) 516 | }, 517 | (cb) => { 518 | closeLog((err) => { 519 | if (err) cb(err) 520 | else getJitdbReady((err) => { 521 | if (err) cb(err) 522 | else runThreeIndexQuery((err) => { 523 | if (err) cb(err) 524 | else { 525 | done = multicb({ pluck: 1 }) 526 | useContactIndex(cb) 527 | } 528 | }) 529 | }) 530 | }) 531 | }, 532 | (err, result) => { 533 | if (err) { 534 | t.fail(err) 535 | } else { 536 | fs.appendFileSync(reportPath, result.toString()) 537 | t.pass(result) 538 | } 539 | t.end() 540 | } 541 | ) 542 | }) 543 | 544 | test('paginate big index with big pageSize', (t) => { 545 | const TOTAL = 20000 546 | const PAGESIZE = 500 547 | const NUMPAGES = TOTAL / PAGESIZE 548 | runBenchmark( 549 | `Paginate ${TOTAL} msgs with pageSize=${PAGESIZE}`, 550 | (cb) => { 551 | let i = 0 552 | pull( 553 | query( 554 | fromDB(db), 555 | where(equal(seekType, 'post', { indexType: 'type' })), 556 | paginate(PAGESIZE), 557 | toPullStream() 558 | ), 559 | pull.take(NUMPAGES), 560 | pull.drain( 561 | (msgs) => { 562 | i++ 563 | }, 564 | (err) => { 565 | if (err) cb(err) 566 | else if (i !== NUMPAGES) cb(new Error('wrong number of pages read: ' + i)) 567 | else cb() 568 | } 569 | ) 570 | ) 571 | }, 572 | (cb) => { 573 | closeLog((err) => { 574 | if (err) cb(err) 575 | else getJitdbReady((err) => { 576 | if (err) cb(err) 577 | else runThreeIndexQuery((err) => { 578 | if (err) cb(err) 579 | else { 580 | done = multicb({ pluck: 1 }) 581 | useContactIndex(cb) 582 | } 583 | }) 584 | }) 585 | }) 586 | }, 587 | (err, result) => { 588 | if (err) { 589 | t.fail(err) 590 | } else { 591 | fs.appendFileSync(reportPath, result.toString()) 592 | t.pass(result) 593 | } 594 | t.end() 595 | } 596 | ) 597 | }) 598 | 599 | const getPrefixMapQueries = () => { 600 | let rootKey 601 | return { 602 | prepareRootKey: (cb) => { 603 | query( 604 | fromDB(db), 605 | paginate(1), 606 | toCallback((err, { results }) => { 607 | if (err) cb(err) 608 | else { 609 | rootKey = results[0].key 610 | cb() 611 | } 612 | }) 613 | ) 614 | }, 615 | queryMap: (cb) => { 616 | let i = 0 617 | pull( 618 | query( 619 | fromDB(db), 620 | where( 621 | equal(seekVoteLink, rootKey, { 622 | indexType: 'value_content_vote_link', 623 | useMap: true, 624 | prefix: 32, 625 | prefixOffset: 1, 626 | }) 627 | ), 628 | paginate(5), 629 | toPullStream() 630 | ), 631 | pull.drain( 632 | (msgs) => { 633 | i++ 634 | }, 635 | (err) => { 636 | if (err) cb(err) 637 | else if (i !== 92) cb(new Error('wrong number of pages read: ' + i)) 638 | else cb() 639 | } 640 | ) 641 | ) 642 | }, 643 | } 644 | } 645 | 646 | test('query a prefix map (first run)', (t) => { 647 | const { prepareRootKey, queryMap } = getPrefixMapQueries() 648 | runBenchmark( 649 | 'Query a prefix map (1st run)', 650 | queryMap, 651 | (cb) => { 652 | closeLog((err) => { 653 | if (err) cb(err) 654 | else getJitdbReady((err) => { 655 | if (err) cb(err) 656 | else runThreeIndexQuery((err) => { 657 | if (err) cb(err) 658 | else { 659 | done = multicb({ pluck: 1 }) 660 | useContactIndex(function(err) { 661 | if (err) cb(err) 662 | else prepareRootKey(cb) 663 | }) 664 | } 665 | }) 666 | }) 667 | }) 668 | }, 669 | (err, result) => { 670 | if (err) { 671 | t.fail(err) 672 | } else { 673 | fs.appendFileSync(reportPath, result.toString()) 674 | t.pass(result) 675 | } 676 | t.end() 677 | } 678 | ) 679 | }) 680 | 681 | test('query a prefix map (second run)', (t) => { 682 | const { prepareRootKey, queryMap } = getPrefixMapQueries() 683 | runBenchmark( 684 | 'Query a prefix map (2nd run)', 685 | queryMap, 686 | (cb) => { 687 | closeLog((err) => { 688 | if (err) cb(err) 689 | else getJitdbReady((err) => { 690 | if (err) cb(err) 691 | else runThreeIndexQuery((err) => { 692 | if (err) cb(err) 693 | else { 694 | done = multicb({ pluck: 1 }) 695 | useContactIndex(function(err) { 696 | if (err) cb(err) 697 | else prepareRootKey((err3) => { 698 | if (err3) cb(err3) 699 | else queryMap(cb) 700 | }) 701 | }) 702 | } 703 | }) 704 | }) 705 | }) 706 | }, 707 | (err, result) => { 708 | if (err) { 709 | t.fail(err) 710 | } else { 711 | fs.appendFileSync(reportPath, result.toString()) 712 | t.pass(result) 713 | } 714 | t.end() 715 | } 716 | ) 717 | }) 718 | 719 | test('paginate ten results', (t) => { 720 | runBenchmark( 721 | `Paginate 10 results`, 722 | (cb) => { 723 | pull( 724 | query( 725 | fromDB(db), 726 | where( 727 | and( 728 | equal(seekType, 'contact', { indexType: 'type' }), 729 | equal(seekAuthor, alice.id, { 730 | indexType: 'author', 731 | prefix: 32, 732 | prefixOffset: 1, 733 | }) 734 | ) 735 | ), 736 | startFrom(0), 737 | paginate(10), 738 | toPullStream() 739 | ), 740 | pull.take(1), 741 | pull.collect((err, msgs) => { 742 | if (err) cb(err) 743 | else if (msgs[0].length !== 10) 744 | cb(new Error('msgs.length is wrong: ' + msgs.length)) 745 | else cb() 746 | }) 747 | ) 748 | }, 749 | (cb) => { 750 | closeLog((err) => { 751 | if (err) cb(err) 752 | else getJitdbReady((err) => { 753 | if (err) cb(err) 754 | else runThreeIndexQuery((err) => { 755 | if (err) cb(err) 756 | else { 757 | done = multicb({ pluck: 1 }) 758 | useContactIndex(cb) 759 | } 760 | }) 761 | }) 762 | }) 763 | }, 764 | (err, result) => { 765 | if (err) { 766 | t.fail(err) 767 | } else { 768 | fs.appendFileSync(reportPath, result.toString()) 769 | t.pass(result) 770 | } 771 | t.end() 772 | } 773 | ) 774 | }) 775 | -------------------------------------------------------------------------------- /copy-json-to-bipf-async.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | var pull = require('pull-stream') 6 | var FlumeLog = require('flumelog-offset') 7 | var AsyncLog = require('async-append-only-log') 8 | var binary = require('bipf') 9 | var json = require('flumecodec/json') 10 | 11 | // copy an old (flumelog-offset) log (json) to a new async log (bipf) 12 | function copy(oldpath, newpath, cb) { 13 | var block = 64 * 1024 14 | var log = FlumeLog(oldpath, { blockSize: block, codec: json }) 15 | var log2 = AsyncLog(newpath, { blockSize: block }) 16 | 17 | var dataTransferred = 0 18 | 19 | pull( 20 | log.stream({ seqs: false, codec: json }), 21 | pull.map(function (data) { 22 | var len = binary.encodingLength(data) 23 | var b = Buffer.alloc(len) 24 | binary.encode(data, b, 0) 25 | return b 26 | }), 27 | function (read) { 28 | read(null, function next(err, data) { 29 | if (err && err !== true) return cb(err) 30 | if (err) return cb() 31 | dataTransferred += data.length 32 | log2.append(data, function () {}) 33 | if (dataTransferred % block == 0) 34 | log2.onDrain(function () { 35 | read(null, next) 36 | }) 37 | else read(null, next) 38 | }) 39 | } 40 | ) 41 | } 42 | 43 | if (require.main === module) { 44 | if (process.argv[2] === process.argv[3]) 45 | throw new Error('input must !== output') 46 | else 47 | copy(process.argv[2], process.argv[3], (err) => { 48 | if (err) throw err 49 | else console.log('done') 50 | }) 51 | } else { 52 | module.exports = copy 53 | } 54 | -------------------------------------------------------------------------------- /copy-json-to-bipf-offset.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | var pull = require('pull-stream') 6 | var FlumeLog = require('flumelog-offset') 7 | var binary = require('bipf') 8 | var json = require('flumecodec/json') 9 | 10 | var block = 64 * 1024 11 | 12 | //copy an old (flumelog-offset) log (json) to a new async log (bipf) 13 | 14 | if (process.argv[2] === process.argv[3]) 15 | throw new Error('input must !== output') 16 | 17 | var log = FlumeLog(process.argv[2], { blockSize: block, codec: json }) 18 | var log2 = FlumeLog(process.argv[3], { blockSize: block }) 19 | 20 | pull( 21 | log.stream({ seqs: false, codec: json }), 22 | pull.map(function (data) { 23 | var len = binary.encodingLength(data) 24 | var b = Buffer.alloc(len) 25 | binary.encode(data, b, 0) 26 | return b 27 | }), 28 | pull.drain( 29 | (data) => { 30 | log2.append(data, function () {}) 31 | }, 32 | () => { 33 | console.log('done') 34 | } 35 | ) 36 | ) 37 | -------------------------------------------------------------------------------- /copy-json-to-bipf.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | var pull = require('pull-stream') 6 | var FlumeLog = require('flumelog-offset') 7 | var FlumeLogAligned = require('flumelog-aligned-offset') 8 | var binary = require('bipf') 9 | var json = require('flumecodec/json') 10 | 11 | var block = 64 * 1024 12 | 13 | //copy an old (flumelog-offset) log (json) to a new raf log (bipf) 14 | 15 | if (process.argv[2] === process.argv[3]) 16 | throw new Error('input must !== output') 17 | 18 | var log = FlumeLog(process.argv[2], { blockSize: block, codec: json }) 19 | var log2 = FlumeLogAligned(process.argv[3], { block: block }) 20 | 21 | pull( 22 | log.stream({ seqs: false, codec: json }), 23 | pull.map(function (data) { 24 | var len = binary.encodingLength(data) 25 | var b = Buffer.alloc(len) 26 | binary.encode(data, b, 0) 27 | return b 28 | }), 29 | function (read) { 30 | read(null, function next(err, data) { 31 | if (err && err !== true) throw err 32 | if (err) return console.error('done') 33 | log2.append(data, function () {}) 34 | if (log2.appendState.offset > log2.appendState.written + block * 10) 35 | log2.onDrain(function () { 36 | read(null, next) 37 | }) 38 | else read(null, next) 39 | }) 40 | } 41 | ) 42 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const Log = require('async-append-only-log') 6 | const pull = require('pull-stream') 7 | const JITDB = require('./index') 8 | const { 9 | query, 10 | fromDB, 11 | and, 12 | or, 13 | slowEqual, 14 | equal, 15 | debug, 16 | author, 17 | paginate, 18 | toCallback, 19 | toPromise, 20 | toPullStream, 21 | toAsyncIter, 22 | descending, 23 | } = require('./operators') 24 | const { seekType, seekAuthor, seekVoteLink } = require('./test/helpers') 25 | 26 | var raf = Log(process.argv[2], { blockSize: 64 * 1024 }) 27 | 28 | var db = JITDB(raf, './indexes') 29 | db.onReady(async () => { 30 | // seems the cache needs to be warmed up to get fast results 31 | 32 | const staltzp = '@+UMKhpbzXAII+2/7ZlsgkJwIsxdfeFi36Z5Rk1gCfY0=.ed25519' 33 | const mix = '@ye+QM09iPcDJD6YvQYjoQc7sLF/IFhmNbEqgdzQo3lQ=.ed25519' 34 | const mixy = '@G98XybiXD/amO9S/UyBKnWTWZnSKYS3YVB/5osSRHvY=.ed25519' 35 | const arj = '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519' 36 | const myroot = '%0cwmRpJFo5qtsesZYrf2TkufWIaxTzLiNhKUZdWNeJM=.sha256' 37 | 38 | if (false) 39 | query( 40 | fromDB(db), 41 | // debug(), 42 | and(slowEqual('value.content.type', 'vote')), 43 | // debug(), 44 | and(slowEqual('value.content.vote.expression', '⛵')), 45 | // debug(), 46 | and(slowEqual('value.author', staltzp)), 47 | // debug(), 48 | toCallback((err, results) => { 49 | console.log(results) 50 | }) 51 | ) 52 | 53 | if (false) { 54 | const before = Date.now() 55 | const results = await query( 56 | fromDB(db), 57 | // debug(), 58 | and(equal(seekType, 'blog', { indexType: 'type' })), 59 | // debug(), 60 | and( 61 | or( 62 | equal(seekAuthor, mix, { indexType: 'author' }), 63 | equal(seekAuthor, mixy, { indexType: 'author' }) 64 | ) 65 | ), 66 | // debug(), 67 | toPromise() 68 | ) 69 | const duration = Date.now() - before 70 | console.log(`duration = ${duration}ms`) 71 | console.log(results.length) 72 | } 73 | 74 | if (true) { 75 | const before = Date.now() 76 | const results = await query( 77 | fromDB(db), 78 | or( 79 | // slowEqual('value.content.vote.link', myroot, { prefix: 32 }) 80 | equal(seekVoteLink, myroot, { prefix: 32, indexType: 'vote_link' }) 81 | ), 82 | toPromise() 83 | ) 84 | const duration = Date.now() - before 85 | console.log(`duration = ${duration}ms`) 86 | console.log(results.length) 87 | } 88 | 89 | var i = 0 90 | if (false) { 91 | const results = query( 92 | fromDB(db), 93 | // debug(), 94 | and(type('blog')), 95 | // debug(), 96 | and(or(author(mix), author(mixy), author(arj))), 97 | // debug(), 98 | startFrom(6), 99 | // debug(), 100 | paginate(3), 101 | // debug(), 102 | toAsyncIter() 103 | ) 104 | for await (let msgs of results) { 105 | console.log('page #' + i++) 106 | console.log(msgs) 107 | } 108 | } 109 | 110 | if (false) { 111 | console.time('get all posts from users') 112 | 113 | const posts = query(fromDB(db), and(type('post'))) 114 | 115 | const postsMix = query( 116 | posts, 117 | // debug(), 118 | and(or(author(mix), author(mixy))), 119 | // debug(), 120 | toPromise() 121 | ) 122 | 123 | const postsArj = query( 124 | posts, 125 | // debug(), 126 | and(author(arj)), 127 | // debug(), 128 | toPromise() 129 | ) 130 | 131 | const [resMix, resArj] = await Promise.all([postsMix, postsArj]) 132 | console.log('mix posts: ' + resMix.length) 133 | console.log('arj posts: ' + resArj.length) 134 | console.timeEnd('get all posts from users') 135 | } 136 | 137 | return 138 | 139 | db.query( 140 | { 141 | type: 'AND', 142 | data: [ 143 | { 144 | type: 'EQUAL', 145 | data: { seek: db.seekType, value: 'post', indexType: 'type' }, 146 | }, 147 | { 148 | type: 'EQUAL', 149 | data: { seek: db.seekAuthor, value: staltzp, indexType: 'author' }, 150 | }, 151 | ], 152 | }, 153 | (err, results) => { 154 | console.timeEnd('get all posts from user') 155 | 156 | console.time('get last 10 posts from user') 157 | 158 | db.query( 159 | { 160 | type: 'AND', 161 | data: [ 162 | { 163 | type: 'EQUAL', 164 | data: { seek: db.seekType, value: 'post', indexType: 'type' }, 165 | }, 166 | { 167 | type: 'EQUAL', 168 | data: { 169 | seek: db.seekAuthor, 170 | value: staltzp, 171 | indexType: 'author', 172 | }, 173 | }, 174 | ], 175 | }, 176 | 0, 177 | 10, 178 | (err, results) => { 179 | console.timeEnd('get last 10 posts from user') 180 | 181 | console.time('get top 50 posts') 182 | 183 | db.query( 184 | { 185 | type: 'EQUAL', 186 | data: { 187 | seek: db.seekType, 188 | value: 'post', 189 | indexType: 'type', 190 | }, 191 | }, 192 | 0, 193 | 50, 194 | (err, results) => { 195 | console.timeEnd('get top 50 posts') 196 | 197 | console.time('author + sequence') 198 | 199 | db.query( 200 | { 201 | type: 'AND', 202 | data: [ 203 | { 204 | type: 'GT', 205 | data: { indexName: 'sequence', value: 7000 }, 206 | }, 207 | { 208 | type: 'EQUAL', 209 | data: { 210 | seek: db.seekAuthor, 211 | value: staltzp, 212 | indexType: 'author', 213 | }, 214 | }, 215 | ], 216 | }, 217 | (err, results) => { 218 | console.timeEnd('author + sequence') 219 | 220 | var hops = {} 221 | const query = { 222 | type: 'AND', 223 | data: [ 224 | { 225 | type: 'EQUAL', 226 | data: { 227 | seek: db.seekAuthor, 228 | value: staltzp, 229 | indexType: 'author', 230 | }, 231 | }, 232 | { 233 | type: 'EQUAL', 234 | data: { 235 | seek: db.seekType, 236 | value: 'contact', 237 | indexType: 'type', 238 | }, 239 | }, 240 | ], 241 | } 242 | const isFeed = require('ssb-ref').isFeed 243 | 244 | console.time('contacts for author') 245 | 246 | db.query(query, (err, results) => { 247 | results.forEach((data) => { 248 | var from = data.value.author 249 | var to = data.value.content.contact 250 | var value = 251 | data.value.content.blocking || 252 | data.value.content.flagged 253 | ? -1 254 | : data.value.content.following === true 255 | ? 1 256 | : -2 257 | 258 | if (isFeed(from) && isFeed(to)) { 259 | hops[from] = hops[from] || {} 260 | hops[from][to] = value 261 | } 262 | }) 263 | 264 | console.timeEnd('contacts for author') 265 | //console.log(hops) 266 | }) 267 | } 268 | ) 269 | } 270 | ) 271 | } 272 | ) 273 | } 274 | ) 275 | 276 | return 277 | 278 | console.time('get all') 279 | db.query( 280 | { 281 | type: 'EQUAL', 282 | data: { seek: db.seekAuthor, value: staltzp, indexType: 'author' }, 283 | }, 284 | (err, results) => { 285 | console.timeEnd('get all') 286 | } 287 | ) 288 | }) 289 | -------------------------------------------------------------------------------- /files.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: LGPL-3.0-only 4 | 5 | const jsesc = require('jsesc') 6 | const sanitize = require('sanitize-filename') 7 | const TypedFastBitSet = require('typedfastbitset') 8 | const { readFile, writeFile } = require('atomic-file-rw') 9 | const toBuffer = require('typedarray-to-buffer') 10 | const crcCalculate = require('crc/lib/crc32') 11 | 12 | const FIELD_SIZE = 4 // bytes 13 | 14 | /* 15 | * ## File format for tarr files 16 | * 17 | * Each header field is 4 bytes in size. 18 | * 19 | * | offset (bytes) | name | type | 20 | * | 0 | version | UInt32LE | 21 | * |----------------|---------|----------| 22 | * | 4 | offset | UInt32LE | 23 | * | 8 | count | UInt32LE | 24 | * | 12 | crc | UInt32LE | 25 | * | 16 | body | Buffer | 26 | */ 27 | 28 | function calculateCRCAndWriteFile(buf, filename, cb) { 29 | try { 30 | const crc = crcCalculate(buf) 31 | buf.writeUInt32LE(crc, 3 * FIELD_SIZE) 32 | } catch (err) { 33 | return cb(err) 34 | } 35 | writeFile(filename, buf, cb) 36 | } 37 | 38 | function readFileAndCheckCRC(filename, cb) { 39 | readFile(filename, (err, buf) => { 40 | if (err) return cb(err) 41 | if (buf.length < 16) return cb(Error('file too short')) 42 | 43 | let crcFile 44 | let crc 45 | try { 46 | crcFile = buf.readUInt32LE(3 * FIELD_SIZE) 47 | buf.writeUInt32LE(0, 3 * FIELD_SIZE) 48 | crc = crcCalculate(buf) 49 | } catch (err) { 50 | return cb(err) 51 | } 52 | 53 | if (crcFile !== 0 && crc !== crcFile) return cb(Error('crc check failed')) 54 | cb(null, buf) 55 | }) 56 | } 57 | 58 | function saveTypedArrayFile(filename, version, offset, count, tarr, cb) { 59 | if (!cb) 60 | cb = (err) => { 61 | if (err) console.error(err) 62 | } 63 | 64 | if (typeof version !== 'number') { 65 | return cb(Error('cannot save file ' + filename + ' without version')) 66 | } 67 | 68 | let buf 69 | try { 70 | const dataBuffer = toBuffer(tarr) 71 | // we try to save an extra 10% so we don't have to immediately grow 72 | // after loading and adding again 73 | const saveSize = Math.min(count * 1.1, tarr.length) 74 | buf = Buffer.alloc(4 * FIELD_SIZE + saveSize * tarr.BYTES_PER_ELEMENT) 75 | buf.writeUInt32LE(version, 0) 76 | buf.writeUInt32LE(offset, FIELD_SIZE) 77 | buf.writeUInt32LE(count, 2 * FIELD_SIZE) 78 | dataBuffer.copy(buf, 4 * FIELD_SIZE) 79 | } catch (err) { 80 | return cb(err) 81 | } 82 | 83 | calculateCRCAndWriteFile(buf, filename, cb) 84 | } 85 | 86 | function loadTypedArrayFile(filename, Type, cb) { 87 | readFileAndCheckCRC(filename, (err, buf) => { 88 | if (err) return cb(err) 89 | 90 | let version, offset, count, tarr 91 | try { 92 | version = buf.readUInt32LE(0) 93 | offset = buf.readUInt32LE(FIELD_SIZE) 94 | count = buf.readUInt32LE(2 * FIELD_SIZE) 95 | const body = buf.slice(4 * FIELD_SIZE) 96 | tarr = new Type( 97 | body.buffer, 98 | body.offset, 99 | body.byteLength / (Type === Float64Array ? 8 : 4) 100 | ) 101 | } catch (err) { 102 | return cb(err) 103 | } 104 | 105 | cb(null, { version, offset, count, tarr }) 106 | }) 107 | } 108 | 109 | function savePrefixMapFile(filename, version, offset, count, map, cb) { 110 | if (!cb) 111 | cb = (err) => { 112 | if (err) console.error(err) 113 | } 114 | 115 | if (typeof version !== 'number') { 116 | return cb(Error('cannot save file ' + filename + ' without version')) 117 | } 118 | 119 | let buf 120 | try { 121 | const jsonMap = JSON.stringify(map) 122 | buf = Buffer.alloc(4 * FIELD_SIZE + jsonMap.length) 123 | buf.writeUInt32LE(version, 0) 124 | buf.writeUInt32LE(offset, FIELD_SIZE) 125 | buf.writeUInt32LE(count, 2 * FIELD_SIZE) 126 | Buffer.from(jsonMap).copy(buf, 4 * FIELD_SIZE) 127 | } catch (err) { 128 | return cb(err) 129 | } 130 | 131 | calculateCRCAndWriteFile(buf, filename, cb) 132 | } 133 | 134 | function loadPrefixMapFile(filename, cb) { 135 | readFileAndCheckCRC(filename, (err, buf) => { 136 | if (err) return cb(err) 137 | 138 | let version, offset, count, map 139 | try { 140 | version = buf.readUInt32LE(0) 141 | offset = buf.readUInt32LE(FIELD_SIZE) 142 | count = buf.readUInt32LE(2 * FIELD_SIZE) 143 | const body = buf.slice(4 * FIELD_SIZE) 144 | map = JSON.parse(body) 145 | } catch (err) { 146 | return cb(err) 147 | } 148 | 149 | cb(null, { version, offset, count, map }) 150 | }) 151 | } 152 | 153 | function saveBitsetFile(filename, version, offset, bitset, cb) { 154 | let count 155 | try { 156 | bitset.trim() 157 | count = bitset.words.length 158 | } catch (err) { 159 | return cb(err) 160 | } 161 | saveTypedArrayFile(filename, version, offset, count, bitset.words, cb) 162 | } 163 | 164 | function loadBitsetFile(filename, cb) { 165 | loadTypedArrayFile(filename, Uint32Array, (err, data) => { 166 | if (err) return cb(err) 167 | 168 | const { version, offset, count, tarr } = data 169 | const bitset = new TypedFastBitSet() 170 | bitset.words = tarr 171 | cb(null, { version, offset, bitset }) 172 | }) 173 | } 174 | 175 | function listFiles(dir, cb) { 176 | if (typeof window !== 'undefined') { 177 | // browser 178 | const IdbKvStore = require('idb-kv-store') 179 | const store = new IdbKvStore(dir, { disableBroadcast: true }) 180 | store.keys(cb) 181 | } else { 182 | // node.js 183 | const fs = require('fs') 184 | const mkdirp = require('mkdirp') 185 | mkdirp(dir).then(() => { 186 | fs.readdir(dir, cb) 187 | }, cb) 188 | } 189 | } 190 | 191 | function safeFilename(filename) { 192 | // in general we want to escape wierd characters 193 | let result = jsesc(filename) 194 | // sanitize will remove special characters, which means that two 195 | // indexes might end up with the same name so lets replace those 196 | // with jsesc escapeEverything values 197 | result = result.replace(/\./g, 'x2E') 198 | result = result.replace(/\//g, 'x2F') 199 | result = result.replace(/\?/g, 'x3F') 200 | result = result.replace(/\/g, 'x3E') 202 | result = result.replace(/\:/g, 'x3A') 203 | result = result.replace(/\*/g, 'x2A') 204 | result = result.replace(/\|/g, 'x7C') 205 | // finally sanitize 206 | return sanitize(result) 207 | } 208 | 209 | const EmptyFile = { 210 | create(filename) { 211 | if (typeof window !== 'undefined') { 212 | // browser 213 | const IdbKvStore = require('idb-kv-store') 214 | const store = new IdbKvStore(filename, { disableBroadcast: true }) 215 | store.set('x', 'y', () => {}) 216 | } else { 217 | // node.js 218 | const fs = require('fs') 219 | fs.closeSync(fs.openSync(filename, 'w')) 220 | } 221 | }, 222 | 223 | exists(filename, cb) { 224 | if (typeof window !== 'undefined') { 225 | // browser 226 | const IdbKvStore = require('idb-kv-store') 227 | const store = new IdbKvStore(filename, { disableBroadcast: true }) 228 | store.get('x', (err, y) => { 229 | if (err) return cb(null, false) 230 | cb(null, y === 'y') 231 | }) 232 | } else { 233 | // node.js 234 | const fs = require('fs') 235 | cb(null, fs.existsSync(filename)) 236 | } 237 | }, 238 | 239 | delete(filename, cb) { 240 | EmptyFile.exists(filename, (err, exists) => { 241 | if (err) return cb(err) 242 | if (!exists) cb(null) 243 | else EmptyFile._actuallyDelete(filename, cb) 244 | }) 245 | }, 246 | 247 | _actuallyDelete(filename, cb) { 248 | if (typeof window !== 'undefined') { 249 | // browser 250 | const IdbKvStore = require('idb-kv-store') 251 | const store = new IdbKvStore(filename, { disableBroadcast: true }) 252 | store.remove('x', cb) 253 | } else { 254 | // node.js 255 | const rimraf = require('rimraf') 256 | cb(null, rimraf.sync(filename)) 257 | } 258 | }, 259 | } 260 | 261 | module.exports = { 262 | saveTypedArrayFile, 263 | loadTypedArrayFile, 264 | savePrefixMapFile, 265 | loadPrefixMapFile, 266 | saveBitsetFile, 267 | loadBitsetFile, 268 | listFiles, 269 | EmptyFile, 270 | safeFilename, 271 | } 272 | -------------------------------------------------------------------------------- /flumelog.diff: -------------------------------------------------------------------------------- 1 | diff --git a/index.js b/index.js 2 | index 84474bd..98b7036 100644 3 | --- a/index.js 4 | +++ b/index.js 5 | @@ -7,6 +7,8 @@ const debounce = require('lodash.debounce') 6 | const AtomicFile = require('atomic-file/buffer') 7 | const toBuffer = require('typedarray-to-buffer') 8 | 9 | +const pull = require('pull-stream') 10 | + 11 | module.exports = function (db, indexesPath) { 12 | function saveTypedArray(name, seq, count, arr, cb) { 13 | const filename = path.join(indexesPath, name + ".index") 14 | @@ -288,9 +290,9 @@ module.exports = function (db, indexesPath) { 15 | var updatedTimestampIndex = false 16 | const start = Date.now() 17 | 18 | - db.stream({ gt: index.seq }).pipe({ 19 | - paused: false, 20 | - write: function (data) { 21 | + pull( 22 | + db.stream({ gt: index.seq }), 23 | + pull.drain((data) => { 24 | if (updateOffsetIndex(offset, data.seq)) 25 | updatedOffsetIndex = true 26 | 27 | @@ -300,8 +302,7 @@ module.exports = function (db, indexesPath) { 28 | updateIndexValue(op.data, index, data.value, offset) 29 | 30 | offset++ 31 | - }, 32 | - end: () => { 33 | + }, () => { 34 | var count = offset // incremented at end 35 | console.log(`time: ${Date.now()-start}ms, total items: ${count}`) 36 | 37 | @@ -315,8 +316,8 @@ module.exports = function (db, indexesPath) { 38 | saveIndex(op.data.indexName, index.seq, index.data) 39 | 40 | cb() 41 | - } 42 | - }) 43 | + }) 44 | + ) 45 | } 46 | 47 | function createIndexes(missingIndexes, cb) { 48 | @@ -333,10 +334,10 @@ module.exports = function (db, indexesPath) { 49 | var updatedOffsetIndex = false 50 | var updatedTimestampIndex = false 51 | const start = Date.now() 52 | - 53 | - db.stream({}).pipe({ 54 | - paused: false, 55 | - write: function (data) { 56 | + 57 | + pull( 58 | + db.stream({}), 59 | + pull.drain((data) => { 60 | var seq = data.seq 61 | var buffer = data.value 62 | 63 | @@ -354,8 +355,8 @@ module.exports = function (db, indexesPath) { 64 | }) 65 | 66 | offset++ 67 | - }, 68 | - end: () => { 69 | + }, () => { 70 | + console.log("done", offset) 71 | var count = offset // incremented at end 72 | console.log(`time: ${Date.now()-start}ms, total items: ${count}`) 73 | 74 | @@ -372,8 +373,8 @@ module.exports = function (db, indexesPath) { 75 | } 76 | 77 | cb() 78 | - } 79 | - }) 80 | + }) 81 | + ) 82 | } 83 | 84 | function setupIndex(op) { 85 | -------------------------------------------------------------------------------- /flumelog.diff.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | 3 | SPDX-License-Identifier: Unlicense -------------------------------------------------------------------------------- /operators.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: LGPL-3.0-only 4 | 5 | const bipf = require('bipf') 6 | const traverse = require('traverse') 7 | const promisify = require('promisify-4loc') 8 | const pull = require('pull-stream') 9 | const multicb = require('multicb') 10 | const pullAwaitable = require('pull-awaitable') 11 | const cat = require('pull-cat') 12 | const { safeFilename } = require('./files') 13 | 14 | //#region Helper functions and util operators 15 | 16 | function copyMeta(orig, dest) { 17 | if (orig.meta) { 18 | dest.meta = orig.meta 19 | } 20 | } 21 | 22 | function updateMeta(orig, key, value) { 23 | const res = Object.assign({}, orig) 24 | res.meta[key] = value 25 | return res 26 | } 27 | 28 | function extractMeta(orig) { 29 | const meta = orig.meta 30 | return meta 31 | } 32 | 33 | const seekFromDescCache = new Map() 34 | function seekFromDesc(desc) { 35 | if (seekFromDescCache.has(desc)) { 36 | return seekFromDescCache.get(desc) 37 | } 38 | const keys = desc.split('.').map((str) => bipf.allocAndEncode(str)) 39 | // The 2nd arg `start` is to support plucks too 40 | const fn = function (buffer, start = 0) { 41 | var p = start 42 | for (let key of keys) { 43 | p = bipf.seekKey2(buffer, p, key, 0) 44 | if (p < 0) return -1 45 | } 46 | return p 47 | } 48 | seekFromDescCache.set(desc, fn) 49 | return fn 50 | } 51 | 52 | function getIndexName(opts, indexType, valueName) { 53 | return safeFilename( 54 | opts.prefix 55 | ? opts.useMap 56 | ? indexType + '__map' 57 | : indexType 58 | : indexType + '_' + valueName 59 | ) 60 | } 61 | 62 | function query(...cbs) { 63 | let res = cbs[0] 64 | for (let i = 1, n = cbs.length; i < n; i++) if (cbs[i]) res = cbs[i](res) 65 | return res 66 | } 67 | 68 | function debug() { 69 | return (ops) => { 70 | const meta = JSON.stringify(ops.meta, (key, val) => 71 | key === 'jitdb' ? void 0 : val 72 | ) 73 | console.log( 74 | 'debug', 75 | JSON.stringify( 76 | ops, 77 | (key, val) => { 78 | if (key === 'meta') return void 0 79 | else if (key === 'task' && typeof val === 'function') 80 | return '[Function]' 81 | else if (key === 'value' && val.type === 'Buffer') 82 | return Buffer.from(val.data).toString() 83 | else return val 84 | }, 85 | 2 86 | ), 87 | meta === '{}' ? '' : 'meta: ' + meta 88 | ) 89 | return ops 90 | } 91 | } 92 | 93 | //#endregion 94 | //#region "Unit operators": they create objects that JITDB interprets 95 | 96 | function slowEqual(seekDesc, target, opts) { 97 | opts = opts || {} 98 | const seek = seekFromDesc(seekDesc) 99 | const value = bipf.allocAndEncode(target) 100 | const valueName = !target ? '' : `${target}` 101 | const indexType = seekDesc.replace(/\./g, '_') 102 | const indexName = getIndexName(opts, indexType, valueName) 103 | return { 104 | type: 'EQUAL', 105 | data: { 106 | seek, 107 | value, 108 | indexType, 109 | indexName, 110 | useMap: opts.useMap, 111 | indexAll: opts.indexAll, 112 | prefix: opts.prefix, 113 | prefixOffset: opts.prefixOffset, 114 | }, 115 | } 116 | } 117 | 118 | function equal(seek, target, opts) { 119 | opts = opts || {} 120 | if (!opts.indexType) 121 | throw new Error('equal() operator needs an indexType in the 3rd arg') 122 | const value = bipf.allocAndEncode(target) 123 | const valueName = !target ? '' : `${target}` 124 | const indexType = opts.indexType 125 | const indexName = getIndexName(opts, indexType, valueName) 126 | return { 127 | type: 'EQUAL', 128 | data: { 129 | seek, 130 | value, 131 | indexType, 132 | indexName, 133 | useMap: opts.useMap, 134 | indexAll: opts.indexAll, 135 | prefix: opts.prefix, 136 | prefixOffset: opts.prefixOffset, 137 | }, 138 | } 139 | } 140 | 141 | function slowPredicate(seekDesc, fn, opts) { 142 | opts = opts || {} 143 | const seek = seekFromDesc(seekDesc) 144 | if (typeof fn !== 'function') 145 | throw new Error('predicate() needs a predicate function in the 2rd arg') 146 | const value = fn 147 | const indexType = seekDesc.replace(/\./g, '_') 148 | const name = opts.name || fn.name 149 | if (!name) throw new Error('predicate() needs opts.name') 150 | const indexName = safeFilename(indexType + '__pred_' + name) 151 | return { 152 | type: 'PREDICATE', 153 | data: { 154 | seek, 155 | value, 156 | indexType, 157 | indexName, 158 | }, 159 | } 160 | } 161 | 162 | function predicate(seek, fn, opts) { 163 | opts = opts || {} 164 | if (!opts.indexType) 165 | throw new Error('predicate() operator needs an indexType in the 3rd arg') 166 | if (typeof fn !== 'function') 167 | throw new Error('predicate() needs a predicate function in the 2rd arg') 168 | const value = fn 169 | const indexType = opts.indexType 170 | const name = opts.name || fn.name 171 | if (!name) throw new Error('predicate() needs opts.name') 172 | const indexName = safeFilename(indexType + '__pred_' + name) 173 | return { 174 | type: 'PREDICATE', 175 | data: { 176 | seek, 177 | value, 178 | indexType, 179 | indexName, 180 | }, 181 | } 182 | } 183 | 184 | function slowAbsent(seekDesc) { 185 | const seek = seekFromDesc(seekDesc) 186 | const indexType = seekDesc.replace(/\./g, '_') 187 | const indexName = safeFilename(indexType + '__absent') 188 | return { 189 | type: 'ABSENT', 190 | data: { 191 | seek, 192 | indexType, 193 | indexName, 194 | }, 195 | } 196 | } 197 | 198 | function absent(seek, opts) { 199 | opts = opts || {} 200 | if (!opts.indexType) 201 | throw new Error('absent() operator needs an indexType in the 3rd arg') 202 | const indexType = opts.indexType 203 | const indexName = safeFilename(indexType + '__absent') 204 | return { 205 | type: 'ABSENT', 206 | data: { 207 | seek, 208 | indexType, 209 | indexName, 210 | }, 211 | } 212 | } 213 | 214 | function slowIncludes(seekDesc, target, opts) { 215 | opts = opts || {} 216 | const seek = seekFromDesc(seekDesc) 217 | const value = bipf.allocAndEncode(target) 218 | if (!value) throw new Error('slowIncludes() 2nd arg needs to be truthy') 219 | const valueName = !target ? '' : `${target}` 220 | const indexType = seekDesc.replace(/\./g, '_') 221 | const indexName = safeFilename(indexType + '_' + valueName) 222 | const pluck = 223 | opts.pluck && typeof opts.pluck === 'string' 224 | ? seekFromDesc(opts.pluck) 225 | : opts.pluck 226 | return { 227 | type: 'INCLUDES', 228 | data: { 229 | seek, 230 | value, 231 | indexType, 232 | indexName, 233 | indexAll: opts.indexAll, 234 | pluck, 235 | }, 236 | } 237 | } 238 | 239 | function includes(seek, target, opts) { 240 | opts = opts || {} 241 | if (!opts.indexType) 242 | throw new Error('includes() operator needs an indexType in the 3rd arg') 243 | const value = bipf.allocAndEncode(target) 244 | if (!value) throw new Error('includes() 2nd arg needs to be truthy') 245 | const valueName = !target ? '' : `${target}` 246 | const indexType = opts.indexType 247 | const indexName = safeFilename(indexType + '_' + valueName) 248 | return { 249 | type: 'INCLUDES', 250 | data: { 251 | seek, 252 | value, 253 | indexType, 254 | indexName, 255 | indexAll: opts.indexAll, 256 | pluck: opts.pluck, 257 | }, 258 | } 259 | } 260 | 261 | function gt(value, indexName) { 262 | if (typeof value !== 'number') throw new Error('gt() needs a number arg') 263 | return { 264 | type: 'GT', 265 | data: { 266 | value, 267 | indexName, 268 | }, 269 | } 270 | } 271 | 272 | function gte(value, indexName) { 273 | if (typeof value !== 'number') throw new Error('gte() needs a number arg') 274 | return { 275 | type: 'GTE', 276 | data: { 277 | value, 278 | indexName, 279 | }, 280 | } 281 | } 282 | 283 | function lt(value, indexName) { 284 | if (typeof value !== 'number') throw new Error('lt() needs a number arg') 285 | return { 286 | type: 'LT', 287 | data: { 288 | value, 289 | indexName, 290 | }, 291 | } 292 | } 293 | 294 | function lte(value, indexName) { 295 | if (typeof value !== 'number') throw new Error('lte() needs a number arg') 296 | return { 297 | type: 'LTE', 298 | data: { 299 | value, 300 | indexName, 301 | }, 302 | } 303 | } 304 | 305 | function seqs(values) { 306 | return { 307 | type: 'SEQS', 308 | seqs: values, 309 | } 310 | } 311 | 312 | function liveSeqs(pullStream) { 313 | return { 314 | type: 'LIVESEQS', 315 | stream: pullStream, 316 | } 317 | } 318 | 319 | function offsets(values) { 320 | return { 321 | type: 'OFFSETS', 322 | offsets: values.slice(0), 323 | } 324 | } 325 | 326 | function deferred(task) { 327 | return { 328 | type: 'DEFERRED', 329 | task, 330 | } 331 | } 332 | 333 | //#endregion 334 | //#region "Combinator operators": they build composite operations 335 | 336 | function not(ops) { 337 | return { 338 | type: 'NOT', 339 | data: [ops], 340 | } 341 | } 342 | 343 | function and(...args) { 344 | const validargs = args.filter((arg) => !!arg) 345 | if (validargs.length === 0) return {} 346 | if (validargs.length === 1) return validargs[0] 347 | return { type: 'AND', data: validargs } 348 | } 349 | 350 | function or(...args) { 351 | const validargs = args.filter((arg) => !!arg) 352 | if (validargs.length === 0) return {} 353 | if (validargs.length === 1) return validargs[0] 354 | return { type: 'OR', data: validargs } 355 | } 356 | 357 | function where(...args) { 358 | return (prevOp) => { 359 | if (args.length !== 1) throw new Error('where() accepts only one argument') 360 | const nextOp = args[0] 361 | if (!nextOp) return prevOp 362 | const res = prevOp.type ? { type: 'AND', data: [prevOp, nextOp] } : nextOp 363 | copyMeta(prevOp, res) 364 | return res 365 | } 366 | } 367 | 368 | //#endregion 369 | //#region "Special operators": they only update meta 370 | 371 | function fromDB(jitdb) { 372 | return { 373 | meta: { jitdb }, 374 | } 375 | } 376 | 377 | function live(opts) { 378 | if (opts && opts.old) return (ops) => updateMeta(ops, 'live', 'liveAndOld') 379 | else return (ops) => updateMeta(ops, 'live', 'liveOnly') 380 | } 381 | 382 | function count() { 383 | return (ops) => updateMeta(ops, 'count', true) 384 | } 385 | 386 | function descending() { 387 | return (ops) => updateMeta(ops, 'descending', true) 388 | } 389 | 390 | function sortByArrival() { 391 | return (ops) => updateMeta(ops, 'sortBy', 'arrival') 392 | } 393 | 394 | function startFrom(seq) { 395 | return (ops) => updateMeta(ops, 'seq', seq) 396 | } 397 | 398 | function paginate(pageSize) { 399 | return (ops) => updateMeta(ops, 'pageSize', pageSize) 400 | } 401 | 402 | function batch(batchSize) { 403 | return (ops) => updateMeta(ops, 'batchSize', batchSize) 404 | } 405 | 406 | function asOffsets() { 407 | return (ops) => updateMeta(ops, 'asOffsets', true) 408 | } 409 | 410 | //#endregion 411 | //#region "Consumer operators": they execute the query tree 412 | 413 | function executeDeferredOps(ops, meta) { 414 | // Collect all deferred tasks and their object-traversal paths 415 | const allDeferred = [] 416 | traverse.forEach(ops, function (val) { 417 | if (!val) return 418 | // this.block() means don't traverse inside these, they won't have DEFERRED 419 | if (this.key === 'meta' && val.jitdb) return this.block() 420 | if (val.type === 'DEFERRED' && val.task) { 421 | allDeferred.push([this.path, val.task]) 422 | } 423 | if (!(Array.isArray(val) || val.type === 'AND' || val.type === 'OR')) { 424 | this.block() 425 | } 426 | }) 427 | if (allDeferred.length === 0) return pull.values([ops]) 428 | 429 | // State needed throughout the execution of the `readable` 430 | const done = multicb({ pluck: 1 }) 431 | let completed = false 432 | const abortListeners = [] 433 | function addAbortListener(listener) { 434 | abortListeners.push(listener) 435 | } 436 | 437 | return function readable(end, cb) { 438 | if (end) { 439 | while (abortListeners.length) abortListeners.shift()() 440 | cb(end) 441 | return 442 | } 443 | if (completed) { 444 | cb(true) 445 | return 446 | } 447 | 448 | // Execute all deferred tasks and collect the results (and the paths) 449 | for (const [path, task] of allDeferred) { 450 | const taskCB = done() 451 | task( 452 | meta, 453 | (err, result) => { 454 | if (err) taskCB(err) 455 | else if (!result) taskCB(null, [path, {}]) 456 | else if (typeof result === 'function') taskCB(null, [path, result()]) 457 | else taskCB(null, [path, result]) 458 | }, 459 | addAbortListener 460 | ) 461 | } 462 | 463 | // When all tasks are done... 464 | done((err, results) => { 465 | if (err) return cb(err) 466 | 467 | // Replace/mutate all deferreds with their respective results 468 | for (const [path, result] of results) { 469 | result.meta = meta 470 | if (path.length === 0) ops = result 471 | else traverse.set(ops, path, result) 472 | } 473 | completed = true 474 | cb(null, ops) 475 | }) 476 | } 477 | } 478 | 479 | function toCallback(cb) { 480 | return (rawOps) => { 481 | const meta = extractMeta(rawOps) 482 | const readable = executeDeferredOps(rawOps, meta) 483 | readable(null, (end, ops) => { 484 | if (end) return cb(end) 485 | 486 | const seq = meta.seq || 0 487 | const { pageSize, descending, asOffsets, sortBy } = meta 488 | if (meta.count) meta.jitdb.count(ops, seq, descending, cb) 489 | else if (pageSize) 490 | meta.jitdb.paginate( 491 | ops, 492 | seq, 493 | pageSize, 494 | descending, 495 | asOffsets, 496 | sortBy, 497 | null, 498 | cb 499 | ) 500 | else meta.jitdb.all(ops, seq, descending, asOffsets, sortBy, cb) 501 | }) 502 | } 503 | } 504 | 505 | function toPromise() { 506 | return (rawOps) => { 507 | return promisify((cb) => toCallback(cb)(rawOps))() 508 | } 509 | } 510 | 511 | function toPullStream() { 512 | return (rawOps) => { 513 | const meta = extractMeta(rawOps) 514 | 515 | function paginateStream(ops) { 516 | let seq = meta.seq || 0 517 | let total = Infinity 518 | const limit = meta.pageSize || meta.batchSize || 20 519 | let shouldEnd = false 520 | let latestMsgKey = null 521 | function readable(end, cb) { 522 | if (end) return cb(end) 523 | if (seq >= total || shouldEnd) return cb(true) 524 | if (meta.count) { 525 | shouldEnd = true 526 | meta.jitdb.count(ops, seq, meta.descending, cb) 527 | } else { 528 | meta.jitdb.paginate( 529 | ops, 530 | seq, 531 | limit, 532 | meta.descending, 533 | meta.asOffsets, 534 | meta.sortBy, 535 | latestMsgKey, 536 | (err, answer) => { 537 | if (err) return cb(err) 538 | else if (answer.total === 0) cb(true) 539 | else { 540 | total = answer.total 541 | seq = answer.nextSeq 542 | if (answer.results.length > 0) { 543 | latestMsgKey = answer.results[answer.results.length - 1].key 544 | } 545 | cb(null, answer.results) 546 | } 547 | } 548 | ) 549 | } 550 | } 551 | 552 | if (meta.count) { 553 | return readable 554 | } else if (meta.pageSize) { 555 | return pull( 556 | readable, 557 | pull.filter((page) => page.length > 0) 558 | ) 559 | } else { 560 | // Flatten the "pages" (arrays) into individual messages 561 | return pull( 562 | readable, 563 | pull.filter((page) => page.length > 0), 564 | pull.map(pull.values), 565 | pull.flatten() 566 | ) 567 | } 568 | } 569 | 570 | return pull( 571 | executeDeferredOps(rawOps, meta), 572 | pull.map((ops) => { 573 | if (meta.live === 'liveOnly') return meta.jitdb.live(ops) 574 | else if (meta.live === 'liveAndOld') 575 | return cat([paginateStream(ops), meta.jitdb.live(ops)]) 576 | else return paginateStream(ops) 577 | }), 578 | pull.flatten() 579 | ) 580 | } 581 | } 582 | 583 | // `async function*` supported in Node 10+ and browsers (except IE11) 584 | function toAsyncIter() { 585 | return async function* (rawOps) { 586 | const ps = toPullStream()(rawOps) 587 | for await (let x of pullAwaitable(ps)) yield x 588 | } 589 | } 590 | 591 | //#endregion 592 | 593 | module.exports = { 594 | fromDB, 595 | query, 596 | 597 | live, 598 | slowEqual, 599 | equal, 600 | slowPredicate, 601 | predicate, 602 | slowAbsent, 603 | absent, 604 | slowIncludes, 605 | includes, 606 | where, 607 | not, 608 | gt, 609 | gte, 610 | lt, 611 | lte, 612 | and, 613 | or, 614 | deferred, 615 | liveSeqs, 616 | 617 | seqs, 618 | offsets, 619 | 620 | descending, 621 | sortByArrival, 622 | count, 623 | startFrom, 624 | paginate, 625 | batch, 626 | asOffsets, 627 | toCallback, 628 | toPullStream, 629 | toPromise, 630 | toAsyncIter, 631 | 632 | debug, 633 | } 634 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jitdb", 3 | "description": "A database on top of a flumelog with automatic index generation and maintenance", 4 | "version": "7.0.7", 5 | "homepage": "https://github.com/ssb-ngi-pointer/jitdb", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/ssb-ngi-pointer/jitdb.git" 9 | }, 10 | "files": [ 11 | "*.js", 12 | "package.json.license", 13 | "LICENSES/*", 14 | "!example.js" 15 | ], 16 | "dependencies": { 17 | "atomic-file-rw": "^0.3.0", 18 | "binary-search-bounds": "^2.0.4", 19 | "bipf": "^1.6.2", 20 | "crc": "3.6.0", 21 | "debug": "^4.2.0", 22 | "fastpriorityqueue": "^0.7.1", 23 | "idb-kv-store": "^4.5.0", 24 | "jsesc": "^3.0.2", 25 | "mkdirp": "^1.0.4", 26 | "multicb": "1.2.2", 27 | "mutexify": "^1.4.0", 28 | "obz": "^1.1.0", 29 | "promisify-4loc": "1.0.0", 30 | "pull-async": "~1.0.0", 31 | "pull-awaitable": "^1.0.0", 32 | "pull-cat": "~1.1.11", 33 | "pull-stream": "^3.6.14", 34 | "push-stream": "^11.2.0", 35 | "push-stream-to-pull-stream": "^1.0.3", 36 | "rimraf": "^3.0.2", 37 | "sanitize-filename": "^1.6.3", 38 | "traverse": "^0.6.6", 39 | "typedarray-to-buffer": "^4.0.0", 40 | "typedfastbitset": "~0.2.1" 41 | }, 42 | "peerDependencies": { 43 | "async-append-only-log": "^4.3.2" 44 | }, 45 | "devDependencies": { 46 | "async-append-only-log": "^4.3.2", 47 | "expose-gc": "^1.0.0", 48 | "flumecodec": "0.0.1", 49 | "flumelog-offset": "3.4.4", 50 | "hash-wasm": "^4.6.0", 51 | "husky": "^4.3.8", 52 | "nodemark": "^0.3.0", 53 | "nyc": "^15.1.0", 54 | "prettier": "^2.1.2", 55 | "pretty-bytes": "^5.6.0", 56 | "pretty-quick": "^3.1.0", 57 | "pull-pushable": "^2.2.0", 58 | "ssb-fixtures": "2.2.0", 59 | "ssb-keys": "^8.1.0", 60 | "ssb-ref": "^2.14.3", 61 | "ssb-validate": "^4.1.1", 62 | "tap-arc": "^0.3.4", 63 | "tap-bail": "^1.0.0", 64 | "tape": "^5.2.2" 65 | }, 66 | "scripts": { 67 | "test": "tape test/*.js | tap-arc --bail", 68 | "coverage": "nyc --reporter=lcov npm run test", 69 | "format-code": "prettier --write \"*.js\" \"test/*.js\"", 70 | "format-code-staged": "pretty-quick --staged --pattern \"*.js\" --pattern \"test/*.js\"", 71 | "benchmark": "node benchmark/index.js | tap-arc", 72 | "benchmark-no-create": "node benchmark/index.js noCreate | tap-arc", 73 | "get-benchmark-matrix": "GET_BENCHMARK_MATRIX=1 node benchmark/index.js", 74 | "benchmark-only-create": "FIXTURES_ONLY=1 node benchmark/index.js" 75 | }, 76 | "husky": { 77 | "hooks": { 78 | "pre-commit": "npm run format-code-staged" 79 | } 80 | }, 81 | "author": "Anders Rune Jensen ", 82 | "contributors": [ 83 | "Andre Staltz " 84 | ], 85 | "license": "LGPL-3.0" 86 | } 87 | -------------------------------------------------------------------------------- /package.json.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | 3 | SPDX-License-Identifier: Unlicense -------------------------------------------------------------------------------- /status.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: LGPL-3.0-only 4 | 5 | const Obv = require('obz') 6 | 7 | module.exports = function Status() { 8 | let indexesStatus = {} 9 | const obv = Obv() 10 | obv.set(indexesStatus) 11 | const EMIT_INTERVAL = 1000 // ms 12 | let i = 0 13 | let iTimer = 0 14 | let timer = null 15 | const activeIndexNames = new Set() 16 | 17 | function setTimer() { 18 | // Turn on 19 | timer = setInterval(() => { 20 | if (i === iTimer) { 21 | // Turn off because nothing has been updated recently 22 | clearInterval(timer) 23 | timer = null 24 | i = iTimer = 0 25 | } else { 26 | iTimer = i 27 | obv.set(indexesStatus) 28 | } 29 | }, EMIT_INTERVAL) 30 | if (timer.unref) timer.unref() 31 | } 32 | 33 | function update(indexes, names) { 34 | let changed = false 35 | for (const name of names) { 36 | const index = indexes.get(name) 37 | const previous = indexesStatus[name] || -Infinity 38 | if (index.offset > previous) { 39 | indexesStatus[name] = index.offset 40 | activeIndexNames.add(name) 41 | changed = true 42 | } 43 | } 44 | if (!changed) return 45 | 46 | ++i 47 | if (!timer) { 48 | iTimer = i 49 | obv.set(indexesStatus) 50 | setTimer() 51 | } 52 | } 53 | 54 | function done(names) { 55 | for (const name of names) activeIndexNames.delete(name) 56 | if (activeIndexNames.size === 0) { 57 | indexesStatus = {} 58 | 59 | ++i 60 | if (!timer) { 61 | iTimer = i 62 | obv.set(indexesStatus) 63 | setTimer() 64 | } 65 | } 66 | } 67 | 68 | return { 69 | obv, 70 | done, 71 | update, 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /test/add.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const test = require('tape') 6 | const validate = require('ssb-validate') 7 | const ssbKeys = require('ssb-keys') 8 | const path = require('path') 9 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 10 | const push = require('push-stream') 11 | const rimraf = require('rimraf') 12 | const mkdirp = require('mkdirp') 13 | const { safeFilename } = require('../files') 14 | 15 | const dir = '/tmp/jitdb-add' 16 | rimraf.sync(dir) 17 | mkdirp.sync(dir) 18 | 19 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 20 | var keys2 = ssbKeys.loadOrCreateSync(path.join(dir, 'secret2')) 21 | var keys3 = ssbKeys.loadOrCreateSync(path.join(dir, 'secret3')) 22 | 23 | prepareAndRunTest('Base', dir, (t, db, raf) => { 24 | const msg = { type: 'post', text: 'Testing!' } 25 | let state = validate.initial() 26 | state = validate.appendNew(state, null, keys, msg, Date.now()) 27 | state = validate.appendNew(state, null, keys2, msg, Date.now() + 1) 28 | 29 | const typeQuery = { 30 | type: 'EQUAL', 31 | data: { 32 | seek: helpers.seekType, 33 | value: helpers.toBipf('post'), 34 | indexType: 'type', 35 | indexName: 'type_post', 36 | }, 37 | } 38 | 39 | addMsg(state.queue[0].value, raf, (err, msg1) => { 40 | addMsg(state.queue[1].value, raf, (err, msg2) => { 41 | db.paginate( 42 | typeQuery, 43 | 0, 44 | 10, 45 | false, 46 | false, 47 | 'declared', 48 | null, 49 | (err, { results }) => { 50 | t.equal(results.length, 2) 51 | 52 | // rerun on created index 53 | db.paginate( 54 | typeQuery, 55 | 0, 56 | 10, 57 | true, 58 | false, 59 | 'declared', 60 | null, 61 | (err, { results }) => { 62 | t.equal(results.length, 2) 63 | t.equal(results[0].value.author, keys2.id) 64 | 65 | db.paginate( 66 | typeQuery, 67 | 0, 68 | 10, 69 | false, 70 | false, 71 | 'declared', 72 | null, 73 | (err, { results }) => { 74 | t.equal(results.length, 2) 75 | t.equal(results[0].value.author, keys.id) 76 | 77 | const authorQuery = { 78 | type: 'EQUAL', 79 | data: { 80 | seek: helpers.seekAuthor, 81 | value: helpers.toBipf(keys.id), 82 | indexType: 'author', 83 | indexName: 'author_' + keys.id, 84 | }, 85 | } 86 | db.paginate( 87 | authorQuery, 88 | 0, 89 | 10, 90 | false, 91 | false, 92 | 'declared', 93 | null, 94 | (err, { results }) => { 95 | t.equal(results.length, 1) 96 | t.equal(results[0].id, msg1.id) 97 | 98 | // rerun on created index 99 | db.paginate( 100 | authorQuery, 101 | 0, 102 | 10, 103 | false, 104 | false, 105 | 'declared', 106 | null, 107 | (err, { results }) => { 108 | t.equal(results.length, 1) 109 | t.equal(results[0].id, msg1.id) 110 | 111 | db.paginate( 112 | { 113 | type: 'AND', 114 | data: [authorQuery, typeQuery], 115 | }, 116 | 0, 117 | 10, 118 | false, 119 | false, 120 | 'declared', 121 | null, 122 | (err, { results }) => { 123 | t.equal(results.length, 1) 124 | t.equal(results[0].id, msg1.id) 125 | 126 | const authorQuery2 = { 127 | type: 'EQUAL', 128 | data: { 129 | seek: helpers.seekAuthor, 130 | value: helpers.toBipf(keys2.id), 131 | indexType: 'author', 132 | indexName: 'author_' + keys2.id, 133 | }, 134 | } 135 | 136 | db.paginate( 137 | { 138 | type: 'AND', 139 | data: [ 140 | typeQuery, 141 | { 142 | type: 'OR', 143 | data: [authorQuery, authorQuery2], 144 | }, 145 | ], 146 | }, 147 | 0, 148 | 10, 149 | false, 150 | false, 151 | 'declared', 152 | null, 153 | (err, { results }) => { 154 | t.equal(results.length, 2) 155 | t.end() 156 | } 157 | ) 158 | } 159 | ) 160 | } 161 | ) 162 | } 163 | ) 164 | } 165 | ) 166 | } 167 | ) 168 | } 169 | ) 170 | }) 171 | }) 172 | }) 173 | 174 | prepareAndRunTest('Update index', dir, (t, db, raf) => { 175 | t.plan(5) 176 | t.timeoutAfter(5000) 177 | const msg = { type: 'post', text: 'Testing!' } 178 | let state = validate.initial() 179 | state = validate.appendNew(state, null, keys, msg, Date.now()) 180 | state = validate.appendNew(state, null, keys2, msg, Date.now() + 1) 181 | 182 | const typeQuery = { 183 | type: 'EQUAL', 184 | data: { 185 | seek: helpers.seekType, 186 | value: helpers.toBipf('post'), 187 | indexType: 'type', 188 | indexName: 'type_post', 189 | }, 190 | } 191 | 192 | const expectedIndexingActive = [0, 1, 0] 193 | 194 | addMsg(state.queue[0].value, raf, (err, msg1) => { 195 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 196 | t.equal(results.length, 1, '1 message') 197 | 198 | db.indexingActive((x) => { 199 | t.equals(x, expectedIndexingActive.shift(), 'indexingActive matches') 200 | }) 201 | 202 | addMsg(state.queue[1].value, raf, (err, msg1) => { 203 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 204 | t.equal(results.length, 2, '2 messages') 205 | }) 206 | }) 207 | }) 208 | }) 209 | }) 210 | 211 | prepareAndRunTest('grow', dir, (t, db, raf) => { 212 | let msg = { type: 'post', text: 'Testing' } 213 | 214 | let state = validate.initial() 215 | for (var i = 0; i < 32 * 1000; ++i) { 216 | msg.text = 'Testing ' + i 217 | state = validate.appendNew(state, null, keys, msg, Date.now() + i) 218 | } 219 | 220 | const typeQuery = { 221 | type: 'EQUAL', 222 | data: { 223 | seek: helpers.seekType, 224 | value: helpers.toBipf('post'), 225 | indexType: 'type', 226 | indexName: 'type_post', 227 | }, 228 | } 229 | 230 | push( 231 | push.values(state.queue), 232 | push.asyncMap((q, cb) => { 233 | addMsg(q.value, raf, cb) 234 | }), 235 | push.collect((err, results) => { 236 | db.paginate( 237 | typeQuery, 238 | 0, 239 | 1, 240 | false, 241 | false, 242 | 'declared', 243 | null, 244 | (err, { results }) => { 245 | t.equal(results.length, 1) 246 | t.equal(results[0].value.content.text, 'Testing 31999') 247 | t.end() 248 | } 249 | ) 250 | }) 251 | ) 252 | }) 253 | 254 | prepareAndRunTest('indexAll', dir, (t, db, raf) => { 255 | const msg = { type: 'post', text: 'Testing 1' } 256 | const msgContact = { type: 'contact' } 257 | const msg2 = { type: 'post', text: 'Testing 2' } 258 | const msg3 = { type: 'post', text: 'Testing 3' } 259 | let state = validate.initial() 260 | state = validate.appendNew(state, null, keys, msg, Date.now()) 261 | state = validate.appendNew(state, null, keys, msgContact, Date.now() + 1) 262 | state = validate.appendNew(state, null, keys2, msg2, Date.now() + 2) 263 | state = validate.appendNew(state, null, keys3, msg3, Date.now() + 3) 264 | 265 | const authorQuery = { 266 | type: 'AND', 267 | data: [ 268 | { 269 | type: 'EQUAL', 270 | data: { 271 | seek: helpers.seekType, 272 | value: helpers.toBipf('post'), 273 | indexType: 'type', 274 | indexName: 'type_post', 275 | }, 276 | }, 277 | { 278 | type: 'EQUAL', 279 | data: { 280 | seek: helpers.seekAuthor, 281 | value: helpers.toBipf(keys.id), 282 | indexType: 'author', 283 | indexAll: true, 284 | indexName: safeFilename('author_' + keys.id), 285 | }, 286 | }, 287 | ], 288 | } 289 | 290 | addMsg(state.queue[0].value, raf, (err, msg) => { 291 | addMsg(state.queue[1].value, raf, (err, msg) => { 292 | addMsg(state.queue[2].value, raf, (err, msg) => { 293 | addMsg(state.queue[3].value, raf, (err, msg) => { 294 | db.all(authorQuery, 0, false, false, 'declared', (err, results) => { 295 | t.error(err) 296 | t.equal(results.length, 1) 297 | t.equal(results[0].value.content.text, 'Testing 1') 298 | t.equal(db.indexes.size, 3 + 2 + 1 + 1) 299 | t.end() 300 | }) 301 | }) 302 | }) 303 | }) 304 | }) 305 | }) 306 | 307 | prepareAndRunTest('indexAll multiple reindexes', dir, (t, db, raf) => { 308 | const msg = { type: 'post', text: 'Testing 1' } 309 | const msgContact = { type: 'contact' } 310 | const msg2 = { type: 'post', text: 'Testing 2' } 311 | const msgAbout = { type: 'about' } 312 | let state = validate.initial() 313 | state = validate.appendNew(state, null, keys, msg, Date.now()) 314 | state = validate.appendNew(state, null, keys, msgContact, Date.now() + 1) 315 | state = validate.appendNew(state, null, keys2, msg2, Date.now() + 2) 316 | state = validate.appendNew(state, null, keys3, msgAbout, Date.now() + 3) 317 | 318 | function typeQuery(value) { 319 | return { 320 | type: 'EQUAL', 321 | data: { 322 | seek: helpers.seekType, 323 | value: helpers.toBipf(value), 324 | indexType: 'type', 325 | indexAll: true, 326 | indexName: safeFilename('type_' + value), 327 | }, 328 | } 329 | } 330 | 331 | addMsg(state.queue[0].value, raf, (err, msg) => { 332 | addMsg(state.queue[1].value, raf, (err, msg) => { 333 | db.all(typeQuery('post'), 0, false, false, 'declared', (err, results) => { 334 | t.equal(results.length, 1) 335 | t.equal(results[0].value.content.text, 'Testing 1') 336 | 337 | addMsg(state.queue[2].value, raf, (err, msg) => { 338 | addMsg(state.queue[3].value, raf, (err, msg) => { 339 | db.all( 340 | typeQuery('about'), 341 | 0, 342 | false, 343 | false, 344 | 'declared', 345 | (err, results) => { 346 | t.equal(results.length, 1) 347 | 348 | db.all( 349 | typeQuery('post'), 350 | 0, 351 | false, 352 | false, 353 | 'declared', 354 | (err, results) => { 355 | t.equal(results.length, 2) 356 | t.deepEqual( 357 | db.indexes.get('type_post').bitset.array(), 358 | [0, 2] 359 | ) 360 | t.deepEqual(db.indexes.get('type_contact').bitset.array(), [ 361 | 1, 362 | ]) 363 | t.deepEqual(db.indexes.get('type_about').bitset.array(), [ 364 | 3, 365 | ]) 366 | t.end() 367 | } 368 | ) 369 | } 370 | ) 371 | }) 372 | }) 373 | }) 374 | }) 375 | }) 376 | }) 377 | 378 | prepareAndRunTest('prepare an index', dir, (t, db, raf) => { 379 | t.plan(10) 380 | t.timeoutAfter(20e3) 381 | let msg = { type: 'post', text: 'Testing' } 382 | let state = validate.initial() 383 | for (var i = 0; i < 1000; ++i) { 384 | msg.text = 'Testing ' + i 385 | state = validate.appendNew(state, null, keys, msg, Date.now() + i) 386 | } 387 | 388 | const typeQuery = { 389 | type: 'EQUAL', 390 | data: { 391 | seek: helpers.seekType, 392 | value: helpers.toBipf('post'), 393 | indexType: 'type', 394 | indexName: 'type_post', 395 | }, 396 | } 397 | 398 | push( 399 | push.values(state.queue), 400 | push.asyncMap((q, cb) => { 401 | addMsg(q.value, raf, cb) 402 | }), 403 | push.collect((err, results) => { 404 | t.notOk(db.indexes.get('type_post')) 405 | t.notOk(db.status.value['type_post']) 406 | const expectedStatus = [undefined, -1, undefined] 407 | db.status((stats) => { 408 | t.deepEqual(stats['type_post'], expectedStatus.shift()) 409 | if (expectedStatus.length === 0) t.end() 410 | }) 411 | db.prepare(typeQuery, (err, duration) => { 412 | t.error(err, 'no error') 413 | t.equals(typeof duration, 'number') 414 | t.ok(duration) 415 | t.ok(db.indexes.get('type_post')) 416 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 417 | t.equal(results.length, 1000) 418 | }) 419 | }) 420 | }) 421 | ) 422 | }) 423 | 424 | prepareAndRunTest('prepare a DEFERRED operation', dir, (t, db, raf) => { 425 | const deferredQuery = { 426 | type: 'DEFERRED', 427 | task: (meta, cb) => { 428 | cb() 429 | }, 430 | } 431 | 432 | db.prepare(deferredQuery, (err, duration) => { 433 | t.error(err, 'no error') 434 | t.equals(typeof duration, 'number') 435 | t.end() 436 | }) 437 | }) 438 | -------------------------------------------------------------------------------- /test/bump-version.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const path = require('path') 8 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 9 | const rimraf = require('rimraf') 10 | const mkdirp = require('mkdirp') 11 | 12 | const dir = '/tmp/jitdb-bump-version' 13 | rimraf.sync(dir) 14 | mkdirp.sync(dir) 15 | 16 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 17 | 18 | const msg1 = { type: 'post', text: 'Testing post 1!' } 19 | const msg2 = { type: 'contact', text: 'Testing contact!' } 20 | const msg3 = { type: 'post', text: 'Testing post 2!' } 21 | 22 | let state = validate.initial() 23 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 24 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 25 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 26 | 27 | prepareAndRunTest('Bitvector index version bumped', dir, (t, db, raf) => { 28 | const msg = { type: 'post', text: 'Testing 1' } 29 | let state = validate.initial() 30 | state = validate.appendNew(state, null, keys, msg, Date.now()) 31 | 32 | const postQuery = { 33 | type: 'EQUAL', 34 | data: { 35 | seek: helpers.seekType, 36 | value: helpers.toBipf('post'), 37 | indexType: 'type', 38 | indexName: 'type_post', 39 | }, 40 | } 41 | 42 | addMsg(state.queue[0].value, raf, (err, msg) => { 43 | db.all(postQuery, 0, false, false, 'declared', (err, results) => { 44 | t.error(err) 45 | t.equal(results.length, 1) 46 | t.equal(results[0].value.content.text, 'Testing 1') 47 | 48 | const postQuery2 = { 49 | type: 'EQUAL', 50 | data: { 51 | seek: helpers.seekType, 52 | value: helpers.toBipf('post'), 53 | indexType: 'type', 54 | indexName: 'type_post', 55 | version: 2, 56 | }, 57 | } 58 | 59 | db.all(postQuery2, 0, false, false, 'declared', (err, results) => { 60 | t.error(err) 61 | t.equal(results.length, 1) 62 | t.equal(results[0].value.content.text, 'Testing 1') 63 | t.end() 64 | }) 65 | }) 66 | }) 67 | }) 68 | 69 | prepareAndRunTest('Prefix map index version bumped', dir, (t, db, raf) => { 70 | addMsg(state.queue[0].value, raf, (err, msg) => { 71 | addMsg(state.queue[1].value, raf, (err, msg) => { 72 | addMsg(state.queue[2].value, raf, (err, msg) => { 73 | const msgKey = state.queue[2].key 74 | const keyQuery = { 75 | type: 'EQUAL', 76 | data: { 77 | seek: helpers.seekKey, 78 | value: helpers.toBipf(msgKey), 79 | indexType: 'key', 80 | indexName: 'value_key', 81 | useMap: true, 82 | prefix: 32, 83 | prefixOffset: 1, 84 | }, 85 | } 86 | 87 | db.all(keyQuery, 0, false, false, 'declared', (err, results) => { 88 | t.equal(results.length, 1) 89 | t.equal(results[0].value.content.text, 'Testing post 2!') 90 | 91 | const keyQuery2 = { 92 | type: 'EQUAL', 93 | data: { 94 | seek: helpers.seekKey, 95 | value: helpers.toBipf(msgKey), 96 | indexType: 'key', 97 | indexName: 'value_key', 98 | useMap: true, 99 | prefix: 32, 100 | prefixOffset: 4, 101 | version: 2, 102 | }, 103 | } 104 | 105 | db.all(keyQuery2, 0, false, false, 'declared', (err, results) => { 106 | t.equal(results.length, 1) 107 | t.equal(results[0].value.content.text, 'Testing post 2!') 108 | t.end() 109 | }) 110 | }) 111 | }) 112 | }) 113 | }) 114 | }) 115 | -------------------------------------------------------------------------------- /test/common.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const Log = require('async-append-only-log') 6 | const bipf = require('bipf') 7 | const hash = require('ssb-keys/util').hash 8 | const path = require('path') 9 | const test = require('tape') 10 | const fs = require('fs') 11 | const JITDB = require('../index') 12 | const helpers = require('./helpers') 13 | 14 | module.exports = function () { 15 | function getId(msg) { 16 | return '%' + hash(JSON.stringify(msg, null, 2)) 17 | } 18 | 19 | function addMsg(msgVal, log, cb) { 20 | const msg = { 21 | key: getId(msgVal), 22 | value: msgVal, 23 | timestamp: Date.now(), 24 | } 25 | const buf = bipf.allocAndEncode(msg) 26 | log.append(buf, (err, offset) => { 27 | if (err) cb(err) 28 | // instead of cluttering the tests with onDrain, we just 29 | // simulate sync adds here 30 | else log.onDrain(() => cb(null, msg, offset)) 31 | }) 32 | } 33 | 34 | function addMsgPromise(msg, raf) { 35 | return new Promise((resolve, reject) => { 36 | addMsg(msg, raf, (err, msg, offset) => { 37 | if (err) reject(err) 38 | else resolve({ msg, offset }) 39 | }) 40 | }) 41 | } 42 | 43 | return { 44 | addMsg, 45 | addMsgPromise, 46 | 47 | prepareAndRunTest: function (name, dir, cb) { 48 | fs.closeSync(fs.openSync(path.join(dir, name), 'w')) // touch 49 | const log = Log(path.join(dir, name), { blockSize: 64 * 1024 }) 50 | const jitdb = JITDB(log, path.join(dir, 'indexes' + name)) 51 | jitdb.onReady(() => { 52 | test(name, (t) => cb(t, jitdb, log)) 53 | }) 54 | }, 55 | 56 | helpers, 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /test/compaction.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 Andre 'Staltz' Medeiros 2 | // 3 | // SPDX-License-Identifier: CC0-1.0 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const path = require('path') 8 | const rimraf = require('rimraf') 9 | const mkdirp = require('mkdirp') 10 | const pify = require('util').promisify 11 | const { prepareAndRunTest, addMsgPromise, helpers } = require('./common')() 12 | const { 13 | query, 14 | fromDB, 15 | where, 16 | slowEqual, 17 | paginate, 18 | toPullStream, 19 | } = require('../operators') 20 | 21 | const dir = '/tmp/jitdb-compaction' 22 | rimraf.sync(dir) 23 | mkdirp.sync(dir) 24 | 25 | const keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 26 | 27 | prepareAndRunTest('toPullStream post-compact', dir, async (t, jitdb, log) => { 28 | const content0 = { type: 'post', text: 'Testing 0' } 29 | const content1 = { type: 'post', text: 'Testing 1' } 30 | const content2 = { type: 'post', text: 'Testing 2' } 31 | const content3 = { type: 'post', text: 'Testing 3' } 32 | const content4 = { type: 'post', text: 'Testing 4' } 33 | const content5 = { type: 'post', text: 'Testing 5' } 34 | const content6 = { type: 'post', text: 'Testing 6' } 35 | const content7 = { type: 'post', text: 'Testing 7' } 36 | let state = validate.initial() 37 | state = validate.appendNew(state, null, keys, content0, Date.now()) 38 | state = validate.appendNew(state, null, keys, content1, Date.now() + 1) 39 | state = validate.appendNew(state, null, keys, content2, Date.now() + 2) 40 | state = validate.appendNew(state, null, keys, content3, Date.now() + 3) 41 | state = validate.appendNew(state, null, keys, content4, Date.now() + 4) 42 | state = validate.appendNew(state, null, keys, content5, Date.now() + 5) 43 | state = validate.appendNew(state, null, keys, content6, Date.now() + 6) 44 | state = validate.appendNew(state, null, keys, content7, Date.now() + 7) 45 | 46 | const offset0 = (await addMsgPromise(state.queue[0].value, log)).offset 47 | const offset1 = (await addMsgPromise(state.queue[1].value, log)).offset 48 | const offset2 = (await addMsgPromise(state.queue[2].value, log)).offset 49 | const offset3 = (await addMsgPromise(state.queue[3].value, log)).offset 50 | const offset4 = (await addMsgPromise(state.queue[4].value, log)).offset 51 | const offset5 = (await addMsgPromise(state.queue[5].value, log)).offset 52 | const offset6 = (await addMsgPromise(state.queue[6].value, log)).offset 53 | const offset7 = (await addMsgPromise(state.queue[7].value, log)).offset 54 | 55 | await pify(log.del)(offset1) 56 | t.pass('delete msg 1') 57 | await pify(log.del)(offset2) 58 | t.pass('delete msg 2') 59 | await pify(log.onDeletesFlushed)() 60 | 61 | const source = query( 62 | fromDB(jitdb), 63 | where(slowEqual('value.content.type', 'post')), 64 | paginate(3), 65 | toPullStream() 66 | ) 67 | 68 | const msgs = await pify(source)(null) 69 | t.equals(msgs.length, 3, '1st page has 3 messages') 70 | t.equals(msgs[0].value.content.text, 'Testing 0', 'msg 0') 71 | t.equals(msgs[1].value.content.text, 'Testing 3', 'msg 3') 72 | t.equals(msgs[2].value.content.text, 'Testing 4', 'msg 4') 73 | 74 | let queryStarted = false 75 | let queryEnded = false 76 | await new Promise((resolve) => { 77 | log.compactionProgress((stats) => { 78 | if (!stats.done && !queryStarted) { 79 | queryStarted = true 80 | source(null, (err, msgs) => { 81 | t.error(err, 'no error') 82 | t.equals(msgs.length, 3, '2nd page has 3 messages') 83 | t.equals(msgs[0].value.content.text, 'Testing 5', 'msg 5') 84 | t.equals(msgs[1].value.content.text, 'Testing 6', 'msg 6') 85 | t.equals(msgs[2].value.content.text, 'Testing 7', 'msg 7') 86 | queryEnded = true 87 | }) 88 | return false // abort listening to compaction progress 89 | } 90 | }) 91 | 92 | log.compact((err) => { 93 | if (err) t.fail(err) 94 | resolve() 95 | }) 96 | }) 97 | 98 | t.true(queryEnded, 'query ended') 99 | }) 100 | 101 | prepareAndRunTest('toPullStream post-compact 2', dir, async (t, jitdb, log) => { 102 | const content0 = { type: 'post', text: 'Testing 0' } 103 | const content1 = { type: 'post', text: 'Testing 1' } 104 | const content2 = { type: 'post', text: 'Testing 2' } 105 | const content3 = { type: 'post', text: 'Testing 3' } 106 | const content4 = { type: 'post', text: 'Testing 4' } 107 | const content5 = { type: 'post', text: 'Testing 5' } 108 | const content6 = { type: 'post', text: 'Testing 6' } 109 | const content7 = { type: 'post', text: 'Testing 7' } 110 | let state = validate.initial() 111 | state = validate.appendNew(state, null, keys, content0, Date.now()) 112 | state = validate.appendNew(state, null, keys, content1, Date.now() + 1) 113 | state = validate.appendNew(state, null, keys, content2, Date.now() + 2) 114 | state = validate.appendNew(state, null, keys, content3, Date.now() + 3) 115 | state = validate.appendNew(state, null, keys, content4, Date.now() + 4) 116 | state = validate.appendNew(state, null, keys, content5, Date.now() + 5) 117 | state = validate.appendNew(state, null, keys, content6, Date.now() + 6) 118 | state = validate.appendNew(state, null, keys, content7, Date.now() + 7) 119 | 120 | const offset0 = (await addMsgPromise(state.queue[0].value, log)).offset 121 | const offset1 = (await addMsgPromise(state.queue[1].value, log)).offset 122 | const offset2 = (await addMsgPromise(state.queue[2].value, log)).offset 123 | const offset3 = (await addMsgPromise(state.queue[3].value, log)).offset 124 | const offset4 = (await addMsgPromise(state.queue[4].value, log)).offset 125 | const offset5 = (await addMsgPromise(state.queue[5].value, log)).offset 126 | const offset6 = (await addMsgPromise(state.queue[6].value, log)).offset 127 | const offset7 = (await addMsgPromise(state.queue[7].value, log)).offset 128 | 129 | await pify(jitdb.prepare)(slowEqual('value.content.type', 'post')) 130 | 131 | await pify(log.del)(offset1) 132 | t.pass('delete msg 1') 133 | await pify(log.del)(offset2) 134 | t.pass('delete msg 2') 135 | await pify(log.onDeletesFlushed)() 136 | 137 | const source = query( 138 | fromDB(jitdb), 139 | where(slowEqual('value.content.type', 'post')), 140 | paginate(3), 141 | toPullStream() 142 | ) 143 | 144 | const msgs = await pify(source)(null) 145 | t.equals(msgs.length, 1, '1st page has 1 message') 146 | t.equals(msgs[0].value.content.text, 'Testing 0', 'msg 0') 147 | 148 | let queryStarted = false 149 | let queryEnded = false 150 | await new Promise((resolve) => { 151 | log.compactionProgress((stats) => { 152 | if (!stats.done && !queryStarted) { 153 | queryStarted = true 154 | source(null, (err, msgs) => { 155 | t.error(err, 'no error') 156 | t.equals(msgs.length, 3, '2nd page has 3 messages') 157 | t.equals(msgs[0].value.content.text, 'Testing 3', 'msg 3') 158 | t.equals(msgs[1].value.content.text, 'Testing 4', 'msg 4') 159 | t.equals(msgs[2].value.content.text, 'Testing 5', 'msg 5') 160 | queryEnded = true 161 | }) 162 | return false // abort listening to compaction progress 163 | } 164 | }) 165 | 166 | log.compact((err) => { 167 | if (err) t.fail(err) 168 | resolve() 169 | }) 170 | }) 171 | 172 | t.true(queryEnded, 'query ended') 173 | }) 174 | -------------------------------------------------------------------------------- /test/del.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const path = require('path') 8 | const pify = require('util').promisify 9 | const rimraf = require('rimraf') 10 | const mkdirp = require('mkdirp') 11 | const { prepareAndRunTest, addMsgPromise, helpers } = require('./common')() 12 | const { 13 | fromDB, 14 | where, 15 | query, 16 | slowEqual, 17 | paginate, 18 | toPullStream, 19 | } = require('../operators') 20 | 21 | const dir = '/tmp/jitdb-add' 22 | rimraf.sync(dir) 23 | mkdirp.sync(dir) 24 | 25 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 26 | 27 | prepareAndRunTest('delete then index', dir, async (t, jitdb, log) => { 28 | const content1 = { type: 'post', text: 'Testing 1' } 29 | const content2 = { type: 'post', text: 'Testing 2' } 30 | const content3 = { type: 'post', text: 'Testing 3' } 31 | let state = validate.initial() 32 | state = validate.appendNew(state, null, keys, content1, Date.now()) 33 | state = validate.appendNew(state, null, keys, content2, Date.now() + 1) 34 | state = validate.appendNew(state, null, keys, content3, Date.now() + 2) 35 | 36 | const query = { 37 | type: 'EQUAL', 38 | data: { 39 | seek: helpers.seekType, 40 | value: helpers.toBipf('post'), 41 | indexType: 'type', 42 | indexName: 'type_post', 43 | }, 44 | } 45 | 46 | const msg1 = (await addMsgPromise(state.queue[0].value, log)).msg 47 | const offset2 = (await addMsgPromise(state.queue[1].value, log)).offset 48 | const msg3 = (await addMsgPromise(state.queue[2].value, log)).msg 49 | 50 | await pify(log.del)(offset2) 51 | await pify(log.onDeletesFlushed)() 52 | 53 | const answer = await pify(jitdb.paginate)( 54 | query, 55 | 0, 56 | 2, 57 | false, 58 | false, 59 | 'declared', 60 | null 61 | ) 62 | t.deepEqual( 63 | answer.results.map((msg) => msg.value.content.text), 64 | ['Testing 1', 'Testing 3'], 65 | 'paginate got msg#1 and msg#3' 66 | ) 67 | 68 | const results = await pify(jitdb.all)(query, 0, false, false, 'declared') 69 | t.deepEqual( 70 | results.map((msg) => msg.value.content.text), 71 | ['Testing 1', 'Testing 3'], 72 | 'all got msg#1 and msg#3' 73 | ) 74 | }) 75 | 76 | prepareAndRunTest('index then delete', dir, async (t, jitdb, log) => { 77 | const content1 = { type: 'post', text: 'Testing 1' } 78 | const content2 = { type: 'post', text: 'Testing 2' } 79 | const content3 = { type: 'post', text: 'Testing 3' } 80 | let state = validate.initial() 81 | state = validate.appendNew(state, null, keys, content1, Date.now()) 82 | state = validate.appendNew(state, null, keys, content2, Date.now() + 1) 83 | state = validate.appendNew(state, null, keys, content3, Date.now() + 2) 84 | 85 | const query = { 86 | type: 'EQUAL', 87 | data: { 88 | seek: helpers.seekType, 89 | value: helpers.toBipf('post'), 90 | indexType: 'type', 91 | indexName: 'type_post', 92 | }, 93 | } 94 | 95 | const msg1 = (await addMsgPromise(state.queue[0].value, log)).msg 96 | const offset2 = (await addMsgPromise(state.queue[1].value, log)).offset 97 | const msg3 = (await addMsgPromise(state.queue[2].value, log)).msg 98 | 99 | await pify(jitdb.prepare)(query) 100 | 101 | await pify(log.del)(offset2) 102 | await pify(log.onDeletesFlushed)() 103 | 104 | const answer = await pify(jitdb.paginate)( 105 | query, 106 | 0, 107 | 2, 108 | false, 109 | false, 110 | 'declared', 111 | null 112 | ) 113 | t.deepEqual( 114 | answer.results.map((msg) => msg.value.content.text), 115 | ['Testing 1'], 116 | 'paginate got msg#1' 117 | ) 118 | 119 | const answer2 = await pify(jitdb.paginate)( 120 | query, 121 | answer.nextSeq, 122 | 2, 123 | false, 124 | false, 125 | 'declared', 126 | null 127 | ) 128 | t.deepEqual( 129 | answer2.results.map((msg) => msg.value.content.text), 130 | ['Testing 3'], 131 | 'paginate got msg#3' 132 | ) 133 | 134 | const results = await pify(jitdb.all)(query, 0, false, false, 'declared') 135 | t.deepEqual( 136 | results.map((msg) => msg.value.content.text), 137 | ['Testing 1', 'Testing 3'], 138 | 'all got msg#1 and msg#3' 139 | ) 140 | }) 141 | 142 | prepareAndRunTest('index then delete many', dir, async (t, jitdb, log) => { 143 | const content1 = { type: 'post', text: 'Testing 1' } 144 | const content2 = { type: 'post', text: 'Testing 2' } 145 | const content3 = { type: 'post', text: 'Testing 3' } 146 | const content4 = { type: 'post', text: 'Testing 4' } 147 | let state = validate.initial() 148 | state = validate.appendNew(state, null, keys, content1, Date.now()) 149 | state = validate.appendNew(state, null, keys, content2, Date.now() + 1) 150 | state = validate.appendNew(state, null, keys, content3, Date.now() + 2) 151 | state = validate.appendNew(state, null, keys, content4, Date.now() + 2) 152 | 153 | const offset1 = (await addMsgPromise(state.queue[0].value, log)).offset 154 | const offset2 = (await addMsgPromise(state.queue[1].value, log)).offset 155 | const msg3 = (await addMsgPromise(state.queue[2].value, log)).msg 156 | const msg4 = (await addMsgPromise(state.queue[3].value, log)).msg 157 | 158 | await pify(jitdb.prepare)(slowEqual('value.content.type', 'post')) 159 | 160 | await pify(log.del)(offset1) 161 | await pify(log.del)(offset2) 162 | await pify(log.onDeletesFlushed)() 163 | 164 | const source = query( 165 | fromDB(jitdb), 166 | where(slowEqual('value.content.type', 'post')), 167 | paginate(2), 168 | toPullStream() 169 | ) 170 | 171 | const page1 = await pify(source)(null) 172 | t.deepEqual( 173 | page1.map((msg) => msg.value.content.text), 174 | ['Testing 3', 'Testing 4'], 175 | 'non-empty page 1' 176 | ) 177 | }) 178 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const bipf = require('bipf') 6 | 7 | const B_KEY = Buffer.from('key') 8 | const B_VOTE = Buffer.from('vote') 9 | const B_LINK = Buffer.from('link') 10 | const B_AUTHOR = Buffer.from('author') 11 | const B_CONTENT = Buffer.from('content') 12 | const B_TYPE = Buffer.from('type') 13 | const B_ROOT = Buffer.from('root') 14 | const B_META = Buffer.from('meta') 15 | const B_ANIMALS = Buffer.from('animals') 16 | const B_WORD = Buffer.from('word') 17 | const B_PRIVATE = Buffer.from('private') 18 | const B_CHANNEL = Buffer.from('channel') 19 | 20 | module.exports = { 21 | toBipf(value) { 22 | return bipf.allocAndEncode(value) 23 | }, 24 | 25 | seekKey(buffer) { 26 | return bipf.seekKey(buffer, 0, B_KEY) 27 | }, 28 | 29 | seekAuthor(buffer, start, pValue) { 30 | if (pValue < 0) return -1 31 | return bipf.seekKey(buffer, pValue, B_AUTHOR) 32 | }, 33 | 34 | seekVoteLink(buffer, start, pValue) { 35 | if (pValue < 0) return -1 36 | const pValueContent = bipf.seekKey(buffer, pValue, B_CONTENT) 37 | if (pValueContent < 0) return -1 38 | const pValueContentVote = bipf.seekKey(buffer, pValueContent, B_VOTE) 39 | if (pValueContentVote < 0) return -1 40 | return bipf.seekKey(buffer, pValueContentVote, B_LINK) 41 | }, 42 | 43 | seekType(buffer, start, pValue) { 44 | if (pValue < 0) return -1 45 | const pValueContent = bipf.seekKey(buffer, pValue, B_CONTENT) 46 | if (pValueContent < 0) return -1 47 | return bipf.seekKey(buffer, pValueContent, B_TYPE) 48 | }, 49 | 50 | seekAnimals(buffer, start, pValue) { 51 | if (pValue < 0) return -1 52 | const pValueContent = bipf.seekKey(buffer, pValue, B_CONTENT) 53 | if (pValueContent < 0) return -1 54 | return bipf.seekKey(buffer, pValueContent, B_ANIMALS) 55 | }, 56 | 57 | pluckWord(buffer, start) { 58 | return bipf.seekKey(buffer, start, B_WORD) 59 | }, 60 | 61 | seekRoot(buffer, start, pValue) { 62 | if (pValue < 0) return -1 63 | const pValueContent = bipf.seekKey(buffer, pValue, B_CONTENT) 64 | if (pValueContent < 0) return -1 65 | return bipf.seekKey(buffer, pValueContent, B_ROOT) 66 | }, 67 | 68 | seekPrivate(buffer, start, pValue) { 69 | if (pValue < 0) return -1 70 | const pValueMeta = bipf.seekKey(buffer, pValue, B_META) 71 | if (pValueMeta < 0) return -1 72 | return bipf.seekKey(buffer, pValueMeta, B_PRIVATE) 73 | }, 74 | 75 | seekChannel(buffer, start, pValue) { 76 | if (pValue < 0) return -1 77 | const pValueContent = bipf.seekKey(buffer, pValue, B_CONTENT) 78 | if (pValueContent < 0) return -1 79 | return bipf.seekKey(buffer, pValueContent, B_CHANNEL) 80 | }, 81 | } 82 | -------------------------------------------------------------------------------- /test/live-then-grow.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const path = require('path') 8 | const rimraf = require('rimraf') 9 | const mkdirp = require('mkdirp') 10 | const pull = require('pull-stream') 11 | const { addMsg, prepareAndRunTest } = require('./common')() 12 | const { 13 | where, 14 | query, 15 | fromDB, 16 | live, 17 | toPullStream, 18 | slowEqual, 19 | } = require('../operators') 20 | 21 | const dir = '/tmp/jitdb-live-then-grow' 22 | rimraf.sync(dir) 23 | mkdirp.sync(dir) 24 | 25 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 26 | 27 | prepareAndRunTest('Live toPullStream from empty log', dir, (t, db, raf) => { 28 | const msg1 = { type: 'post', text: '1st' } 29 | const msg2 = { type: 'post', text: '2nd' } 30 | let state = validate.initial() 31 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 32 | state = validate.appendNew(state, null, keys, msg2, Date.now()) 33 | 34 | var i = 1 35 | pull( 36 | query( 37 | fromDB(db), 38 | where(slowEqual('value.content.type', 'post')), 39 | live(), 40 | toPullStream() 41 | ), 42 | pull.drain((result) => { 43 | if (i++ === 1) { 44 | t.equal(result.value.content.text, '1st') 45 | addMsg(state.queue[1].value, raf, (err, m2) => {}) 46 | } else { 47 | t.equal(result.value.content.text, '2nd') 48 | t.end() 49 | } 50 | }) 51 | ) 52 | 53 | addMsg(state.queue[0].value, raf, (err, m1) => {}) 54 | }) 55 | -------------------------------------------------------------------------------- /test/live.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const test = require('tape') 6 | const validate = require('ssb-validate') 7 | const ssbKeys = require('ssb-keys') 8 | const path = require('path') 9 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 10 | const rimraf = require('rimraf') 11 | const mkdirp = require('mkdirp') 12 | const pull = require('pull-stream') 13 | 14 | const dir = '/tmp/jitdb-live' 15 | rimraf.sync(dir) 16 | mkdirp.sync(dir) 17 | 18 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 19 | var keys2 = ssbKeys.loadOrCreateSync(path.join(dir, 'secret2')) 20 | var keys3 = ssbKeys.loadOrCreateSync(path.join(dir, 'secret3')) 21 | 22 | prepareAndRunTest('Live', dir, (t, db, raf) => { 23 | const msg = { type: 'post', text: 'Testing!' } 24 | let state = validate.initial() 25 | state = validate.appendNew(state, null, keys, msg, Date.now()) 26 | state = validate.appendNew(state, null, keys2, msg, Date.now() + 1) 27 | 28 | const typeQuery = { 29 | type: 'EQUAL', 30 | data: { 31 | seek: helpers.seekType, 32 | value: helpers.toBipf('post'), 33 | indexType: 'type', 34 | indexName: 'type_post', 35 | }, 36 | } 37 | 38 | var i = 1 39 | pull( 40 | db.live(typeQuery), 41 | pull.drain((result) => { 42 | if (i++ == 1) { 43 | t.equal(result.key, state.queue[0].key) 44 | addMsg(state.queue[1].value, raf, (err, msg1) => {}) 45 | } else { 46 | t.equal(result.key, state.queue[1].key) 47 | t.end() 48 | } 49 | }) 50 | ) 51 | 52 | addMsg(state.queue[0].value, raf, (err, msg1) => { 53 | // console.log("waiting for live query") 54 | }) 55 | }) 56 | 57 | prepareAndRunTest('Live AND', dir, (t, db, raf) => { 58 | const msg = { type: 'post', text: 'Testing ' + keys.id } 59 | const msg2 = { type: 'post', text: 'Testing ' + keys2.id } 60 | let state = validate.initial() 61 | state = validate.appendNew(state, null, keys, msg, Date.now()) 62 | state = validate.appendNew(state, null, keys2, msg2, Date.now() + 1) 63 | 64 | const filterQuery = { 65 | type: 'AND', 66 | data: [ 67 | { 68 | type: 'EQUAL', 69 | data: { 70 | seek: helpers.seekAuthor, 71 | value: helpers.toBipf(keys.id), 72 | indexType: 'author', 73 | indexName: 'author_' + keys.id, 74 | }, 75 | }, 76 | { 77 | type: 'EQUAL', 78 | data: { 79 | seek: helpers.seekType, 80 | value: helpers.toBipf('post'), 81 | indexType: 'type', 82 | indexName: 'type_post', 83 | }, 84 | }, 85 | ], 86 | } 87 | 88 | var i = 1 89 | pull( 90 | db.live(filterQuery), 91 | pull.drain((result) => { 92 | if (i++ == 1) { 93 | t.equal(result.key, state.queue[0].key) 94 | addMsg(state.queue[1].value, raf, (err, msg1) => {}) 95 | 96 | setTimeout(() => { 97 | t.end() 98 | }, 500) 99 | } else { 100 | t.fail('should only be called once') 101 | } 102 | }) 103 | ) 104 | 105 | addMsg(state.queue[0].value, raf, (err, msg1) => { 106 | // console.log("waiting for live query") 107 | }) 108 | }) 109 | 110 | prepareAndRunTest('Live OR', dir, (t, db, raf) => { 111 | const msg = { type: 'post', text: 'Testing ' + keys.id } 112 | const msg2 = { type: 'post', text: 'Testing ' + keys2.id } 113 | const msg3 = { type: 'post', text: 'Testing ' + keys3.id } 114 | let state = validate.initial() 115 | state = validate.appendNew(state, null, keys, msg, Date.now()) 116 | state = validate.appendNew(state, null, keys2, msg2, Date.now() + 1) 117 | state = validate.appendNew(state, null, keys3, msg3, Date.now() + 2) 118 | 119 | const authorQuery = { 120 | type: 'OR', 121 | data: [ 122 | { 123 | type: 'EQUAL', 124 | data: { 125 | seek: helpers.seekAuthor, 126 | value: helpers.toBipf(keys.id), 127 | indexType: 'author', 128 | indexName: 'author_' + keys.id, 129 | }, 130 | }, 131 | { 132 | type: 'EQUAL', 133 | data: { 134 | seek: helpers.seekAuthor, 135 | value: helpers.toBipf(keys2.id), 136 | indexType: 'author', 137 | indexName: 'author_' + keys2.id, 138 | }, 139 | }, 140 | ], 141 | } 142 | 143 | const filterQuery = { 144 | type: 'AND', 145 | data: [ 146 | authorQuery, 147 | { 148 | type: 'EQUAL', 149 | data: { 150 | seek: helpers.seekType, 151 | value: helpers.toBipf('post'), 152 | indexType: 'type', 153 | indexName: 'type_post', 154 | }, 155 | }, 156 | ], 157 | } 158 | 159 | var i = 1 160 | pull( 161 | db.live(filterQuery), 162 | pull.drain((result) => { 163 | if (i == 1) { 164 | t.equal(result.key, state.queue[0].key) 165 | addMsg(state.queue[1].value, raf, () => {}) 166 | } else if (i == 2) { 167 | t.equal(result.key, state.queue[1].key) 168 | addMsg(state.queue[2].value, raf, () => {}) 169 | 170 | setTimeout(() => { 171 | t.end() 172 | }, 500) 173 | } else { 174 | t.fail('should only be called for the first 2') 175 | } 176 | 177 | i += 1 178 | }) 179 | ) 180 | 181 | addMsg(state.queue[0].value, raf, (err, msg1) => { 182 | // console.log("waiting for live query") 183 | }) 184 | }) 185 | 186 | prepareAndRunTest('Live GTE', dir, (t, db, raf) => { 187 | const msg = { type: 'post', text: 'Testing ' + keys.id } 188 | const msg2 = { type: 'post', text: 'Testing ' + keys2.id } 189 | let state = validate.initial() 190 | state = validate.appendNew(state, null, keys, msg, Date.now()) 191 | state = validate.appendNew(state, null, keys2, msg2, Date.now() + 1) 192 | 193 | const filterQuery = { 194 | type: 'AND', 195 | data: [ 196 | { 197 | type: 'EQUAL', 198 | data: { 199 | seek: helpers.seekAuthor, 200 | value: helpers.toBipf(keys.id), 201 | indexType: 'author', 202 | indexName: 'author_' + keys.id, 203 | }, 204 | }, 205 | { 206 | type: 'GTE', 207 | data: { 208 | value: 1, 209 | indexName: 'sequence', 210 | }, 211 | }, 212 | ], 213 | } 214 | 215 | var i = 1 216 | pull( 217 | db.live(filterQuery), 218 | pull.drain((result) => { 219 | if (i++ == 1) { 220 | t.equal(result.key, state.queue[0].key) 221 | addMsg(state.queue[1].value, raf, (err, msg1) => {}) 222 | 223 | setTimeout(() => { 224 | t.end() 225 | }, 500) 226 | } else { 227 | t.fail('should only be called once') 228 | } 229 | }) 230 | ) 231 | 232 | addMsg(state.queue[0].value, raf, (err, msg1) => { 233 | // console.log("waiting for live query") 234 | }) 235 | }) 236 | 237 | prepareAndRunTest('Live with initial values', dir, (t, db, raf) => { 238 | const msg = { type: 'post', text: 'Testing!' } 239 | let state = validate.initial() 240 | state = validate.appendNew(state, null, keys, msg, Date.now()) 241 | state = validate.appendNew(state, null, keys2, msg, Date.now() + 1) 242 | 243 | const typeQuery = { 244 | type: 'EQUAL', 245 | data: { 246 | seek: helpers.seekType, 247 | value: helpers.toBipf('post'), 248 | indexType: 'type', 249 | indexName: 'type_post', 250 | }, 251 | } 252 | 253 | addMsg(state.queue[0].value, raf, (err, msg1) => { 254 | // create index 255 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 256 | t.equal(results.length, 1) 257 | 258 | pull( 259 | db.live(typeQuery), 260 | pull.drain((result) => { 261 | t.equal(result.key, state.queue[1].key) 262 | 263 | // rerun on updated index 264 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 265 | t.equal(results.length, 2) 266 | t.end() 267 | }) 268 | }) 269 | ) 270 | 271 | addMsg(state.queue[1].value, raf, (err, msg1) => { 272 | // console.log("waiting for live query") 273 | }) 274 | }) 275 | }) 276 | }) 277 | 278 | prepareAndRunTest('Live with seq values', dir, (t, db, raf) => { 279 | let state = validate.initial() 280 | 281 | const n = 11 282 | 283 | let a = [] 284 | for (var i = 0; i < n; ++i) { 285 | let msg = { type: 'post', text: 'Testing!' } 286 | msg.i = i 287 | if (i > 0 && i % 2 == 0) msg.type = 'non-post' 288 | else msg.type = 'post' 289 | state = validate.appendNew(state, null, keys, msg, Date.now() + i) 290 | if (i > 0) a.push(i) 291 | } 292 | 293 | let ps = pull( 294 | pull.values(a), 295 | pull.asyncMap((i, cb) => { 296 | addMsg(state.queue[i].value, raf, (err) => cb(err, i)) 297 | }) 298 | ) 299 | 300 | const typeQuery = { 301 | type: 'AND', 302 | data: [ 303 | { 304 | type: 'EQUAL', 305 | data: { 306 | seek: helpers.seekType, 307 | value: helpers.toBipf('post'), 308 | indexType: 'type', 309 | indexName: 'type_post', 310 | }, 311 | }, 312 | { 313 | type: 'OR', 314 | data: [ 315 | { 316 | type: 'SEQS', 317 | seqs: [0], 318 | }, 319 | { 320 | type: 'LIVESEQS', 321 | stream: ps, 322 | }, 323 | ], 324 | }, 325 | ], 326 | } 327 | 328 | addMsg(state.queue[0].value, raf, (err, msg1) => { 329 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 330 | t.equal(results.length, 1) 331 | 332 | let liveI = 1 333 | 334 | pull( 335 | db.live(typeQuery), 336 | pull.drain((result) => { 337 | if (result.key !== state.queue[liveI].key) { 338 | t.fail('result.key did not match state.queue[liveI].key') 339 | } 340 | liveI += 2 341 | if (liveI == n) t.end() 342 | }) 343 | ) 344 | }) 345 | }) 346 | }) 347 | 348 | prepareAndRunTest('Live with cleanup', dir, (t, db, raf) => { 349 | const msg = { type: 'post', text: 'Testing!' } 350 | let state = validate.initial() 351 | state = validate.appendNew(state, null, keys, msg, Date.now()) 352 | state = validate.appendNew(state, null, keys2, msg, Date.now() + 1) 353 | 354 | const typeQuery = { 355 | type: 'EQUAL', 356 | data: { 357 | seek: helpers.seekType, 358 | value: helpers.toBipf('post'), 359 | indexType: 'type', 360 | indexName: 'type_post', 361 | }, 362 | } 363 | 364 | pull( 365 | db.live(typeQuery), 366 | pull.drain((result) => { 367 | t.equal(result.key, state.queue[0].key) 368 | 369 | // add second live query 370 | pull( 371 | db.live(typeQuery), 372 | pull.drain((result) => { 373 | t.equal(result.key, state.queue[1].key) 374 | t.end() 375 | }) 376 | ) 377 | 378 | return false // abort 379 | }) 380 | ) 381 | 382 | addMsg(state.queue[0].value, raf, (err, msg1) => { 383 | setTimeout(() => { 384 | addMsg(state.queue[1].value, raf, (err, msg1) => { 385 | // console.log("waiting for live query") 386 | }) 387 | }, 100) 388 | }) 389 | }) 390 | -------------------------------------------------------------------------------- /test/lookup.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const { prepareAndRunTest, addMsg } = require('./common')() 8 | const rimraf = require('rimraf') 9 | const mkdirp = require('mkdirp') 10 | const { slowEqual } = require('../operators') 11 | 12 | const dir = '/tmp/jitdb-lookup-api' 13 | rimraf.sync(dir) 14 | mkdirp.sync(dir) 15 | 16 | const alice = ssbKeys.generate('ed25519', Buffer.alloc(32, 'a')) 17 | 18 | prepareAndRunTest('lookup "seq"', dir, (t, jitdb, log) => { 19 | log.append(Buffer.from('hello'), (e1, offset0) => { 20 | log.append(Buffer.from('world'), (e2, offset1) => { 21 | log.append(Buffer.from('foobar'), (e3, offset2) => { 22 | log.onDrain(() => { 23 | jitdb.lookup('seq', 0, (err, offset) => { 24 | t.error(err, 'no error') 25 | t.equals(offset, offset0) 26 | jitdb.lookup('seq', 1, (err, offset) => { 27 | t.error(err, 'no error') 28 | t.equals(offset, offset1) 29 | jitdb.lookup('seq', 2, (err, offset) => { 30 | t.error(err, 'no error') 31 | t.equals(offset, offset2) 32 | t.end() 33 | }) 34 | }) 35 | }) 36 | }) 37 | }) 38 | }) 39 | }) 40 | }) 41 | 42 | prepareAndRunTest('lookup operation', dir, (t, jitdb, log) => { 43 | const msg1 = { type: 'post', text: '1st', animals: ['cat', 'dog', 'bird'] } 44 | const msg2 = { type: 'contact', text: '2nd', animals: ['bird'] } 45 | const msg3 = { type: 'post', text: '3rd', animals: ['cat'] } 46 | 47 | let state = validate.initial() 48 | state = validate.appendNew(state, null, alice, msg1, Date.now()) 49 | state = validate.appendNew(state, null, alice, msg2, Date.now() + 1) 50 | state = validate.appendNew(state, null, alice, msg3, Date.now() + 2) 51 | 52 | addMsg(state.queue[0].value, log, (e1, m1) => { 53 | addMsg(state.queue[1].value, log, (e2, m2) => { 54 | addMsg(state.queue[2].value, log, (e3, m3) => { 55 | const op = slowEqual('value.author', 'whatever', { 56 | prefix: 32, 57 | indexType: 'value_author', 58 | }) 59 | jitdb.prepare(op, (err) => { 60 | t.error(err, 'no error') 61 | jitdb.lookup(op, 0, (err, authorAsUint32LE) => { 62 | t.error(err, 'no error') 63 | const buf = Buffer.alloc(4) 64 | buf.writeUInt32LE(authorAsUint32LE) 65 | const prefix = buf.toString('ascii') 66 | t.equals(prefix, alice.id.slice(0, 4)) 67 | t.end() 68 | }) 69 | }) 70 | }) 71 | }) 72 | }) 73 | }) 74 | 75 | prepareAndRunTest('lookup operation on unready index', dir, (t, jitdb, log) => { 76 | const msg1 = { type: 'post', text: '1st', animals: ['cat', 'dog', 'bird'] } 77 | const msg2 = { type: 'contact', text: '2nd', animals: ['bird'] } 78 | const msg3 = { type: 'post', text: '3rd', animals: ['cat'] } 79 | 80 | let state = validate.initial() 81 | state = validate.appendNew(state, null, alice, msg1, Date.now()) 82 | state = validate.appendNew(state, null, alice, msg2, Date.now() + 1) 83 | state = validate.appendNew(state, null, alice, msg3, Date.now() + 2) 84 | 85 | addMsg(state.queue[0].value, log, (e1, m1) => { 86 | addMsg(state.queue[1].value, log, (e2, m2) => { 87 | addMsg(state.queue[2].value, log, (e3, m3) => { 88 | const op = slowEqual('value.author', 'whatever', { 89 | prefix: 32, 90 | indexType: 'value_author', 91 | }) 92 | jitdb.lookup(op, 0, (err, authorAsUint32LE) => { 93 | t.ok(err, 'has error') 94 | t.notOk(authorAsUint32LE, 'no result') 95 | t.match(err.message, /not found/) 96 | t.end() 97 | }) 98 | }) 99 | }) 100 | }) 101 | }) 102 | -------------------------------------------------------------------------------- /test/prefix.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const validate = require('ssb-validate') 6 | const ssbKeys = require('ssb-keys') 7 | const path = require('path') 8 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 9 | const rimraf = require('rimraf') 10 | const mkdirp = require('mkdirp') 11 | 12 | const jitdb = require('../index') 13 | 14 | const dir = '/tmp/jitdb-prefix' 15 | rimraf.sync(dir) 16 | mkdirp.sync(dir) 17 | 18 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 19 | 20 | prepareAndRunTest('Prefix equal', dir, (t, db, raf) => { 21 | const msg1 = { type: 'post', text: 'Testing!' } 22 | const msg2 = { type: 'contact', text: 'Testing!' } 23 | const msg3 = { type: 'post', text: 'Testing 2!' } 24 | 25 | let state = validate.initial() 26 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 27 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 28 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 29 | 30 | const typeQuery = { 31 | type: 'EQUAL', 32 | data: { 33 | seek: helpers.seekType, 34 | value: helpers.toBipf('post'), 35 | indexType: 'type', 36 | indexName: 'value_content_type_post', 37 | prefix: 32, 38 | }, 39 | } 40 | 41 | addMsg(state.queue[0].value, raf, (err, msg) => { 42 | addMsg(state.queue[1].value, raf, (err, msg) => { 43 | addMsg(state.queue[2].value, raf, (err, msg) => { 44 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 45 | t.equal(results.length, 2) 46 | t.equal(results[0].value.content.type, 'post') 47 | t.equal(results[1].value.content.type, 'post') 48 | t.end() 49 | }) 50 | }) 51 | }) 52 | }) 53 | }) 54 | 55 | prepareAndRunTest('Normal index renamed to prefix', dir, (t, db, raf) => { 56 | const msg1 = { type: 'post', text: 'Testing!' } 57 | const msg2 = { type: 'contact', text: 'Testing!' } 58 | const msg3 = { type: 'post', text: 'Testing 2!' } 59 | 60 | let state = validate.initial() 61 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 62 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 63 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 64 | 65 | const normalQuery = { 66 | type: 'EQUAL', 67 | data: { 68 | seek: helpers.seekType, 69 | value: helpers.toBipf('post'), 70 | indexType: 'type', 71 | indexName: 'value_content_type_post', 72 | }, 73 | } 74 | 75 | const prefixQuery = { 76 | type: 'EQUAL', 77 | data: { 78 | seek: helpers.seekType, 79 | value: helpers.toBipf('post'), 80 | indexType: 'type', 81 | indexName: 'value_content_type', 82 | prefix: 32, 83 | }, 84 | } 85 | 86 | addMsg(state.queue[0].value, raf, (err, msg) => { 87 | addMsg(state.queue[1].value, raf, (err, msg) => { 88 | addMsg(state.queue[2].value, raf, (err, msg) => { 89 | db.all(normalQuery, 0, false, false, 'declared', (err, results) => { 90 | t.equal(results.length, 2) 91 | t.equal(results[0].value.content.type, 'post') 92 | t.equal(results[1].value.content.type, 'post') 93 | db.all(prefixQuery, 0, false, false, 'declared', (err, results2) => { 94 | t.equal(results2.length, 2) 95 | t.equal(results2[0].value.content.type, 'post') 96 | t.equal(results2[1].value.content.type, 'post') 97 | t.end() 98 | }) 99 | }) 100 | }) 101 | }) 102 | }) 103 | }) 104 | 105 | prepareAndRunTest('Prefix index skips deleted records', dir, (t, db, raf) => { 106 | const msg1 = { type: 'post', text: 'Testing!' } 107 | const msg2 = { type: 'contact', text: 'Testing!' } 108 | const msg3 = { type: 'post', text: 'Testing 2!' } 109 | 110 | let state = validate.initial() 111 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 112 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 113 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 114 | 115 | const prefixQuery = { 116 | type: 'EQUAL', 117 | data: { 118 | seek: helpers.seekType, 119 | value: helpers.toBipf('post'), 120 | indexType: 'type', 121 | indexName: 'value_content_type_post', 122 | prefix: 32, 123 | }, 124 | } 125 | 126 | // Cloned to avoid the op-to-bitset cache 127 | const prefixQuery2 = { 128 | type: 'EQUAL', 129 | data: { 130 | seek: helpers.seekType, 131 | value: helpers.toBipf('post'), 132 | indexType: 'type', 133 | indexName: 'value_content_type_post', 134 | prefix: 32, 135 | }, 136 | } 137 | 138 | addMsg(state.queue[0].value, raf, (err1) => { 139 | addMsg(state.queue[1].value, raf, (err2) => { 140 | addMsg(state.queue[2].value, raf, (err3) => { 141 | db.all(prefixQuery, 0, false, true, 'declared', (err4, offsets) => { 142 | t.error(err4, 'no err4') 143 | t.deepEqual(offsets, [0, 760]) 144 | raf.del(760, (err5) => { 145 | t.error(err5, 'no err5') 146 | db.all( 147 | prefixQuery2, 148 | 0, 149 | false, 150 | false, 151 | 'declared', 152 | (err6, results) => { 153 | t.error(err6, 'no err6') 154 | t.equal(results.length, 1) 155 | t.equal(results[0].value.content.type, 'post') 156 | t.equal(results[0].value.content.text, 'Testing!') 157 | t.end() 158 | } 159 | ) 160 | }) 161 | }) 162 | }) 163 | }) 164 | }) 165 | }) 166 | 167 | prepareAndRunTest('Prefix larger than actual value', dir, (t, db, raf) => { 168 | const msg1 = { type: 'post', text: 'First', channel: 'foo' } 169 | const msg2 = { type: 'contact', text: 'Second' } 170 | const msg3 = { type: 'post', text: 'Third' } 171 | 172 | let state = validate.initial() 173 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 174 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 175 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 176 | 177 | const channelQuery = { 178 | type: 'EQUAL', 179 | data: { 180 | seek: helpers.seekChannel, 181 | value: helpers.toBipf('foo'), 182 | indexType: 'channel', 183 | indexName: 'value_content_channel_foo', 184 | prefix: 32, 185 | }, 186 | } 187 | 188 | addMsg(state.queue[0].value, raf, (err, msg) => { 189 | addMsg(state.queue[1].value, raf, (err, msg) => { 190 | addMsg(state.queue[2].value, raf, (err, msg) => { 191 | db.all(channelQuery, 0, false, false, 'declared', (err, results) => { 192 | t.equal(results.length, 1) 193 | t.equal(results[0].value.content.text, 'First') 194 | t.end() 195 | }) 196 | }) 197 | }) 198 | }) 199 | }) 200 | 201 | prepareAndRunTest('Prefix equal falsy', dir, (t, db, raf) => { 202 | const msg1 = { type: 'post', text: 'First', channel: 'foo' } 203 | const msg2 = { type: 'contact', text: 'Second' } 204 | const msg3 = { type: 'post', text: 'Third' } 205 | 206 | let state = validate.initial() 207 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 208 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 209 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 210 | 211 | const channelQuery = { 212 | type: 'EQUAL', 213 | data: { 214 | seek: helpers.seekChannel, 215 | value: null, 216 | indexType: 'channel', 217 | indexName: 'value_content_channel_', 218 | prefix: 32, 219 | }, 220 | } 221 | 222 | addMsg(state.queue[0].value, raf, (err, msg) => { 223 | addMsg(state.queue[1].value, raf, (err, msg) => { 224 | addMsg(state.queue[2].value, raf, (err, msg) => { 225 | db.all(channelQuery, 0, false, false, 'declared', (err, results) => { 226 | t.equal(results.length, 2) 227 | t.equal(results[0].value.content.text, 'Second') 228 | t.equal(results[1].value.content.text, 'Third') 229 | t.end() 230 | }) 231 | }) 232 | }) 233 | }) 234 | }) 235 | 236 | prepareAndRunTest('Prefix equal', dir, (t, db, raf) => { 237 | const name = 'Prefix equal' 238 | 239 | const msg1 = { type: 'post', text: 'Testing!' } 240 | const msg2 = { type: 'contact', text: 'Testing!' } 241 | const msg3 = { 242 | type: 'vote', 243 | vote: { 244 | link: '%wOtfXXopI3mTHL6F7Y3XXNtpxws9mQdaEocNJuKtAZo=.sha256', 245 | value: 1, 246 | expression: 'Like', 247 | }, 248 | } 249 | const msg4 = { 250 | type: 'vote', 251 | vote: { 252 | link: '%wOtfXXopI3mTHL6F7Y3XXNtpxws9mQdaEocNJuKtAZo=.sha256', 253 | value: 0, 254 | expression: 'UnLike', 255 | }, 256 | } 257 | const msg5 = { 258 | type: 'vote', 259 | vote: { 260 | link: '%wOtfXXopI3mTHL6F7Y3XXNtpxws9mQdaEocNJuKtAZo=.sha256', 261 | value: 1, 262 | expression: 'Like', 263 | }, 264 | } 265 | 266 | let state = validate.initial() 267 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 268 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 269 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 270 | state = validate.appendNew(state, null, keys, msg4, Date.now() + 3) 271 | state = validate.appendNew(state, null, keys, msg5, Date.now() + 4) 272 | 273 | const voteQuery = { 274 | type: 'EQUAL', 275 | data: { 276 | seek: helpers.seekVoteLink, 277 | value: helpers.toBipf( 278 | '%wOtfXXopI3mTHL6F7Y3XXNtpxws9mQdaEocNJuKtAZo=.sha256' 279 | ), 280 | indexType: 'value_content_vote_link', 281 | indexName: 'value_content_vote_link', 282 | prefix: 32, 283 | }, 284 | } 285 | 286 | addMsg(state.queue[0].value, raf, (err, msg) => { 287 | addMsg(state.queue[1].value, raf, (err, msg) => { 288 | addMsg(state.queue[2].value, raf, (err, msg) => { 289 | db.all(voteQuery, 0, false, false, 'declared', (err, results) => { 290 | t.equal(results.length, 1) 291 | t.equal(results[0].value.content.type, 'vote') 292 | 293 | db = jitdb(raf, path.join(dir, 'indexes' + name)) 294 | db.onReady(() => { 295 | addMsg(state.queue[3].value, raf, (err, msg) => { 296 | db.all(voteQuery, 0, false, false, 'declared', (err, results) => { 297 | t.equal(results.length, 2) 298 | 299 | db = jitdb(raf, path.join(dir, 'indexes' + name)) 300 | db.onReady(() => { 301 | addMsg(state.queue[4].value, raf, (err, msg) => { 302 | db.all( 303 | voteQuery, 304 | 0, 305 | false, 306 | false, 307 | 'declared', 308 | (err, results) => { 309 | t.equal(results.length, 3) 310 | t.equal(results[0].value.content.type, 'vote') 311 | t.equal(results[1].value.content.type, 'vote') 312 | t.equal(results[2].value.content.type, 'vote') 313 | t.end() 314 | } 315 | ) 316 | }) 317 | }) 318 | }) 319 | }) 320 | }) 321 | }) 322 | }) 323 | }) 324 | }) 325 | }) 326 | 327 | prepareAndRunTest('Prefix equal unknown value', dir, (t, db, raf) => { 328 | const msg1 = { type: 'post', text: 'First', channel: 'foo' } 329 | const msg2 = { type: 'contact', text: 'Second' } 330 | const msg3 = { type: 'post', text: 'Third' } 331 | 332 | let state = validate.initial() 333 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 334 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 335 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 336 | 337 | const authorQuery = { 338 | type: 'EQUAL', 339 | data: { 340 | seek: helpers.seekAuthor, 341 | value: helpers.toBipf('abc'), 342 | indexType: 'author', 343 | indexName: 'value_author_abc', 344 | prefix: 32, 345 | }, 346 | } 347 | 348 | addMsg(state.queue[0].value, raf, (err, msg) => { 349 | addMsg(state.queue[1].value, raf, (err, msg) => { 350 | addMsg(state.queue[2].value, raf, (err, msg) => { 351 | db.all(authorQuery, 0, false, false, 'declared', (err, results) => { 352 | t.equal(results.length, 0) 353 | t.end() 354 | }) 355 | }) 356 | }) 357 | }) 358 | }) 359 | 360 | prepareAndRunTest('Prefix map equal', dir, (t, db, raf) => { 361 | const msg1 = { type: 'post', text: 'Testing!' } 362 | const msg2 = { type: 'contact', text: 'Testing!' } 363 | const msg3 = { type: 'post', text: 'Testing 2!' } 364 | 365 | let state = validate.initial() 366 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 367 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 368 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 369 | 370 | const typeQuery = { 371 | type: 'EQUAL', 372 | data: { 373 | seek: helpers.seekType, 374 | value: helpers.toBipf('post'), 375 | indexType: 'type', 376 | indexName: 'value_content_type_post', 377 | useMap: true, 378 | prefix: 32, 379 | }, 380 | } 381 | 382 | addMsg(state.queue[0].value, raf, (err, msg) => { 383 | addMsg(state.queue[1].value, raf, (err, msg) => { 384 | addMsg(state.queue[2].value, raf, (err, msg) => { 385 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 386 | t.equal(results.length, 2) 387 | t.equal(results[0].value.content.type, 'post') 388 | t.equal(results[1].value.content.type, 'post') 389 | t.end() 390 | }) 391 | }) 392 | }) 393 | }) 394 | }) 395 | 396 | prepareAndRunTest('Prefix offset', dir, (t, db, raf) => { 397 | const msg1 = { type: 'post', text: 'Testing!' } 398 | const msg2 = { type: 'contact', text: 'Testing!' } 399 | const msg3 = { type: 'post', text: 'Testing 2!' } 400 | 401 | let state = validate.initial() 402 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 403 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 404 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 405 | 406 | addMsg(state.queue[0].value, raf, (err, msg) => { 407 | addMsg(state.queue[1].value, raf, (err, msg) => { 408 | addMsg(state.queue[2].value, raf, (err, msg) => { 409 | const keyQuery = { 410 | type: 'EQUAL', 411 | data: { 412 | seek: helpers.seekKey, 413 | value: helpers.toBipf(msg.key), 414 | indexType: 'key', 415 | indexName: 'value_key_' + msg.key, 416 | useMap: true, 417 | prefix: 32, 418 | prefixOffset: 1, 419 | }, 420 | } 421 | 422 | db.all(keyQuery, 0, false, false, 'declared', (err, results) => { 423 | t.equal(results.length, 1) 424 | t.equal(results[0].value.content.text, 'Testing 2!') 425 | t.end() 426 | }) 427 | }) 428 | }) 429 | }) 430 | }) 431 | 432 | prepareAndRunTest('Prefix offset 1 on empty', dir, (t, db, raf) => { 433 | const msg1 = { type: 'post', root: 'test', text: 'Testing!' } 434 | 435 | let state = validate.initial() 436 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 437 | 438 | addMsg(state.queue[0].value, raf, (err, msg) => { 439 | const rootQuery = { 440 | type: 'EQUAL', 441 | data: { 442 | seek: helpers.seekRoot, 443 | value: helpers.toBipf('test'), 444 | indexType: 'root', 445 | indexName: 'value_content_root', 446 | useMap: true, 447 | prefix: 32, 448 | prefixOffset: 1, 449 | }, 450 | } 451 | 452 | db.all(rootQuery, 0, false, false, 'declared', (err, results) => { 453 | t.equal(results.length, 1) 454 | t.equal(results[0].value.content.text, 'Testing!') 455 | t.end() 456 | }) 457 | }) 458 | }) 459 | 460 | prepareAndRunTest('Prefix delete', dir, (t, db, raf) => { 461 | const msg1 = { type: 'post', text: 'Testing!' } 462 | const msg2 = { type: 'contact', text: 'Contact!' } 463 | const msg3 = { type: 'post', text: 'Testing 2!' } 464 | 465 | let state = validate.initial() 466 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 467 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 468 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 469 | 470 | const typeQuery = { 471 | type: 'EQUAL', 472 | data: { 473 | seek: helpers.seekType, 474 | value: helpers.toBipf('post'), 475 | indexType: 'type', 476 | indexName: 'value_content_type_post', 477 | useMap: true, 478 | prefix: 32, 479 | }, 480 | } 481 | const authorQuery = { 482 | type: 'EQUAL', 483 | data: { 484 | seek: helpers.seekAuthor, 485 | value: helpers.toBipf(keys.id), 486 | indexType: 'author', 487 | indexName: 'value_author', 488 | prefix: 32, 489 | }, 490 | } 491 | 492 | addMsg(state.queue[0].value, raf, (err, msg, offset1) => { 493 | addMsg(state.queue[1].value, raf, (err, msg) => { 494 | addMsg(state.queue[2].value, raf, (err, msg) => { 495 | db.all( 496 | { 497 | type: 'AND', 498 | data: [typeQuery, authorQuery], 499 | }, 500 | 0, 501 | false, 502 | false, 503 | 'declared', 504 | (err, results) => { 505 | t.equal(results.length, 2) 506 | t.equal(results[0].value.content.type, 'post') 507 | t.equal(results[1].value.content.type, 'post') 508 | 509 | raf.del(offset1, () => { 510 | db.paginate( 511 | { 512 | type: 'AND', 513 | data: [typeQuery, authorQuery], 514 | }, 515 | 0, 516 | 1, 517 | false, 518 | false, 519 | 'declared', 520 | null, 521 | (err, answer) => { 522 | t.equal(answer.results.length, 1) 523 | t.equal(answer.results[0].value.content.text, 'Testing 2!') 524 | 525 | t.end() 526 | } 527 | ) 528 | }) 529 | } 530 | ) 531 | }) 532 | }) 533 | }) 534 | }) 535 | -------------------------------------------------------------------------------- /test/reindex.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const test = require('tape') 6 | const validate = require('ssb-validate') 7 | const ssbKeys = require('ssb-keys') 8 | const path = require('path') 9 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 10 | const rimraf = require('rimraf') 11 | const mkdirp = require('mkdirp') 12 | const bipf = require('bipf') 13 | const { safeFilename } = require('../files') 14 | const { readFile, writeFile } = require('atomic-file-rw') 15 | 16 | const dir = '/tmp/jitdb-reindex' 17 | rimraf.sync(dir) 18 | mkdirp.sync(dir) 19 | 20 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 21 | 22 | function filterStream(raf, filterOffsets) { 23 | const originalStream = raf.stream 24 | raf.stream = function (opts) { 25 | const stream = originalStream(opts) 26 | const originalPipe = stream.pipe.bind(stream) 27 | stream.pipe = function (o) { 28 | const originalWrite = o.write 29 | o.write = function (record) { 30 | if (filterOffsets.includes(record.offset)) { 31 | const v = bipf.decode(record.value, 0) 32 | v.value.content = 'secret' 33 | const buf = Buffer.alloc(bipf.encodingLength(v)) 34 | bipf.encode(v, buf, 0) 35 | record.value = buf 36 | } 37 | originalWrite(record) 38 | } 39 | return originalPipe(o) 40 | } 41 | return stream 42 | } 43 | 44 | function removeFilter() { 45 | raf.stream = originalStream 46 | } 47 | 48 | return removeFilter 49 | } 50 | 51 | function addThreeMessages(raf, cb) { 52 | const msg1 = { type: 'post', text: 'Testing!' } 53 | const msg2 = { type: 'contact', text: 'Testing!' } 54 | const msg3 = { type: 'post', text: 'Testing 2!' } 55 | 56 | let state = validate.initial() 57 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 58 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 59 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 60 | 61 | addMsg(state.queue[0].value, raf, (err, msg) => { 62 | addMsg(state.queue[1].value, raf, (err, msg) => { 63 | addMsg(state.queue[2].value, raf, (err, msg) => { 64 | cb() 65 | }) 66 | }) 67 | }) 68 | } 69 | 70 | prepareAndRunTest('reindex seq offset', dir, (t, db, raf) => { 71 | const typeQuery = { 72 | type: 'EQUAL', 73 | data: { 74 | seek: helpers.seekType, 75 | value: helpers.toBipf('post'), 76 | indexType: 'type', 77 | indexName: 'type_post', 78 | }, 79 | } 80 | 81 | addThreeMessages(raf, () => { 82 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 83 | t.equal(results.length, 2) 84 | t.equal(results[0].value.content.type, 'post') 85 | t.equal(results[1].value.content.type, 'post') 86 | 87 | db.reindex(0, () => { 88 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 89 | t.equal(results.length, 2) 90 | 91 | const secondMsgOffset = 352 92 | db.reindex(secondMsgOffset, () => { 93 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 94 | t.equal(results.length, 2) 95 | 96 | t.end() 97 | }) 98 | }) 99 | }) 100 | }) 101 | }) 102 | }) 103 | }) 104 | 105 | prepareAndRunTest('reindex bitset', dir, (t, db, raf) => { 106 | const removeFilter = filterStream(raf, [0]) 107 | 108 | const typeQuery = { 109 | type: 'EQUAL', 110 | data: { 111 | seek: helpers.seekType, 112 | value: helpers.toBipf('post'), 113 | indexType: 'type', 114 | indexName: 'type_post', 115 | }, 116 | } 117 | 118 | addThreeMessages(raf, () => { 119 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 120 | t.equal(results.length, 1) 121 | t.equal(results[0].value.content.type, 'post') 122 | 123 | db.reindex(0, () => { 124 | removeFilter() 125 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 126 | t.equal(results.length, 2) 127 | t.end() 128 | }) 129 | }) 130 | }) 131 | }) 132 | }) 133 | 134 | prepareAndRunTest('reindex prefix', dir, (t, db, raf) => { 135 | const removeFilter = filterStream(raf, [0]) 136 | 137 | const typeQuery = { 138 | type: 'EQUAL', 139 | data: { 140 | seek: helpers.seekType, 141 | value: helpers.toBipf('post'), 142 | indexType: 'type', 143 | indexName: 'type_post_prefix', 144 | prefix: 32, 145 | prefixOffset: 0, 146 | }, 147 | } 148 | 149 | addThreeMessages(raf, () => { 150 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 151 | t.equal(results.length, 1) 152 | t.equal(results[0].value.content.type, 'post') 153 | 154 | db.reindex(0, () => { 155 | removeFilter() 156 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 157 | t.equal(results.length, 2) 158 | t.end() 159 | }) 160 | }) 161 | }) 162 | }) 163 | }) 164 | 165 | prepareAndRunTest('reindex prefix map', dir, (t, db, raf) => { 166 | const removeFilter = filterStream(raf, [0]) 167 | 168 | const typeQuery = { 169 | type: 'EQUAL', 170 | data: { 171 | seek: helpers.seekType, 172 | value: helpers.toBipf('post'), 173 | indexType: 'type', 174 | indexName: 'type_post_prefix', 175 | prefix: 32, 176 | prefixOffset: 0, 177 | useMap: true, 178 | }, 179 | } 180 | 181 | addThreeMessages(raf, () => { 182 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 183 | t.equal(results.length, 1) 184 | t.equal(results[0].value.content.type, 'post') 185 | 186 | db.reindex(0, () => { 187 | removeFilter() 188 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 189 | t.equal(results.length, 2) 190 | 191 | const secondMsgOffset = 352 192 | db.reindex(secondMsgOffset, () => { 193 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 194 | t.equal(results.length, 2) 195 | t.end() 196 | }) 197 | }) 198 | }) 199 | }) 200 | }) 201 | }) 202 | }) 203 | -------------------------------------------------------------------------------- /test/save-load.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const test = require('tape') 6 | const path = require('path') 7 | const TypedFastBitSet = require('typedfastbitset') 8 | const rimraf = require('rimraf') 9 | const mkdirp = require('mkdirp') 10 | const { 11 | saveTypedArrayFile, 12 | loadTypedArrayFile, 13 | savePrefixMapFile, 14 | loadPrefixMapFile, 15 | saveBitsetFile, 16 | loadBitsetFile, 17 | } = require('../files') 18 | const { readFile, writeFile } = require('atomic-file-rw') 19 | const oldCRC32 = require('hash-wasm').crc32 20 | const newCRC32 = require('crc/lib/crc32') 21 | 22 | const dir = '/tmp/jitdb-save-load' 23 | rimraf.sync(dir) 24 | mkdirp.sync(dir) 25 | 26 | test('save and load bitsets', (t) => { 27 | const idxDir = path.join(dir, 'test-bitsets') 28 | mkdirp.sync(idxDir) 29 | const filename = path.join(idxDir, 'test.index') 30 | 31 | var bitset = new TypedFastBitSet() 32 | for (var i = 0; i < 10; i += 2) bitset.add(i) 33 | 34 | saveBitsetFile(filename, 1, 123, bitset, (err) => { 35 | loadBitsetFile(filename, (err, index) => { 36 | t.error(err, 'no error') 37 | t.equal(index.version, 1) 38 | let loadedBitset = index.bitset 39 | t.deepEqual(bitset.array(), loadedBitset.array()) 40 | loadedBitset.add(10) 41 | 42 | saveBitsetFile(filename, 1, 1234, loadedBitset, (err) => { 43 | loadBitsetFile(filename, (err, index) => { 44 | t.equal(index.version, 1) 45 | t.error(err, 'no error') 46 | let loadedBitset2 = index.bitset 47 | t.deepEqual(loadedBitset.array(), loadedBitset2.array()) 48 | t.end() 49 | }) 50 | }) 51 | }) 52 | }) 53 | }) 54 | 55 | test('save and load TypedArray for seq', (t) => { 56 | const idxDir = path.join(dir, 'indexesSaveLoadSeq') 57 | mkdirp.sync(idxDir) 58 | const filename = path.join(idxDir, 'test.index') 59 | 60 | var tarr = new Uint32Array(16 * 1000) 61 | for (var i = 0; i < 10; i += 1) tarr[i] = i 62 | 63 | saveTypedArrayFile(filename, 1, 123, 10, tarr, (err) => { 64 | loadTypedArrayFile(filename, Uint32Array, (err, loadedIdx) => { 65 | t.equal(loadedIdx.tarr.length, 10 * 1.1, 'file trimmed') 66 | 67 | for (var i = 0; i < 10; i += 1) t.equal(tarr[i], loadedIdx.tarr[i]) 68 | 69 | loadedIdx.tarr[10] = 10 70 | 71 | saveTypedArrayFile(filename, 1, 1234, 11, loadedIdx.tarr, (err) => { 72 | loadTypedArrayFile(filename, Uint32Array, (err, loadedIdx2) => { 73 | for (var i = 0; i < 11; i += 1) 74 | t.equal(loadedIdx.tarr[i], loadedIdx2.tarr[i]) 75 | t.end() 76 | }) 77 | }) 78 | }) 79 | }) 80 | }) 81 | 82 | test('save and load TypedArray for timestamp', (t) => { 83 | const idxDir = path.join(dir, 'indexesSaveLoadTimestamp') 84 | mkdirp.sync(idxDir) 85 | const filename = path.join(idxDir, 'test.index') 86 | 87 | var tarr = new Float64Array(16 * 1000) 88 | for (var i = 0; i < 10; i += 1) tarr[i] = i * 1000000 89 | 90 | saveTypedArrayFile(filename, 1, 123, 10, tarr, (err) => { 91 | loadTypedArrayFile(filename, Float64Array, (err, loadedIdx) => { 92 | t.error(err, 'no error') 93 | for (var i = 0; i < 10; i += 1) t.equal(tarr[i], loadedIdx.tarr[i]) 94 | 95 | loadedIdx.tarr[10] = 10 * 1000000 96 | 97 | saveTypedArrayFile(filename, 1, 1234, 11, loadedIdx.tarr, (err) => { 98 | loadTypedArrayFile(filename, Float64Array, (err, loadedIdx2) => { 99 | for (var i = 0; i < 11; i += 1) 100 | t.equal(loadedIdx.tarr[i], loadedIdx2.tarr[i]) 101 | t.end() 102 | }) 103 | }) 104 | }) 105 | }) 106 | }) 107 | 108 | test('save and load prefix map', (t) => { 109 | const idxDir = path.join(dir, 'indexesSaveLoadPrefix') 110 | mkdirp.sync(idxDir) 111 | const filename = path.join(idxDir, 'test.index') 112 | 113 | var map = { 1: [1, 2, 3] } 114 | 115 | savePrefixMapFile(filename, 1, 123, 10, map, (err) => { 116 | loadPrefixMapFile(filename, (err, loadedIdx) => { 117 | t.error(err, 'no error') 118 | t.deepEqual(map, loadedIdx.map) 119 | 120 | map[2] = [1, 2] 121 | 122 | savePrefixMapFile(filename, 1, 1234, 11, map, (err) => { 123 | loadPrefixMapFile(filename, (err, loadedIdx2) => { 124 | t.deepEqual(map, loadedIdx2.map) 125 | t.end() 126 | }) 127 | }) 128 | }) 129 | }) 130 | }) 131 | 132 | test('load non-existing file', (t) => { 133 | const filename = path.join('/IDontExist', 'test.index') 134 | 135 | loadBitsetFile(filename, (err, index) => { 136 | t.equal(err.code, 'ENOENT') 137 | t.end() 138 | }) 139 | }) 140 | 141 | test('load corrupted header', (t) => { 142 | const filename = path.join(dir, 'corrupted-header.index') 143 | const data = new Uint8Array([0, 0, 0, 0, 0, 0]) 144 | writeFile(filename, data, (err) => { 145 | loadTypedArrayFile(filename, Uint32Array, (err, index) => { 146 | t.equal(err.message, 'file too short') 147 | t.end() 148 | }) 149 | }) 150 | }) 151 | 152 | test('save and load corrupt bitset', (t) => { 153 | const idxDir = path.join(dir, 'test-bitset-corrupt') 154 | mkdirp.sync(idxDir) 155 | const filename = path.join(idxDir, 'test.index') 156 | 157 | var bitset = new TypedFastBitSet() 158 | for (var i = 0; i < 10; i += 2) bitset.add(i) 159 | 160 | saveBitsetFile(filename, 1, 123, bitset, (err) => { 161 | readFile(filename, (err, b) => { 162 | b.writeUInt32LE(123456, 4 * 4) 163 | writeFile(filename, b, (err) => { 164 | loadBitsetFile(filename, (err, index) => { 165 | t.equal(err.message, 'crc check failed') 166 | t.end() 167 | }) 168 | }) 169 | }) 170 | }) 171 | }) 172 | 173 | test('crc32 calculation is stable across crc implementations', (t) => { 174 | const idxDir = path.join(dir, 'test-crc32-impls') 175 | mkdirp.sync(idxDir) 176 | const filename = path.join(idxDir, 'test.index') 177 | 178 | var bitset = new TypedFastBitSet() 179 | for (var i = 0; i < 10; i += 2) bitset.add(i) 180 | 181 | saveBitsetFile(filename, 1, 123, bitset, (err) => { 182 | t.error(err, 'no err') 183 | readFile(filename, (err, buf) => { 184 | t.error(err, 'no err') 185 | oldCRC32(buf).then((hex) => { 186 | const oldCode = parseInt(hex, 16) 187 | const newCode = newCRC32(buf) 188 | t.equals(newCode, oldCode, 'same crc code') 189 | t.end() 190 | }) 191 | }) 192 | }) 193 | }) 194 | -------------------------------------------------------------------------------- /test/seq-index-not-uptodate.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const test = require('tape') 6 | const validate = require('ssb-validate') 7 | const ssbKeys = require('ssb-keys') 8 | const path = require('path') 9 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 10 | const rimraf = require('rimraf') 11 | const mkdirp = require('mkdirp') 12 | const { safeFilename } = require('../files') 13 | 14 | const dir = '/tmp/jitdb-query' 15 | rimraf.sync(dir) 16 | mkdirp.sync(dir) 17 | 18 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 19 | 20 | prepareAndRunTest('Ensure seq index is updated always', dir, (t, db, raf) => { 21 | const msg1 = { type: 'post', text: 'Testing!' } 22 | const msg2 = { type: 'contact', text: 'Testing!' } 23 | const msg3 = { type: 'post', text: 'Testing 2!' } 24 | 25 | let state = validate.initial() 26 | state = validate.appendNew(state, null, keys, msg1, Date.now()) 27 | state = validate.appendNew(state, null, keys, msg2, Date.now() + 1) 28 | state = validate.appendNew(state, null, keys, msg3, Date.now() + 2) 29 | 30 | const typeQuery = { 31 | type: 'EQUAL', 32 | data: { 33 | seek: helpers.seekType, 34 | value: helpers.toBipf('post'), 35 | indexType: 'type', 36 | indexName: 'type_post', 37 | }, 38 | } 39 | 40 | addMsg(state.queue[0].value, raf, (err, msg) => { 41 | addMsg(state.queue[1].value, raf, (err, msg) => { 42 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 43 | t.equal(results.length, 1) 44 | t.equal(results[0].value.content.type, 'post') 45 | 46 | db.indexes.set('seq', { 47 | offset: -1, 48 | count: 0, 49 | tarr: new Uint32Array(16 * 1000), 50 | version: 1, 51 | }) 52 | 53 | addMsg(state.queue[2].value, raf, (err, msg) => { 54 | db.all(typeQuery, 0, false, false, 'declared', (err, results) => { 55 | t.equal(results.length, 2) 56 | t.equal(results[0].value.content.type, 'post') 57 | t.equal(results[1].value.content.type, 'post') 58 | t.end() 59 | }) 60 | }) 61 | }) 62 | }) 63 | }) 64 | }) 65 | -------------------------------------------------------------------------------- /test/slow-save.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2021 Anders Rune Jensen 2 | // 3 | // SPDX-License-Identifier: Unlicense 4 | 5 | const fs = require('fs') 6 | const validate = require('ssb-validate') 7 | const ssbKeys = require('ssb-keys') 8 | const path = require('path') 9 | const push = require('push-stream') 10 | const { prepareAndRunTest, addMsg, helpers } = require('./common')() 11 | const { loadTypedArrayFile } = require('../files') 12 | const rimraf = require('rimraf') 13 | const mkdirp = require('mkdirp') 14 | 15 | const dir = '/tmp/jitdb-slow-save' 16 | rimraf.sync(dir) 17 | mkdirp.sync(dir) 18 | 19 | var keys = ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) 20 | 21 | prepareAndRunTest('wip-index-save', dir, (t, db, raf) => { 22 | if (!process.env.CI) { 23 | // This test takes at least 1min and is heavy, let's not run it locally 24 | t.pass('skip this test because we are not in CI') 25 | t.end() 26 | return 27 | } 28 | 29 | t.timeoutAfter(500e3) 30 | let post = { type: 'post', text: 'Testing' } 31 | // with our simulated slow log, each msg takes 1.2ms to index 32 | // so we need at least 50k msgs 33 | const TOTAL = 90000 34 | 35 | let state = validate.initial() 36 | for (var i = 0; i < TOTAL; ++i) { 37 | post.text = 'Testing ' + i 38 | state = validate.appendNew(state, null, keys, post, Date.now() + i) 39 | } 40 | t.pass('generated ' + TOTAL + ' msgs') 41 | 42 | const typeQuery = { 43 | type: 'EQUAL', 44 | data: { 45 | seek: helpers.seekType, 46 | value: helpers.toBipf('post'), 47 | indexType: 'type', 48 | indexName: 'type_post', 49 | }, 50 | } 51 | 52 | const indexPath = path.join( 53 | dir, 54 | 'indexes' + 'wip-index-save', 55 | 'type_post.index' 56 | ) 57 | 58 | const seqIndexPath = path.join(dir, 'indexes' + 'wip-index-save', 'seq.index') 59 | 60 | push( 61 | push.values(state.queue), 62 | push.asyncMap((m, cb) => { 63 | addMsg(m.value, raf, cb) 64 | }), 65 | push.collect((err1, results1) => { 66 | t.error(err1, 'posted ' + TOTAL + ' msgs with no error') 67 | t.equal(results1.length, TOTAL) 68 | 69 | // Run some empty query to update the core indexes 70 | db.all({}, 0, false, false, 'declared', (err2, results2) => { 71 | t.error(err2, 'indexed core with ' + TOTAL + ' msgs with no error') 72 | t.equal(results2.length, TOTAL) 73 | 74 | let savedAfter1min = false 75 | 76 | // Hack log.stream to make it run slower 77 | const originalStream = raf.stream 78 | raf.stream = function (opts) { 79 | const s = originalStream(opts) 80 | const originalAbort = s.abort 81 | s.abort = function (...args) { 82 | setTimeout(() => { 83 | originalAbort.apply(s, args) 84 | }, 30e3) 85 | } 86 | const originalPipe = s.pipe.bind(s) 87 | s.pipe = function pipe(o) { 88 | let originalWrite = o.write 89 | o.write = (record) => { 90 | originalWrite(record) 91 | if (s.sink.paused) t.fail('log.stream didnt respect paused') 92 | if (!savedAfter1min) { 93 | s.sink.paused = true 94 | setTimeout(() => { 95 | s.sink.paused = false 96 | s.resume() 97 | }, 1) 98 | } 99 | } 100 | return originalPipe(o) 101 | } 102 | return s 103 | } 104 | 105 | setTimeout(() => { 106 | t.equal(fs.existsSync(indexPath), false, 'type_post.index isnt saved') 107 | }, 55e3) 108 | 109 | setTimeout(() => { 110 | t.equal(fs.existsSync(indexPath), true, 'type_post.index is saved') 111 | loadTypedArrayFile(seqIndexPath, Uint32Array, (err, loadedIdx) => { 112 | t.error(err, 'loaded seq.index') 113 | t.equal(loadedIdx.count, TOTAL, 'seq.index count is correct') 114 | t.equal(loadedIdx.offset, 37095426, 'seq.index offset is correct') 115 | savedAfter1min = true 116 | }) 117 | }, 65e3) 118 | 119 | // Run an actual query to check if it saves every 1min 120 | db.all(typeQuery, 0, false, false, 'declared', (err3, results3) => { 121 | t.error(err3, 'indexed ' + TOTAL + ' msgs no error') 122 | t.equal(results3.length, TOTAL) 123 | 124 | t.true(savedAfter1min, 'saved after 1 min') 125 | rimraf.sync(dir) // this folder is quite large, lets save space 126 | t.end() 127 | }) 128 | }) 129 | }) 130 | ) 131 | }) 132 | --------------------------------------------------------------------------------