├── .clang-format ├── .clang-tidy ├── .editorconfig ├── .eslintignore ├── .eslintrc.json ├── .github └── workflows │ ├── ci.yml │ ├── npm-publish.yml │ └── publish.yml ├── .gitignore ├── .gitmodules ├── .husky ├── .gitignore ├── pre-commit └── pre-push ├── .prettierignore ├── .prettierrc ├── .vscode ├── c_cpp_properties.json ├── extensions.json └── settings.json ├── LICENSE ├── README.md ├── RoaringBitmap32.d.ts ├── RoaringBitmap32.js ├── RoaringBitmap32Iterator.d.ts ├── RoaringBitmap32Iterator.js ├── RoaringBitmap32ReverseIterator.d.ts ├── RoaringBitmap32ReverseIterator.js ├── benchmarks ├── .eslintrc.json ├── add.bench.js ├── intersection-inplace.bench.js ├── intersection-new.bench.js ├── intersection-size.bench.js ├── iterator.bench.js ├── union-inplace.bench.js ├── union-new.bench.js └── union-size.bench.js ├── binding.gyp ├── index.d.ts ├── index.js ├── jsconfig.json ├── lint-staged.config.js ├── node-pre-gyp.js ├── package-lock.json ├── package.json ├── roaring-node.cpp ├── scripts ├── .eslintrc.json ├── benchmarks.js ├── benchmarks │ ├── bench.js │ ├── benchReport.js │ ├── promiseMap.js │ └── spinner.js ├── build.js ├── lib │ ├── unity.js │ └── utils.js ├── node-pre-gyp-publish.js ├── prebuild-local.js ├── precommit.js ├── prepush.js ├── rebuild.js ├── system-info.js ├── test-memory-leaks.js ├── test.js └── update-roaring.sh ├── src └── cpp │ ├── RoaringBitmap32-main.h │ ├── RoaringBitmap32-ops.h │ ├── RoaringBitmap32-ranges.h │ ├── RoaringBitmap32-serialization.h │ ├── RoaringBitmap32-static-ops.h │ ├── RoaringBitmap32.h │ ├── RoaringBitmap32BufferedIterator.h │ ├── WorkerError.h │ ├── addon-data.h │ ├── addon-strings.h │ ├── aligned-buffers.h │ ├── async-workers.h │ ├── croaring.cpp │ ├── croaring.h │ ├── includes.h │ ├── main.cpp │ ├── memory.h │ ├── mmap.h │ ├── object-wrap.h │ ├── serialization-csv.h │ ├── serialization-format.h │ ├── serialization.h │ └── v8utils.h ├── test ├── RoaringBitmap32 │ ├── RoaringBitmap32.basic.test.ts │ ├── RoaringBitmap32.comparisons.test.ts │ ├── RoaringBitmap32.deserializeParallelAsync.test.ts │ ├── RoaringBitmap32.fromAsync.test.ts │ ├── RoaringBitmap32.frozen.test.ts │ ├── RoaringBitmap32.import.test.ts │ ├── RoaringBitmap32.operations.test.ts │ ├── RoaringBitmap32.ranges.test.ts │ ├── RoaringBitmap32.serialization-file.test.ts │ ├── RoaringBitmap32.serialization.test.ts │ ├── RoaringBitmap32.set-methods.test.ts │ ├── RoaringBitmap32.static.test.ts │ ├── RoaringBitmap32.worker-threads.test.ts │ ├── RoaringBitmap33.deserializeAsync.test.ts │ ├── data │ │ └── serialized.json │ └── worker-thread-test.js ├── RoaringBitmap32Iterator │ ├── RoaringBitmap32Iterator.import.test.ts │ └── RoaringBitmap32Iterator.test.ts ├── RoaringBitmap32ReverseIterator │ ├── RoaringBitmap32ReverseIterator.import.test.ts │ └── RoaringBitmap32ReverseIterator.test.ts ├── aligned-buffers.test.ts └── roaring.test.ts ├── tsconfig.json └── tsconfig.test.json /.clang-format: -------------------------------------------------------------------------------- 1 | DisableFormat: false 2 | BasedOnStyle: Chromium 3 | IndentWidth: 2 4 | UseTab: Never 5 | ColumnLimit: 125 6 | NamespaceIndentation: All 7 | AlignAfterOpenBracket: AlwaysBreak 8 | AlignConsecutiveAssignments: false 9 | AlignConsecutiveBitFields: false 10 | AlignConsecutiveDeclarations: false 11 | AlignConsecutiveMacros: false 12 | AlignTrailingComments: false 13 | AlignEscapedNewlines: Left 14 | AlignOperands: false 15 | BitFieldColonSpacing: After 16 | AllowShortEnumsOnASingleLine: true 17 | AllowShortBlocksOnASingleLine: false 18 | AllowShortCaseLabelsOnASingleLine: true 19 | AlwaysBreakBeforeMultilineStrings: true 20 | AllowShortFunctionsOnASingleLine: true 21 | AllowShortIfStatementsOnASingleLine: true 22 | AllowShortLambdasOnASingleLine: false 23 | AllowShortLoopsOnASingleLine: false 24 | AlwaysBreakAfterDefinitionReturnType: false 25 | AllowAllArgumentsOnNextLine: true 26 | AllowAllConstructorInitializersOnNextLine: true 27 | AllowAllParametersOfDeclarationOnNextLine: true 28 | AlwaysBreakTemplateDeclarations: true 29 | BreakBeforeTernaryOperators: true 30 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 31 | ConstructorInitializerIndentWidth: 2 32 | ContinuationIndentWidth: 2 33 | LambdaBodyIndentation: Signature 34 | PointerAlignment: Middle 35 | BinPackParameters: false 36 | BinPackArguments: false 37 | MaxEmptyLinesToKeep: 1 38 | BreakBeforeBraces: Attach 39 | BreakBeforeConceptDeclarations: true 40 | BreakConstructorInitializers: AfterColon 41 | BreakInheritanceList: AfterColon 42 | IndentPPDirectives: AfterHash 43 | SortIncludes: false 44 | SortUsingDeclarations: true 45 | 46 | --- 47 | Language: JavaScript 48 | DisableFormat: true 49 | ColumnLimit: 120 50 | JavaScriptQuotes: Single 51 | -------------------------------------------------------------------------------- /.clang-tidy: -------------------------------------------------------------------------------- 1 | --- 2 | Checks: ' 3 | -*, 4 | bugprone-*, 5 | cert-*, 6 | clang-analyzer-*, 7 | hicpp-multiway-paths-covered, 8 | hicpp-signed-bitwise, 9 | misc-*, 10 | modernize-*, 11 | performance-*, 12 | portability-*, 13 | readability-*, 14 | cppcoreguidelines-*, 15 | modernize-avoid-c-arrays, 16 | google-explicit-constructor, 17 | -cert-dcl37-c, 18 | -cert-dcl16-c, 19 | -cert-dcl51-cpp, 20 | -cert-dcl58-cpp, 21 | -cert-err60-cpp, 22 | -modernize-deprecated-headers, 23 | -bugprone-reserved-identifier, 24 | -cppcoreguidelines-pro-bounds-array-to-pointer-decay, 25 | -readability-identifier-naming, 26 | -cppcoreguidelines-avoid-non-const-global-variables, 27 | -cppcoreguidelines-avoid-magic-numbers, 28 | -cppcoreguidelines-owning-memory, 29 | -readability-magic-numbers, 30 | -modernize-use-trailing-return-type, 31 | -readability-named-parameter, 32 | -readability-function-cognitive-complexity, 33 | -misc-non-private-member-variables-in-classes, 34 | -modernize-use-nodiscard, 35 | -modernize-avoid-c-arrays, 36 | -misc-no-recursion, 37 | -cppcoreguidelines-pro-type-reinterpret-cast, 38 | -cppcoreguidelines-pro-bounds-constant-array-index, 39 | -bugprone-easily-swappable-parameters, 40 | -cppcoreguidelines-pro-bounds-pointer-arithmetic, 41 | -misc-unused-using-decls, 42 | -cppcoreguidelines-non-private-member-variables-in-classes, 43 | -readability-braces-around-statements, 44 | -cppcoreguidelines-no-malloc, 45 | -cppcoreguidelines-pro-type-vararg, 46 | -cert-err58-cpp, 47 | -cppcoreguidelines-init-variables, 48 | -modernize-loop-convert, 49 | -cppcoreguidelines-macro-usage, 50 | -bugprone-implicit-widening-of-multiplication-result, 51 | -cert-oop54-cpp, 52 | -bugprone-suspicious-include, 53 | -cppcoreguidelines-avoid-c-arrays, 54 | -readability-isolate-declaration, 55 | -cppcoreguidelines-special-member-functions, 56 | -modernize-use-default-member-init, 57 | -modernize-use-equals-default, 58 | -cppcoreguidelines-pro-type-member-init, 59 | -bugprone-branch-clone, 60 | -misc-unused-parameters, 61 | -modernize-use-using' 62 | 63 | WarningsAsErrors: '' 64 | AnalyzeTemporaryDtors: false 65 | InheritParentConfig: false 66 | 67 | CheckOptions: 68 | - { key: hicpp-signed-bitwise.IgnorePositiveIntegerLiterals, value: true } 69 | - { key: readability-braces-around-statements.ShortStatementLines, value: 1 } 70 | - { key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor, value: true } 71 | - { key: cppcoreguidelines-special-member-functions.AllowMissingMoveFunctions, value: true } 72 | - { key: cppcoreguidelines-special-member-functions.AllowMissingMoveFunctionsWhenCopyIsDeleted, value: true } 73 | - { key: modernize-use-override.IgnoreDestructors, value: 1 } 74 | 75 | --- 76 | 77 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | 7 | [*.{js,ts}] 8 | charset = utf-8 9 | indent_style = space 10 | indent_size = 2 11 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | src/cpp/**/* 2 | docs/**/* 3 | 4 | # dotfiles 5 | 6 | !.*.js 7 | !.*._js 8 | !.*.jsx 9 | !.*.mjs 10 | !.*.cjs 11 | !.*.es 12 | !.*.es6 13 | !.*.assetgen 14 | !.*.ts 15 | !.*.tsx 16 | !.*.json 17 | !.eslintrc.js 18 | 19 | # Dependencies and vendor 20 | vendor 21 | typings 22 | node_modules 23 | bower_components 24 | jspm_packages 25 | package-lock.json 26 | /.pnp 27 | .pnp.js 28 | .serverless 29 | .webpack 30 | .next/ 31 | 32 | # Build outputs 33 | bundle.js 34 | *.bundle.js 35 | .serverless 36 | build/Release 37 | *.min.* 38 | *-min.* 39 | 40 | # Cache 41 | .git 42 | .eslintcache 43 | .tmp 44 | .temp 45 | .cache 46 | .fuse_hidden* 47 | .fusebox 48 | .dynamodb 49 | 50 | # Test coverage and snapshots 51 | lib-cov 52 | coverage 53 | .nyc_output 54 | __snapshots__ 55 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": ["@balsamic"], 3 | "plugins": ["node", "import"], 4 | "rules": { 5 | "global-require": 0 6 | }, 7 | "overrides": [ 8 | { 9 | "files": ["index.js"], 10 | "rules": { 11 | "node/no-unpublished-require": 0, 12 | "import/no-unresolved": 0 13 | } 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - master 9 | 10 | jobs: 11 | test: 12 | name: test 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/setup-node@v4 16 | with: 17 | node-version: "20.x" 18 | - uses: actions/checkout@v4 19 | - run: npm install --exact --no-audit --no-save 20 | env: 21 | ROARING_NODE_PRE_GYP: "false" 22 | - run: npx tsc --noEmit 23 | - run: npx eslint --no-error-on-unmatched-pattern --max-warnings=0 24 | - run: npx prettier --loglevel=warn --check . 25 | - run: node ./node-pre-gyp.js 26 | env: 27 | ROARING_NODE_PRE_GYP: "custom-rebuild" 28 | - run: node ./scripts/test.js 29 | - run: node --expose-gc ./scripts/test-memory-leaks.js 30 | 31 | ci: 32 | needs: test 33 | strategy: 34 | matrix: 35 | node-version: ["16.14.0", "18.1.0", "20.9.0", "21.1.0", "22.8.0"] 36 | os: [ubuntu-latest, macos-13, macos-latest, windows-2019] 37 | runs-on: ${{ matrix.os }} 38 | steps: 39 | - uses: actions/setup-node@v4 40 | with: 41 | node-version: ${{ matrix.node-version }} 42 | - uses: actions/checkout@v4 43 | - run: npm install --exact --no-audit --no-save 44 | env: 45 | ROARING_NODE_PRE_GYP: "false" 46 | - run: node ./node-pre-gyp.js 47 | env: 48 | ROARING_NODE_PRE_GYP: "custom-rebuild" 49 | - run: node ./scripts/test.js 50 | -------------------------------------------------------------------------------- /.github/workflows/npm-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to NPM 2 | 3 | # Only allows manual triggering 4 | on: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | npm-publish: 9 | name: npm-publish 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: read 13 | id-token: write 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | ref: publish 18 | - uses: actions/setup-node@v4 19 | with: 20 | node-version: "22.x" 21 | registry-url: "https://registry.npmjs.org" 22 | - run: npm install --exact --no-audit --no-save 23 | - run: npx tsc --noEmit 24 | - run: node ./scripts/test.js 25 | - run: node --expose-gc ./scripts/test-memory-leaks.js 26 | - run: npm publish --provenance --access public 27 | env: 28 | NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} 29 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - publish 8 | 9 | jobs: 10 | test: 11 | name: test 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/setup-node@v4 15 | with: 16 | node-version: "20.x" 17 | - uses: actions/checkout@v4 18 | - run: npm install --exact --no-audit --no-save 19 | env: 20 | ROARING_NODE_PRE_GYP: "false" 21 | - run: node ./node-pre-gyp.js 22 | env: 23 | ROARING_NODE_PRE_GYP: "custom-rebuild" 24 | - run: node ./scripts/test.js 25 | - run: node --expose-gc ./scripts/test-memory-leaks.js 26 | 27 | prebuild: 28 | needs: test 29 | strategy: 30 | matrix: 31 | node-version: ["16.14.0", "18.1.0", "20.9.0", "21.1.0", "22.8.0"] 32 | os: [ubuntu-20.04, windows-2019, macos-13, macos-latest] 33 | runs-on: ${{ matrix.os }} 34 | steps: 35 | - uses: actions/setup-node@v4 36 | with: 37 | node-version: ${{ matrix.node-version }} 38 | - uses: actions/checkout@v4 39 | - run: npm install --exact --no-audit --no-save 40 | env: 41 | ROARING_NODE_PRE_GYP: "false" 42 | - run: node ./node-pre-gyp.js 43 | env: 44 | ROARING_NODE_PRE_GYP: "custom-rebuild" 45 | - run: node ./scripts/test.js 46 | - name: Prebuild 47 | env: 48 | NODE_PRE_GYP_GITHUB_TOKEN: ${{ secrets.PREBUILD_GITHUB_TOKEN }} 49 | run: node ./scripts/node-pre-gyp-publish.js 50 | 51 | post-test: 52 | needs: [test, prebuild] 53 | strategy: 54 | matrix: 55 | os: [ubuntu-20.04, windows-2019, macos-13, macos-latest] 56 | runs-on: ${{ matrix.os }} 57 | steps: 58 | - uses: actions/setup-node@v4 59 | with: 60 | node-version: "18.16.0" 61 | - uses: actions/checkout@v4 62 | - run: npm install 63 | - run: node ./scripts/test.js 64 | 65 | docs: 66 | needs: [test, prebuild, post-test] 67 | runs-on: ubuntu-latest 68 | steps: 69 | - uses: actions/checkout@v2 70 | - uses: actions/setup-node@v4 71 | with: 72 | node-version: 18.x 73 | 74 | - name: npm install 75 | run: npm install --exact --no-audit --no-save 76 | env: 77 | ROARING_NODE_PRE_GYP: "false" 78 | 79 | - name: npm run doc 80 | run: npm run doc 81 | 82 | - name: Deploy documentation 🚀 83 | uses: JamesIves/github-pages-deploy-action@4.1.4 84 | with: 85 | branch: gh-pages 86 | folder: docs 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # build output 2 | 3 | *.o 4 | native/ 5 | build/ 6 | package/ 7 | docs/ 8 | .tmp/ 9 | .github-key 10 | 11 | temp 12 | 13 | # Logs 14 | logs 15 | *.log 16 | npm-debug.log* 17 | yarn-debug.log* 18 | yarn-error.log* 19 | 20 | # Runtime data 21 | pids 22 | *.pid 23 | *.seed 24 | *.pid.lock 25 | 26 | # Directory for instrumented libs generated by jscoverage/JSCover 27 | lib-cov 28 | 29 | # Coverage directory used by tools like istanbul 30 | coverage 31 | 32 | # nyc test coverage 33 | .nyc_output 34 | 35 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 36 | .grunt 37 | 38 | # Bower dependency directory (https://bower.io/) 39 | bower_components 40 | 41 | # node-waf configuration 42 | .lock-wscript 43 | 44 | # Compiled binary addons (http://nodejs.org/api/addons.html) 45 | build/Release 46 | 47 | # Dependency directories 48 | **/node_modules/ 49 | node_modules/ 50 | jspm_packages/ 51 | 52 | # Typescript v1 declaration files 53 | typings/ 54 | 55 | # Optional npm cache directory 56 | .npm 57 | 58 | # Optional eslint cache 59 | .eslintcache 60 | 61 | # Optional REPL history 62 | .node_repl_history 63 | 64 | # Output of 'npm pack' 65 | *.tgz 66 | 67 | # Yarn Integrity file 68 | .yarn-integrity 69 | 70 | # dotenv environment variables file 71 | .env 72 | 73 | 74 | 75 | # General 76 | .DS_Store 77 | .AppleDouble 78 | .LSOverride 79 | 80 | # Icon must end with two \r 81 | Icon 82 | 83 | 84 | # Thumbnails 85 | ._* 86 | 87 | # Files that might appear in the root of a volume 88 | .DocumentRevisions-V100 89 | .fseventsd 90 | .Spotlight-V100 91 | .TemporaryItems 92 | .Trashes 93 | .VolumeIcon.icns 94 | .com.apple.timemachine.donotpresent 95 | 96 | # Directories potentially created on remote AFP share 97 | .AppleDB 98 | .AppleDesktop 99 | Network Trash Folder 100 | Temporary Items 101 | .apdisk 102 | 103 | # Windows thumbnail cache files 104 | Thumbs.db 105 | ehthumbs.db 106 | ehthumbs_vista.db 107 | 108 | # Dump file 109 | *.stackdump 110 | 111 | # Folder config file 112 | [Dd]esktop.ini 113 | 114 | # Recycle Bin used on file shares 115 | $RECYCLE.BIN/ 116 | 117 | # Windows Installer files 118 | *.cab 119 | *.msi 120 | *.msix 121 | *.msm 122 | *.msp 123 | 124 | # Windows shortcuts 125 | *.lnk 126 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "submodules/CRoaring"] 2 | path = submodules/CRoaring 3 | url = https://github.com/RoaringBitmap/CRoaring.git 4 | -------------------------------------------------------------------------------- /.husky/.gitignore: -------------------------------------------------------------------------------- 1 | _ 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | node scripts/precommit.js 2 | -------------------------------------------------------------------------------- /.husky/pre-push: -------------------------------------------------------------------------------- 1 | node scripts/prepush.js 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | src/cpp/**/* 2 | docs/**/* 3 | submodules/**/* 4 | .clang-tidy 5 | 6 | # Dependencies and vendor 7 | vendor 8 | typings 9 | node_modules 10 | bower_components 11 | jspm_packages 12 | package-lock.json 13 | /.pnp 14 | .pnp.js 15 | .serverless 16 | .webpack 17 | .next/ 18 | 19 | # Build outputs 20 | bundle.js 21 | *.bundle.js 22 | .serverless 23 | build/Debug 24 | build/Release 25 | *.min.* 26 | *-min.* 27 | 28 | # Cache 29 | .tmp 30 | .temp 31 | .cache 32 | .fuse_hidden* 33 | .fusebox 34 | .dynamodb 35 | 36 | # Test coverage and snapshots 37 | lib-cov 38 | coverage 39 | .nyc_output 40 | __snapshots__ 41 | 42 | .vercel 43 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "always", 3 | "bracketSpacing": true, 4 | "endOfLine": "lf", 5 | "htmlWhitespaceSensitivity": "css", 6 | "jsxSingleQuote": false, 7 | "printWidth": 120, 8 | "semi": true, 9 | "singleQuote": false, 10 | "tabWidth": 2, 11 | "trailingComma": "all", 12 | "useTabs": false 13 | } 14 | -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Mac", 5 | "includePath": [ 6 | "${workspaceFolder}", 7 | "${workspaceFolder}/submodules/CRoaring/include", 8 | "/Users/sp/n/include/node", 9 | "/usr/local/include", 10 | "/usr/local/include/node", 11 | "${default}" 12 | ], 13 | "defines": [], 14 | "intelliSenseMode": "clang-x64", 15 | "browse": { 16 | "path": [ 17 | "${workspaceFolder}", 18 | "${workspaceFolder}/submodules/CRoaring/include", 19 | "/Users/sp/n/include/node", 20 | "/usr/local/include", 21 | "/usr/local/include/node", 22 | "${default}" 23 | ], 24 | "limitSymbolsToIncludedHeaders": true, 25 | "databaseFilename": "" 26 | }, 27 | "macFrameworkPath": ["/System/Library/Frameworks", "/Library/Frameworks"], 28 | "compilerPath": "/usr/bin/clang", 29 | "cStandard": "c11", 30 | "cppStandard": "c++17" 31 | } 32 | ], 33 | "version": 4 34 | } 35 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "esbenp.prettier-vscode", 5 | "pflannery.vscode-versionlens", 6 | "visualstudioexptteam.vscodeintellicode", 7 | "donjayamanne.githistory", 8 | "eamodio.gitlens", 9 | "ms-vscode.cpptools", 10 | "xaver.clang-format" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[json]": { 3 | "editor.defaultFormatter": "esbenp.prettier-vscode", 4 | "editor.formatOnSave": true, 5 | "editor.quickSuggestions": { 6 | "comments": true, 7 | "other": true, 8 | "strings": true 9 | } 10 | }, 11 | "[jsonc]": { 12 | "editor.defaultFormatter": "esbenp.prettier-vscode", 13 | "editor.formatOnSave": true, 14 | "editor.quickSuggestions": { 15 | "comments": true, 16 | "other": true, 17 | "strings": true 18 | } 19 | }, 20 | "[html]": { 21 | "editor.defaultFormatter": "esbenp.prettier-vscode", 22 | "editor.formatOnSave": true 23 | }, 24 | "[javascript]": { 25 | "editor.defaultFormatter": "esbenp.prettier-vscode", 26 | "editor.formatOnSave": true 27 | }, 28 | "[javascriptreact]": { 29 | "editor.defaultFormatter": "esbenp.prettier-vscode", 30 | "editor.formatOnSave": true 31 | }, 32 | "[typescript]": { 33 | "editor.defaultFormatter": "esbenp.prettier-vscode", 34 | "editor.formatOnSave": true 35 | }, 36 | "[typescriptreact]": { 37 | "editor.defaultFormatter": "esbenp.prettier-vscode", 38 | "editor.formatOnSave": true 39 | }, 40 | "editor.codeActionsOnSave": { 41 | "source.fixAll": "explicit", 42 | "source.fixAll.eslint": "explicit" 43 | }, 44 | "[c]": { 45 | "editor.defaultFormatter": "xaver.clang-format" 46 | }, 47 | "[cpp]": { 48 | "editor.defaultFormatter": "xaver.clang-format" 49 | }, 50 | "clang-tidy.blacklist": ["submodules"], 51 | "clang-tidy.fixOnSave": true, 52 | "clang-tidy.compilerArgs": [ 53 | "-std=c++14", 54 | "-I/usr/local/include", 55 | "-I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1", 56 | "-I/usr/local/include/node", 57 | "-I/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include" 58 | ], 59 | "editor.detectIndentation": false, 60 | "editor.formatOnSave": true, 61 | "editor.lineNumbers": "on", 62 | "editor.tabSize": 2, 63 | "eslint.codeActionsOnSave.mode": "all", 64 | "eslint.format.enable": true, 65 | "eslint.lintTask.enable": true, 66 | "eslint.validate": [], 67 | "javascript.format.semicolons": "insert", 68 | "typescript.format.semicolons": "insert", 69 | "typescript.tsdk": "node_modules/typescript/lib", 70 | "files.eol": "\n", 71 | "search.exclude": { 72 | "roaring-node.cpp": true, 73 | "**/node_modules": true, 74 | "**/bower_components": true, 75 | "**/*.code-search": true 76 | }, 77 | "cmake.configureOnOpen": false, 78 | "files.associations": { 79 | "__bit_reference": "cpp", 80 | "__node_handle": "cpp", 81 | "atomic": "cpp", 82 | "bitset": "cpp", 83 | "deque": "cpp", 84 | "__memory": "cpp", 85 | "limits": "cpp", 86 | "locale": "cpp", 87 | "optional": "cpp", 88 | "ratio": "cpp", 89 | "system_error": "cpp", 90 | "tuple": "cpp", 91 | "type_traits": "cpp", 92 | "vector": "cpp", 93 | "chrono": "cpp", 94 | "cstddef": "cpp", 95 | "filesystem": "cpp", 96 | "random": "cpp", 97 | "__locale": "cpp", 98 | "regex": "cpp", 99 | "string": "cpp", 100 | "string_view": "cpp" 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # roaring 2 | 3 | Official port of [Roaring Bitmaps](http://roaringbitmap.org) for NodeJS as a native addon. 4 | 5 | It is interoperable with other implementations via the [Roaring format](https://github.com/RoaringBitmap/RoaringFormatSpec/). 6 | It takes advantage of AVX2 or SSE4.2 instructions on 64 bit platforms that supports it. 7 | 8 | Roaring bitmaps are compressed bitmaps. They can be hundreds of times faster. 9 | 10 | For a precompiled binary of this package compatible with AWS Lambda NodeJS v8.10.0, use [roaring-aws](https://www.npmjs.com/package/roaring-aws). 11 | 12 | ## Branches 13 | 14 | Branch `publish` contains the latest published stable version. 15 | 16 | Branch `master` is the development branch that may contain code not yet published or ready for production. 17 | If you want to contribute and submit a pull request, use the master branch. 18 | 19 | ## Supported node versions 20 | 21 | Node 16.14+, 18+, 20+, 21, and 22 are currently supported. 22 | 23 | Node 8 and 10 support was dropped in release 2.0 24 | 25 | Node 12 and 14 support was dropped in release 2.3 26 | 27 | ## Worker thread support 28 | 29 | Directly transferring an instance without copy between worker threads is not currently supported, but you can create a frozen view on a SharedArrayBuffer using bufferAlignedAllocShared and pass it to the worker thread. 30 | 31 | ## Installation 32 | 33 | ```sh 34 | npm install --save roaring 35 | ``` 36 | 37 | ## References 38 | 39 | - This package - 40 | - Source code and build tools for this package - 41 | - Roaring Bitmaps - 42 | - Portable Roaring bitmaps in C - 43 | - Portable Roaring bitmaps in C (unity build) - https://github.com/lemire/CRoaringUnityBuild 44 | 45 | # Licenses 46 | 47 | - This package is provided as open source software using Apache License. 48 | - CRoaring is provided as open source software using Apache License. 49 | 50 | # API 51 | 52 | See the [roaring module documentation](https://salvatorepreviti.github.io/roaring-node/modules.html) 53 | 54 | See the [RoaringBitmap32 class documentation](https://salvatorepreviti.github.io/roaring-node/classes/RoaringBitmap32.html) 55 | 56 | # Code sample: 57 | 58 | ```javascript 59 | // npm install --save roaring 60 | // create this file as demo.js 61 | // type node demo.js 62 | 63 | const RoaringBitmap32 = require("roaring/RoaringBitmap32"); 64 | 65 | const bitmap1 = new RoaringBitmap32([1, 2, 3, 4, 5]); 66 | bitmap1.addMany([100, 1000]); 67 | console.log("bitmap1.toArray():", bitmap1.toArray()); 68 | 69 | const bitmap2 = new RoaringBitmap32([3, 4, 1000]); 70 | console.log("bitmap2.toArray():", bitmap2.toArray()); 71 | 72 | const bitmap3 = new RoaringBitmap32(); 73 | console.log("bitmap1.size:", bitmap1.size); 74 | console.log("bitmap3.has(3):", bitmap3.has(3)); 75 | bitmap3.add(3); 76 | console.log("bitmap3.has(3):", bitmap3.has(3)); 77 | 78 | bitmap3.add(111); 79 | bitmap3.add(544); 80 | bitmap3.orInPlace(bitmap1); 81 | bitmap1.runOptimize(); 82 | bitmap1.shrinkToFit(); 83 | console.log("contentToString:", bitmap3.contentToString()); 84 | 85 | console.log("bitmap3.toArray():", bitmap3.toArray()); 86 | console.log("bitmap3.maximum():", bitmap3.maximum()); 87 | console.log("bitmap3.rank(100):", bitmap3.rank(100)); 88 | 89 | const iterated = []; 90 | for (const value of bitmap3) { 91 | iterated.push(value); 92 | } 93 | console.log("iterated:", iterated); 94 | 95 | const serialized = bitmap3.serialize(false); 96 | console.log("serialized:", serialized.toString("base64")); 97 | console.log("deserialized:", RoaringBitmap32.deserialize(serialized, false).toArray()); 98 | ``` 99 | 100 | # Other 101 | 102 | Wanna play an open source game made by the author of this library? Try [Dante](https://github.com/SalvatorePreviti/js13k-2022) 103 | 104 | # Development, local building 105 | 106 | Clone the repository and install all the dependencies 107 | 108 | ``` 109 | git clone https://github.com/SalvatorePreviti/roaring-node.git 110 | 111 | cd roaring-node 112 | 113 | npm install 114 | ``` 115 | 116 | # To rebuild the roaring unity build updating to the latest version 117 | 118 | ``` 119 | ./scripts/update-roaring.sh 120 | 121 | npm run build 122 | 123 | ``` 124 | 125 | ### To run the unit test 126 | 127 | ``` 128 | npm test 129 | ``` 130 | 131 | ### To regenerate the API documentation 132 | 133 | ``` 134 | npm run doc 135 | ``` 136 | 137 | ### To run the performance benchmarks 138 | 139 | ```sh 140 | npm run benchmarks 141 | ``` 142 | 143 | It will produce a result similar to this one: 144 | 145 | ``` 146 | Platform : Darwin 17.6.0 x64 147 | CPU : Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz AVX2 148 | Cores : 4 physical - 8 logical 149 | Memory : 16.00 GB 150 | NodeJS : v10.5.0 - V8 v6.7.288.46-node.8 151 | 152 | * running 8 files... 153 | 154 | • suite intersection (in place) 155 | 65536 elements 156 | ✔ Set 186.82 ops/sec ±2.33% 66 runs -99.98% 157 | ✔ FastBitSet 100,341.63 ops/sec ±2.10% 85 runs -87.10% 158 | ✔ RoaringBitmap32 777,765.97 ops/sec ±2.14% 87 runs fastest 159 | ➔ Fastest is RoaringBitmap32 160 | 161 | • suite intersection (new) 162 | 1048576 elements 163 | ✔ Set 3.49 ops/sec ±3.82% 13 runs -99.89% 164 | ✔ FastBitSet 1,463.87 ops/sec ±1.13% 84 runs -55.54% 165 | ✔ RoaringBitmap32 3,292.51 ops/sec ±20.89% 44 runs fastest 166 | ➔ Fastest is RoaringBitmap32 167 | 168 | • suite add 169 | 65535 elements 170 | ✔ Set.add 438.37 ops/sec ±4.48% 68 runs -85.21% 171 | ✔ RoaringBitmap32.tryAdd 489.70 ops/sec ±3.19% 76 runs -83.48% 172 | ✔ RoaringBitmap32.add 528.09 ops/sec ±3.38% 76 runs -82.18% 173 | ✔ RoaringBitmap32.addMany Array 1,652.62 ops/sec ±4.20% 64 runs -44.25% 174 | ✔ RoaringBitmap32.addMany Uint32Array 2,964.19 ops/sec ±1.47% 86 runs fastest 175 | ➔ Fastest is RoaringBitmap32.addMany Uint32Array 176 | 177 | • suite iterator 178 | 65536 elements 179 | ✔ Set 1,648.55 ops/sec ±1.21% 86 runs fastest 180 | ✔ RoaringBitmap32 1,239.06 ops/sec ±1.62% 86 runs -24.84% 181 | ➔ Fastest is Set 182 | 183 | • suite intersection size 184 | 262144 elements 185 | ✔ Set 29.10 ops/sec ±5.65% 51 runs -99.99% 186 | ✔ FastBitSet 15,938.45 ops/sec ±2.01% 86 runs -94.09% 187 | ✔ RoaringBitmap32 269,502.51 ops/sec ±2.08% 84 runs fastest 188 | ➔ Fastest is RoaringBitmap32 189 | 190 | • suite union (in place) 191 | 65536 elements 192 | ✔ Set 298.53 ops/sec ±3.07% 65 runs -99.97% 193 | ✔ FastBitSet 144,914.65 ops/sec ±2.10% 75 runs -83.38% 194 | ✔ RoaringBitmap32 871,794.31 ops/sec ±3.85% 82 runs fastest 195 | ➔ Fastest is RoaringBitmap32 196 | 197 | • suite union size 198 | 262144 elements 199 | ✔ Set 17.69 ops/sec ±3.40% 33 runs -99.99% 200 | ✔ FastBitSet 8,481.07 ops/sec ±1.54% 84 runs -97.02% 201 | ✔ RoaringBitmap32 284,749.52 ops/sec ±1.35% 86 runs fastest 202 | ➔ Fastest is RoaringBitmap32 203 | 204 | • suite union (new) 205 | 1048576 elements 206 | ✔ Set 1.83 ops/sec ±6.60% 9 runs -99.90% 207 | ✔ FastBitSet 744.68 ops/sec ±1.11% 85 runs -60.45% 208 | ✔ RoaringBitmap32 1,882.93 ops/sec ±16.32% 44 runs fastest 209 | ➔ Fastest is RoaringBitmap32 210 | 211 | 212 | * completed: 53610.279ms 213 | ``` 214 | 215 | Works also on M1 216 | 217 | ```` 218 | Platform : Darwin 21.1.0 arm64 219 | CPU : Apple M1 Pro 220 | Cores : 10 physical - 10 logical 221 | Memory : 16.00 GB 222 | NodeJS : v16.13.1 - V8 v9.4.146.24-node.14 223 | 224 | * running 8 files... 225 | 226 | • suite union (in place) 227 | 65536 elements 228 | ✔ Set 456.60 ops/sec ±1.88% 77 runs -99.97% 229 | ✔ FastBitSet 232,794.03 ops/sec ±0.94% 91 runs -86.52% 230 | ✔ RoaringBitmap32 1,727,310.24 ops/sec ±1.12% 95 runs fastest 231 | ➔ Fastest is RoaringBitmap32 232 | 233 | • suite intersection size 234 | 262144 elements 235 | ✔ Set 71.20 ops/sec ±2.30% 62 runs -99.98% 236 | ✔ FastBitSet 21,525.29 ops/sec ±0.71% 97 runs -93.72% 237 | ✔ RoaringBitmap32 342,892.37 ops/sec ±0.93% 95 runs fastest 238 | ➔ Fastest is RoaringBitmap32 239 | 240 | • suite intersection (in place) 241 | 65536 elements 242 | ✔ Set 280.62 ops/sec ±1.66% 76 runs -99.97% 243 | ✔ FastBitSet 136,148.03 ops/sec ±0.56% 96 runs -87.11% 244 | ✔ RoaringBitmap32 1,055,978.14 ops/sec ±1.16% 92 runs fastest 245 | ➔ Fastest is RoaringBitmap32 246 | 247 | • suite union size 248 | 262144 elements 249 | ✔ Set 37.95 ops/sec ±2.33% 51 runs -99.99% 250 | ✔ FastBitSet 12,111.37 ops/sec ±0.67% 96 runs -96.35% 251 | ✔ RoaringBitmap32 331,510.04 ops/sec ±1.02% 95 runs fastest 252 | ➔ Fastest is RoaringBitmap32 253 | 254 | • suite intersection (new) 255 | 1048576 elements 256 | ✔ Set 6.54 ops/sec ±5.41% 21 runs -99.93% 257 | ✔ FastBitSet 2,087.94 ops/sec ±5.23% 42 runs -78.88% 258 | ✔ RoaringBitmap32 9,888.24 ops/sec ±1.77% 51 runs fastest 259 | ➔ Fastest is RoaringBitmap32 260 | 261 | • suite union (new) 262 | 1048576 elements 263 | ✔ Set 3.80 ops/sec ±7.91% 14 runs -99.93% 264 | ✔ FastBitSet 1,693.77 ops/sec ±3.90% 64 runs -70.63% 265 | ✔ RoaringBitmap32 5,767.35 ops/sec ±1.74% 51 runs fastest 266 | ➔ Fastest is RoaringBitmap32 267 | 268 | • suite iterator 269 | 65536 elements 270 | ✔ Set.iterator 10,033.75 ops/sec ±1.40% 92 runs fastest 271 | ✔ Set.forEach 1,595.08 ops/sec ±2.02% 89 runs -84.10% 272 | ✔ RoaringBitmap32.iterator 2,879.57 ops/sec ±1.06% 93 runs -71.30% 273 | ✔ RoaringBitmap32.forEach 1,764.98 ops/sec ±0.68% 97 runs -82.41% 274 | ➔ Fastest is Set.iterator 275 | 276 | • suite add 277 | 65535 elements 278 | ✔ Set.add 508.76 ops/sec ±1.27% 86 runs -91.37% 279 | ✔ RoaringBitmap32.tryAdd 707.70 ops/sec ±0.81% 96 runs -87.99% 280 | ✔ RoaringBitmap32.add 699.27 ops/sec ±1.08% 90 runs -88.13% 281 | ✔ RoaringBitmap32.addMany Array 4,457.70 ops/sec ±0.54% 90 runs -24.35% 282 | ✔ RoaringBitmap32.addMany Uint32Array 5,892.57 ops/sec ±0.07% 101 runs fastest 283 | ➔ Fastest is RoaringBitmap32.addMany Uint32Array 284 | 285 | 286 | * completed: 27.170s``` 287 | 288 | ```` 289 | -------------------------------------------------------------------------------- /RoaringBitmap32.d.ts: -------------------------------------------------------------------------------- 1 | import roaring from "."; 2 | 3 | /** 4 | * Roaring bitmap that supports 32 bit unsigned integers. 5 | * 6 | * See http://roaringbitmap.org/ 7 | * 8 | * @type {roaring.RoaringBitmap32} 9 | */ 10 | export = roaring.RoaringBitmap32; 11 | -------------------------------------------------------------------------------- /RoaringBitmap32.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./index").RoaringBitmap32; 2 | -------------------------------------------------------------------------------- /RoaringBitmap32Iterator.d.ts: -------------------------------------------------------------------------------- 1 | import roaring from "."; 2 | 3 | /** 4 | * Iterator class for RoaringBitmap32 5 | * 6 | * @type {roaring.RoaringBitmap32} 7 | */ 8 | export = roaring.RoaringBitmap32Iterator; 9 | -------------------------------------------------------------------------------- /RoaringBitmap32Iterator.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./index").RoaringBitmap32Iterator; 2 | -------------------------------------------------------------------------------- /RoaringBitmap32ReverseIterator.d.ts: -------------------------------------------------------------------------------- 1 | import roaring from "."; 2 | 3 | /** 4 | * Iterator class for RoaringBitmap32 5 | * 6 | * @type {roaring.RoaringReverseBitmap32} 7 | */ 8 | export = roaring.RoaringBitmap32ReverseIterator; 9 | -------------------------------------------------------------------------------- /RoaringBitmap32ReverseIterator.js: -------------------------------------------------------------------------------- 1 | module.exports = require("./index").RoaringBitmap32ReverseIterator; 2 | -------------------------------------------------------------------------------- /benchmarks/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../.eslintrc.json", 3 | "rules": { 4 | "import/no-extraneous-dependencies": 0 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /benchmarks/add.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const N = 65535; 6 | 7 | bench.suite("add", (suite) => { 8 | suite.detail(`${N} elements`); 9 | 10 | const data = new Uint32Array(N); 11 | for (let i = 0; i < N; i++) { 12 | data[i] = 3 * i + 5; 13 | } 14 | 15 | const dataArray = Array.from(data); 16 | 17 | suite.scope(() => { 18 | let x; 19 | suite.benchmark("Set.add", { 20 | setup() { 21 | x = new Set(); 22 | }, 23 | fn() { 24 | for (let i = 0; i < N; ++i) { 25 | x.add(data[i]); 26 | } 27 | }, 28 | }); 29 | }); 30 | 31 | suite.scope(() => { 32 | let x; 33 | suite.benchmark("RoaringBitmap32.tryAdd", { 34 | setup() { 35 | x = new RoaringBitmap32(); 36 | }, 37 | fn() { 38 | for (let i = 0; i < N; ++i) { 39 | x.tryAdd(data[i]); 40 | } 41 | }, 42 | }); 43 | }); 44 | 45 | suite.scope(() => { 46 | let x; 47 | suite.benchmark("RoaringBitmap32.add", { 48 | setup() { 49 | x = new RoaringBitmap32(); 50 | }, 51 | fn() { 52 | for (let i = 0; i < N; ++i) { 53 | x.add(data[i]); 54 | } 55 | }, 56 | }); 57 | }); 58 | 59 | suite.scope(() => { 60 | let x; 61 | suite.benchmark("RoaringBitmap32.addMany Array", { 62 | setup() { 63 | x = new RoaringBitmap32(); 64 | }, 65 | fn() { 66 | x.addMany(dataArray); 67 | }, 68 | }); 69 | }); 70 | 71 | suite.scope(() => { 72 | let x; 73 | suite.benchmark("RoaringBitmap32.addMany Uint32Array", { 74 | setup() { 75 | x = new RoaringBitmap32(); 76 | }, 77 | fn() { 78 | x.addMany(data); 79 | }, 80 | }); 81 | }); 82 | }); 83 | 84 | if (require.main === module) { 85 | bench.run().catch((err) => { 86 | // eslint-disable-next-line no-console 87 | console.error(err); 88 | }); 89 | } 90 | -------------------------------------------------------------------------------- /benchmarks/intersection-inplace.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 256 * 256; 8 | 9 | bench.suite("intersection (in place)", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | const b1 = new Uint32Array(N); 13 | const b2 = new Uint32Array(N); 14 | for (let i = 0; i < N; i++) { 15 | b1[i] = 3 * i + 5; 16 | b2[i] = 6 * i + 5; 17 | } 18 | 19 | suite.scope(() => { 20 | const s1 = new Set(b1); 21 | let s2; 22 | suite.benchmark("Set", { 23 | setup() { 24 | s2 = new Set(b2); 25 | }, 26 | fn() { 27 | for (const j of s1) { 28 | s2.delete(j); 29 | } 30 | }, 31 | }); 32 | }); 33 | 34 | suite.scope(() => { 35 | const s1 = new FastBitSet(b1); 36 | let s2; 37 | suite.benchmark("FastBitSet", { 38 | setup() { 39 | s2 = new FastBitSet(b2); 40 | }, 41 | fn() { 42 | s2.intersection(s1); 43 | }, 44 | }); 45 | }); 46 | 47 | suite.scope(() => { 48 | const s1 = new RoaringBitmap32(b1); 49 | let s2; 50 | suite.benchmark("RoaringBitmap32", { 51 | setup() { 52 | s2 = new RoaringBitmap32(b2); 53 | }, 54 | fn() { 55 | s2.andInPlace(s1); 56 | }, 57 | }); 58 | }); 59 | }); 60 | 61 | if (require.main === module) { 62 | bench.run().catch((err) => { 63 | // eslint-disable-next-line no-console 64 | console.error(err); 65 | }); 66 | } 67 | -------------------------------------------------------------------------------- /benchmarks/intersection-new.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 1024 * 1024; 8 | 9 | bench.suite("intersection (new)", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | suite.scope(() => { 13 | const s1 = new Set(); 14 | const s2 = new Set(); 15 | for (let i = 0; i < N; i++) { 16 | s1.add(3 * i + 5); 17 | s2.add(6 * i + 5); 18 | } 19 | suite.benchmark("Set", () => { 20 | const answer = new Set(); 21 | if (s2.size > s1.size) { 22 | for (const j of s1) { 23 | if (s2.has(j)) { 24 | answer.add(j); 25 | } 26 | } 27 | } else { 28 | for (const j of s2) { 29 | if (s1.has(j)) { 30 | answer.add(j); 31 | } 32 | } 33 | } 34 | return answer; 35 | }); 36 | }); 37 | 38 | suite.scope(() => { 39 | const s1 = new FastBitSet(); 40 | const s2 = new FastBitSet(); 41 | for (let i = 0; i < N; i++) { 42 | s1.add(3 * i + 5); 43 | s2.add(6 * i + 5); 44 | } 45 | suite.benchmark("FastBitSet", () => { 46 | return s1.new_intersection(s2); 47 | }); 48 | }); 49 | 50 | suite.scope(() => { 51 | const s1 = new RoaringBitmap32(); 52 | const s2 = new RoaringBitmap32(); 53 | for (let i = 0; i < N; i++) { 54 | s1.add(3 * i + 5); 55 | s2.add(6 * i + 5); 56 | } 57 | suite.benchmark("RoaringBitmap32", () => { 58 | return RoaringBitmap32.and(s1, s2); 59 | }); 60 | }); 61 | }); 62 | 63 | if (require.main === module) { 64 | bench.run().catch((err) => { 65 | // eslint-disable-next-line no-console 66 | console.error(err); 67 | }); 68 | } 69 | -------------------------------------------------------------------------------- /benchmarks/intersection-size.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 512 * 512; 8 | 9 | bench.suite("intersection size", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | const b1 = new Uint32Array(N); 13 | const b2 = new Uint32Array(N); 14 | for (let i = 0; i < N; i++) { 15 | b1[i] = 3 * i + 5; 16 | b2[i] = 6 * i + 5; 17 | } 18 | 19 | suite.scope(() => { 20 | const s1 = new Set(b1); 21 | const s2 = new Set(b2); 22 | suite.benchmark("Set", () => { 23 | let answer = 0; 24 | if (s2.size > s1.size) { 25 | for (const j of s1) { 26 | if (s2.has(j)) { 27 | answer++; 28 | } 29 | } 30 | } else { 31 | for (const j of s2.values()) { 32 | if (s1.has(j)) { 33 | answer++; 34 | } 35 | } 36 | } 37 | return answer; 38 | }); 39 | }); 40 | 41 | suite.scope(() => { 42 | const s1 = new FastBitSet(b1); 43 | const s2 = new FastBitSet(b2); 44 | suite.benchmark("FastBitSet", () => { 45 | return s1.intersection_size(s2); 46 | }); 47 | }); 48 | 49 | suite.scope(() => { 50 | const s1 = new RoaringBitmap32(b1); 51 | const s2 = new RoaringBitmap32(b2); 52 | suite.benchmark("RoaringBitmap32", () => { 53 | return s1.andCardinality(s2); 54 | }); 55 | }); 56 | }); 57 | 58 | if (require.main === module) { 59 | bench.run().catch((err) => { 60 | // eslint-disable-next-line no-console 61 | console.error(err); 62 | }); 63 | } 64 | -------------------------------------------------------------------------------- /benchmarks/iterator.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const N = 65536; 6 | 7 | bench.suite("iterator", (suite) => { 8 | suite.detail(`${N} elements`); 9 | 10 | const data = new Uint32Array(N); 11 | for (let i = 0; i < N; i++) { 12 | data[i] = 3 * i + 5; 13 | } 14 | 15 | suite.scope(() => { 16 | const x = new Set(data); 17 | suite.benchmark("Set.iterator", () => { 18 | let n = 0; 19 | for (const j of x) { 20 | n += j; 21 | } 22 | return n; 23 | }); 24 | }); 25 | 26 | suite.scope(() => { 27 | const x = new Set(data); 28 | suite.benchmark("Set.forEach", () => { 29 | let n = 0; 30 | x.forEach((v) => { 31 | n += v; 32 | }); 33 | return n; 34 | }); 35 | }); 36 | 37 | suite.scope(() => { 38 | const x = new RoaringBitmap32(data); 39 | suite.benchmark("RoaringBitmap32.iterator", () => { 40 | let n = 0; 41 | for (const j of x) { 42 | n += j; 43 | } 44 | return n; 45 | }); 46 | }); 47 | 48 | suite.scope(() => { 49 | const x = new RoaringBitmap32(data); 50 | suite.benchmark("RoaringBitmap32.forEach", () => { 51 | let n = 0; 52 | x.forEach((v) => { 53 | n += v; 54 | }); 55 | return n; 56 | }); 57 | }); 58 | }); 59 | 60 | if (require.main === module) { 61 | bench.run().catch((err) => { 62 | // eslint-disable-next-line no-console 63 | console.error(err); 64 | }); 65 | } 66 | -------------------------------------------------------------------------------- /benchmarks/union-inplace.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 256 * 256; 8 | 9 | bench.suite("union (in place)", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | const b1 = new Uint32Array(N); 13 | const b2 = new Uint32Array(N); 14 | for (let i = 0; i < N; i++) { 15 | b1[i] = 3 * i + 5; 16 | b2[i] = 6 * i + 5; 17 | } 18 | 19 | suite.scope(() => { 20 | const s1 = new Set(b1); 21 | let s2; 22 | suite.benchmark("Set", { 23 | setup() { 24 | s2 = new Set(b2); 25 | }, 26 | fn() { 27 | for (const j of s1) { 28 | s2.add(j); 29 | } 30 | }, 31 | }); 32 | }); 33 | 34 | suite.scope(() => { 35 | const s1 = new FastBitSet(b1); 36 | let s2; 37 | suite.benchmark("FastBitSet", { 38 | setup() { 39 | s2 = new FastBitSet(b2); 40 | }, 41 | fn() { 42 | s2.union(s1); 43 | }, 44 | }); 45 | }); 46 | 47 | suite.scope(() => { 48 | const s1 = new RoaringBitmap32(b1); 49 | let s2; 50 | suite.benchmark("RoaringBitmap32", { 51 | setup() { 52 | s2 = new RoaringBitmap32(b2); 53 | }, 54 | fn() { 55 | s2.orInPlace(s1); 56 | }, 57 | }); 58 | }); 59 | }); 60 | 61 | if (require.main === module) { 62 | bench.run().catch((err) => { 63 | // eslint-disable-next-line no-console 64 | console.error(err); 65 | }); 66 | } 67 | -------------------------------------------------------------------------------- /benchmarks/union-new.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 1024 * 1024; 8 | 9 | bench.suite("union (new) ", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | suite.scope(() => { 13 | const s1 = new Set(); 14 | const s2 = new Set(); 15 | for (let i = 0; i < N; i++) { 16 | s1.add(3 * i + 5); 17 | s2.add(6 * i + 5); 18 | } 19 | suite.benchmark("Set", () => { 20 | genericSetUnion(s1, s2); 21 | }); 22 | }); 23 | 24 | suite.scope(() => { 25 | const s1 = new FastBitSet(); 26 | const s2 = new FastBitSet(); 27 | for (let i = 0; i < N; i++) { 28 | s1.add(3 * i + 5); 29 | s2.add(6 * i + 5); 30 | } 31 | suite.benchmark("FastBitSet", () => { 32 | s1.new_union(s2); 33 | }); 34 | }); 35 | 36 | suite.scope(() => { 37 | const s1 = new RoaringBitmap32(); 38 | const s2 = new RoaringBitmap32(); 39 | for (let i = 0; i < N; i++) { 40 | s1.add(3 * i + 5); 41 | s2.add(6 * i + 5); 42 | } 43 | suite.benchmark("RoaringBitmap32", () => { 44 | RoaringBitmap32.or(s1, s2); 45 | }); 46 | }); 47 | }); 48 | 49 | if (require.main === module) { 50 | bench.run().catch((err) => { 51 | // eslint-disable-next-line no-console 52 | console.error(err); 53 | }); 54 | } 55 | 56 | function genericSetUnion(set1, set2) { 57 | const answer = new Set(set1); 58 | for (const j of set2) { 59 | answer.add(j); 60 | } 61 | return answer; 62 | } 63 | -------------------------------------------------------------------------------- /benchmarks/union-size.bench.js: -------------------------------------------------------------------------------- 1 | const bench = require("../scripts/benchmarks/bench"); 2 | const roaring = require("../"); 3 | 4 | const RoaringBitmap32 = roaring.RoaringBitmap32; 5 | const FastBitSet = require("fastbitset"); 6 | 7 | const N = 512 * 512; 8 | 9 | bench.suite("union size", (suite) => { 10 | suite.detail(`${N} elements`); 11 | 12 | const b1 = new Uint32Array(N); 13 | const b2 = new Uint32Array(N); 14 | for (let i = 0; i < N; i++) { 15 | b1[i] = 3 * i + 5; 16 | b2[i] = 6 * i + 5; 17 | } 18 | 19 | suite.scope(() => { 20 | const s1 = new Set(b1); 21 | const s2 = new Set(b2); 22 | suite.benchmark("Set", () => { 23 | let answer; 24 | if (s1.size > s2.size) { 25 | answer = s1.size; 26 | for (const j of s2) { 27 | if (!s1.has(j) && s2.has(j)) { 28 | answer++; 29 | } 30 | } 31 | } else { 32 | answer = s2.size; 33 | for (const j of s1) { 34 | if (!s2.has(j) && s1.has(j)) { 35 | answer++; 36 | } 37 | } 38 | } 39 | return answer; 40 | }); 41 | }); 42 | 43 | suite.scope(() => { 44 | const s1 = new FastBitSet(b1); 45 | const s2 = new FastBitSet(b2); 46 | suite.benchmark("FastBitSet", () => { 47 | s1.union_size(s2); 48 | }); 49 | }); 50 | 51 | suite.scope(() => { 52 | const s1 = new RoaringBitmap32(b1); 53 | const s2 = new RoaringBitmap32(b2); 54 | suite.benchmark("RoaringBitmap32", () => { 55 | s1.orCardinality(s2); 56 | }); 57 | }); 58 | }); 59 | 60 | if (require.main === module) { 61 | bench.run().catch((err) => { 62 | // eslint-disable-next-line no-console 63 | console.error(err); 64 | }); 65 | } 66 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "variables": { 3 | "openssl_fips": '' 4 | }, 5 | "targets": [ 6 | { 7 | "target_name": "<(module_name)", 8 | "product_dir": "<(module_path)", 9 | "default_configuration": "Release", 10 | "include_dirs": ["submodules/CRoaring/include"], 11 | "cflags_cc": ["-O3", "-g0", "-std=c++17", "-fno-rtti", "-fno-exceptions", "-fvisibility=hidden", "-flto", "-Wno-unused-function", "-Wno-unused-variable", "-Wno-cast-function-type"], 12 | "ldflags": ["-s"], 13 | "xcode_settings": { 14 | "GCC_GENERATE_DEBUGGING_SYMBOLS": "NO", 15 | "OTHER_CFLAGS": ["-O3", "-g0", "-std=c++17", "-fno-rtti", "-fno-exceptions", "-fvisibility=hidden", "-flto", "-Wno-unused-function", "-Wno-unused-variable", "-Wno-cast-function-type"], 16 | }, 17 | "msvs_settings": { 18 | "VCCLCompilerTool": { 19 | "DebugInformationFormat": 0, 20 | "Optimization": 3, 21 | "AdditionalOptions": ["/O2", "/std:c++latest"] 22 | } 23 | }, 24 | "sources": ["roaring-node.cpp"] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noEmit": true, 4 | "target": "es6", 5 | "module": "commonjs", 6 | "allowSyntheticDefaultImports": true 7 | }, 8 | "typeAcquisition": { 9 | "enable": true 10 | }, 11 | "exclude": ["**/node_modules", "**/.eslintcache", "package"] 12 | } 13 | -------------------------------------------------------------------------------- /lint-staged.config.js: -------------------------------------------------------------------------------- 1 | // lint-staged.config.js 2 | const { ESLint } = require("eslint"); 3 | 4 | const removeIgnoredFiles = async (files) => { 5 | const eslint = new ESLint(); 6 | const ignoredFiles = await Promise.all(files.map((file) => eslint.isPathIgnored(file))); 7 | const filteredFiles = files.filter((_, i) => !ignoredFiles[i]); 8 | return filteredFiles.join(" "); 9 | }; 10 | 11 | module.exports = { 12 | "*.{js,jsx,ts,tsx}": async (files) => { 13 | const filesToLint = await removeIgnoredFiles(files); 14 | return [`eslint --max-warnings=0 ${filesToLint}`, `prettier --write ${filesToLint}`]; 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /node-pre-gyp.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | "use strict"; 3 | 4 | const { fork } = require("child_process"); 5 | const fs = require("fs"); 6 | const path = require("path"); 7 | 8 | const ROARING_NODE_PRE_GYP = process.env.ROARING_NODE_PRE_GYP; 9 | if (ROARING_NODE_PRE_GYP) { 10 | process.env.ROARING_NODE_PRE_GYP = ""; 11 | } 12 | 13 | function roaring32NodePreGyp() { 14 | console.time("node-pre-gyp"); 15 | process.on("exit", () => { 16 | console.timeEnd("node-pre-gyp"); 17 | }); 18 | 19 | process.chdir(__dirname); 20 | 21 | if (ROARING_NODE_PRE_GYP !== "custom-rebuild") { 22 | try { 23 | process.env.npm_config_node_gyp = require.resolve("node-gyp/bin/node-gyp.js"); 24 | } catch (_e) {} 25 | 26 | require("@mapbox/node-pre-gyp/lib/main"); 27 | } else { 28 | const forkAsync = (modulePath, args, options) => { 29 | return new Promise((resolve, reject) => { 30 | const childProcess = fork(modulePath, args, { stdio: "inherit", ...options }); 31 | childProcess.on("error", (e) => { 32 | reject(e); 33 | }); 34 | childProcess.on("close", (code) => { 35 | if (code === 0) { 36 | resolve(); 37 | } else { 38 | reject(new Error(`${modulePath} exited with code ${code}`)); 39 | } 40 | }); 41 | }); 42 | }; 43 | 44 | const main = async () => { 45 | console.log("* rebuild..."); 46 | 47 | let glibc; 48 | try { 49 | // eslint-disable-next-line node/no-unsupported-features/node-builtins 50 | const header = process.report.getReport().header; 51 | glibc = header.glibcVersionRuntime; 52 | } catch {} 53 | 54 | console.log("versions:", { 55 | node: process.version, 56 | v8: process.versions.v8, 57 | glibc: glibc || null, 58 | }); 59 | 60 | console.time("rebuild"); 61 | await forkAsync(__filename, ["rebuild"]); 62 | console.log(); 63 | console.timeEnd("rebuild"); 64 | console.log(); 65 | 66 | // Clean debug info and temporary files 67 | for (let d of fs.readdirSync("native")) { 68 | d = path.resolve(__dirname, "native", d); 69 | if (fs.lstatSync(d).isDirectory()) { 70 | for (let f of fs.readdirSync(d)) { 71 | f = path.resolve(d, f); 72 | for (const ext of [".ipdb", ".iobj", ".pdb", ".obj", ".lib", ".exp", ".tmp"]) { 73 | if (f.endsWith(ext)) { 74 | console.log(`- removing file ${f}`); 75 | fs.unlinkSync(f); 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | console.log(); 83 | console.log("* packaging..."); 84 | console.time("packaging"); 85 | await forkAsync(__filename, ["package", "testpackage"]); 86 | console.timeEnd("packaging"); 87 | console.log(); 88 | }; 89 | 90 | main().catch((e) => { 91 | console.error(e); 92 | if (!process.exitCode) { 93 | process.exitCode = 1; 94 | } 95 | }); 96 | } 97 | } 98 | 99 | if (ROARING_NODE_PRE_GYP !== "false") { 100 | roaring32NodePreGyp(); 101 | } 102 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "roaring", 3 | "version": "2.4.1", 4 | "private": false, 5 | "description": "CRoaring official port for NodeJS", 6 | "keywords": [ 7 | "CRoaring", 8 | "Roaring", 9 | "bitmaps" 10 | ], 11 | "license": "Apache-2.0", 12 | "author": "Salvatore Previti", 13 | "homepage": "https://github.com/SalvatorePreviti/roaring-node#readme", 14 | "bugs": { 15 | "url": "https://github.com/SalvatorePreviti/roaring/issues" 16 | }, 17 | "documentation": [ 18 | "https://salvatorepreviti.github.io/roaring-node/modules.html", 19 | "https://salvatorepreviti.github.io/roaring-node/classes/RoaringBitmap32.html" 20 | ], 21 | "repository": { 22 | "type": "git", 23 | "url": "git+https://github.com/SalvatorePreviti/roaring-node.git" 24 | }, 25 | "engines": { 26 | "node": ">=16.14.0" 27 | }, 28 | "main": "index.js", 29 | "types": "index.d.ts", 30 | "typings": "index.d.ts", 31 | "files": [ 32 | "roaring-node.cpp", 33 | "LICENSE.md", 34 | "README.md", 35 | "binding.gyp", 36 | "index.js", 37 | "build.js", 38 | "index.d.ts", 39 | "node-pre-gyp.js", 40 | "RoaringBitmap32.js", 41 | "RoaringBitmap32.d.ts", 42 | "RoaringBitmap32Iterator.js", 43 | "RoaringBitmap32Iterator.d.ts", 44 | "RoaringBitmap32ReverseIterator.js", 45 | "RoaringBitmap32ReverseIterator.d.ts" 46 | ], 47 | "binary": { 48 | "module_name": "roaring", 49 | "module_path": "./native/roaring-{node_abi}-{platform}-{arch}", 50 | "host": "https://github.com/SalvatorePreviti/roaring-node/releases/download/", 51 | "remote_path": "{version}" 52 | }, 53 | "scripts": { 54 | "install": "node ./node-pre-gyp.js install --fallback-to-build", 55 | "compile": "node ./node-pre-gyp.js rebuild", 56 | "build": "node ./scripts/build.js", 57 | "update-roaring": "./scripts/update-roaring.sh", 58 | "build-dev": "node ./scripts/build.js --dev", 59 | "test": "node ./scripts/test.js", 60 | "lint": "eslint . && tsc", 61 | "lint:fix": "eslint . --fix && prettier --write . && tsc", 62 | "doc": "typedoc ./index.d.ts", 63 | "benchmarks": "node --expose-gc ./scripts/benchmarks.js" 64 | }, 65 | "husky": { 66 | "hooks": { 67 | "pre-commit": "lint-staged" 68 | } 69 | }, 70 | "dependencies": { 71 | "@mapbox/node-pre-gyp": "^1.0.11" 72 | }, 73 | "peerDependencies": { 74 | "node-gyp": "^10.2.0" 75 | }, 76 | "peerDependenciesMeta": { 77 | "node-gyp": { 78 | "optional": true 79 | } 80 | }, 81 | "optionalDependencies": { 82 | "node-gyp": "^10.2.0" 83 | }, 84 | "devDependencies": { 85 | "@balsamic/eslint-config": "^0.7.0", 86 | "@octokit/rest": "20.1.1", 87 | "@types/chai": "^4.3.19", 88 | "@types/chai-as-promised": "^7.1.8", 89 | "@types/mocha": "^10.0.9", 90 | "@types/node": "^22.7.5", 91 | "@typescript-eslint/eslint-plugin": "^7.18.0", 92 | "@typescript-eslint/parser": "^7.18.0", 93 | "benchmark": "^2.1.4", 94 | "chai": "^4.5.0", 95 | "chai-as-promised": "^7.1.2", 96 | "esbuild": "^0.24.0", 97 | "eslint": "^8.57.1", 98 | "eslint-plugin-chai-expect": "^3.1.0", 99 | "eslint-plugin-import": "^2.31.0", 100 | "eslint-plugin-json": "^4.0.1", 101 | "eslint-plugin-node": "^11.1.0", 102 | "fastbitset": "^0.4.1", 103 | "husky": "^9.1.6", 104 | "lint-staged": "^15.2.10", 105 | "mocha": "^10.7.3", 106 | "node-fetch-cjs": "^3.3.2", 107 | "physical-cpu-count": "^2.0.0", 108 | "prettier": "^3.3.3", 109 | "tslib": "^2.7.0", 110 | "tsx": "^4.19.1", 111 | "typedoc": "^0.26.8", 112 | "typescript": "5.6.2" 113 | }, 114 | "gypfile": true, 115 | "roaring_version": "4.2.1" 116 | } 117 | -------------------------------------------------------------------------------- /scripts/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../.eslintrc.json", 3 | "rules": { 4 | "import/no-extraneous-dependencies": 0, 5 | "no-console": 0 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /scripts/benchmarks.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const systemInfo = require("./system-info"); 4 | const promiseMap = require("./benchmarks/promiseMap"); 5 | const colors = require("chalk"); 6 | const path = require("path"); 7 | const fs = require("fs"); 8 | const { fork } = require("child_process"); 9 | const benchReport = require("./benchmarks/benchReport"); 10 | const spinner = require("./benchmarks/spinner"); 11 | const { runMain } = require("./lib/utils"); 12 | 13 | function listBenchFiles() { 14 | return new Promise((resolve, reject) => { 15 | const folder = path.resolve(path.join(__dirname, "../benchmarks")); 16 | fs.readdir(folder, (err, files) => { 17 | if (err) { 18 | reject(err); 19 | } else { 20 | resolve( 21 | files 22 | .filter((file) => file.endsWith(".bench.js")) 23 | .sort() 24 | .map((file) => { 25 | return path.join(folder, file); 26 | }), 27 | ); 28 | } 29 | }); 30 | }); 31 | } 32 | 33 | function runBenchFileAsync(benchFile) { 34 | return new Promise((resolve, reject) => { 35 | let hasErrors = false; 36 | const forked = fork(benchFile, [benchReport.colorFlags], { 37 | env: { NODE_BENCHMARK_FORKED: 1 }, 38 | }); 39 | 40 | forked.on("error", reject); 41 | 42 | forked.on("message", (message) => { 43 | if (message && message.type === "suiteReport") { 44 | spinner.clear(); 45 | if (message.hasErrors) { 46 | hasErrors = true; 47 | } 48 | benchReport.printSuiteName(message.name); 49 | benchReport.printSuiteReport(message); 50 | } 51 | }); 52 | 53 | forked.on("exit", (code) => { 54 | if (code !== 0) { 55 | const error = new Error(`"${benchFile}" failed.`); 56 | error.stack = `Error: ${error.message}`; 57 | reject(error); 58 | } else { 59 | resolve(!hasErrors); 60 | } 61 | }); 62 | }); 63 | } 64 | 65 | async function benchmarks() { 66 | systemInfo.print(); 67 | spinner.start(); 68 | let hasErrors = false; 69 | try { 70 | const benchFiles = await listBenchFiles(); 71 | console.log( 72 | colors.green("*"), 73 | colors.greenBright("running"), 74 | colors.cyanBright(benchFiles.length), 75 | colors.greenBright("files..."), 76 | "\n", 77 | ); 78 | await promiseMap( 79 | benchFiles, 80 | async (benchFile) => { 81 | if (!(await runBenchFileAsync(benchFile))) { 82 | hasErrors = true; 83 | } 84 | }, 85 | Math.max(1, systemInfo.physicalCpuCount - 1), 86 | ); 87 | if (hasErrors) { 88 | const error = new Error("Benchmarks failed"); 89 | error.stack = `Error: ${error.message}`; 90 | throw error; 91 | } 92 | } finally { 93 | spinner.stop(); 94 | } 95 | } 96 | 97 | function run() { 98 | const completedMessage = `\n${colors.green("*")} ${colors.greenBright("completed")}`; 99 | console.time(completedMessage); 100 | 101 | return benchmarks() 102 | .then(() => { 103 | spinner.clear(); 104 | console.timeEnd(completedMessage); 105 | console.log(); 106 | return true; 107 | }) 108 | .catch((error) => { 109 | spinner.clear(); 110 | console.error(colors.red("*"), colors.redBright("FAIL"), colors.red("-"), error, "\n"); 111 | if (!process.exitCode) { 112 | process.exitCode = 1; 113 | } 114 | return false; 115 | }); 116 | } 117 | 118 | module.exports = benchmarks; 119 | 120 | benchmarks.run = run; 121 | 122 | if (require.main === module) { 123 | runMain(run, "benchmarks"); 124 | } 125 | -------------------------------------------------------------------------------- /scripts/benchmarks/bench.js: -------------------------------------------------------------------------------- 1 | const Benchmark = require("benchmark"); 2 | const benchReport = require("./benchReport"); 3 | 4 | const { defineProperty } = Reflect; 5 | 6 | const suitesDeclarations = []; 7 | 8 | function errorToString(error) { 9 | return error.stack || String(error || "Error"); 10 | } 11 | 12 | const actions = 13 | process.send && process.env.NODE_BENCHMARK_FORKED === "1" 14 | ? { 15 | suiteStart() {}, 16 | suiteReport(report) { 17 | process.send(report); 18 | }, 19 | } 20 | : { 21 | suiteStart: benchReport.printSuiteName, 22 | suiteReport: benchReport.printSuiteReport, 23 | }; 24 | 25 | function pushBenchResult(results, b) { 26 | const failed = b.error || b.aborted; 27 | results.push({ 28 | name: b.name, 29 | aborted: b.aborted, 30 | error: b.error !== undefined && errorToString(b.error), 31 | hz: failed ? 0 : b.hz || 0, 32 | ops: failed ? "" : b.hz.toFixed(2).replace(/\B(?=(\d{3})+\b)/g, ","), 33 | rme: failed ? "" : b.stats.rme.toFixed(2), 34 | runs: failed ? "" : `${b.stats.sample.length}`, 35 | diff: "", 36 | fastest: false, 37 | }); 38 | } 39 | 40 | class BenchSuite { 41 | constructor(name) { 42 | this.name = name; 43 | this.results = []; 44 | this.details = []; 45 | this.hasErrors = false; 46 | } 47 | 48 | detail(text) { 49 | this.details.push(text); 50 | } 51 | 52 | scope(fn) { 53 | return fn(); 54 | } 55 | 56 | benchmark(name, fn) { 57 | if (typeof global.gc === "function") { 58 | global.gc(); 59 | } 60 | 61 | { 62 | let b; 63 | 64 | if (typeof fn === "function") { 65 | defineProperty(fn, "name", { 66 | value: `${this.name}:${fn.name || name}`, 67 | configurable: true, 68 | writable: true, 69 | }); 70 | b = new Benchmark({ name, fn, async: false }); 71 | } else { 72 | b = new Benchmark({ name, fn: fn.fn, async: false }); 73 | } 74 | 75 | if (typeof fn.setup === "function") { 76 | b.on("start cycle", () => { 77 | fn.setup(this); 78 | }); 79 | } 80 | 81 | b.on("complete", () => { 82 | if (b.error) { 83 | this.hasErrors = true; 84 | } 85 | pushBenchResult(this.results, b); 86 | }); 87 | 88 | b.run(); 89 | } 90 | 91 | if (typeof global.gc === "function") { 92 | global.gc(); 93 | } 94 | 95 | return this; 96 | } 97 | } 98 | 99 | async function runSuite(suiteDeclaration) { 100 | const benchSuite = new BenchSuite(suiteDeclaration.name); 101 | let fastest = null; 102 | let error = null; 103 | try { 104 | actions.suiteStart(suiteDeclaration.name); 105 | 106 | await suiteDeclaration.fn(benchSuite); 107 | 108 | let fastValue = 0; 109 | for (const b of benchSuite.results) { 110 | if (fastValue < b.hz) { 111 | fastValue = b.hz; 112 | fastest = b; 113 | } 114 | } 115 | 116 | if (fastest) { 117 | fastest.fastest = true; 118 | for (const b of benchSuite.results) { 119 | b.diff = b.fastest ? "" : `-${((1.0 - b.hz / fastest.hz) * 100).toFixed(2)}%`; 120 | } 121 | } 122 | } catch (e) { 123 | benchSuite.hasErrors = true; 124 | error = errorToString(e); 125 | } finally { 126 | actions.suiteReport({ 127 | type: "suiteReport", 128 | name: suiteDeclaration.name, 129 | details: benchSuite.details, 130 | error, 131 | hasErrors: benchSuite.hasErrors, 132 | benchs: benchSuite.results, 133 | fastest: fastest && fastest.name, 134 | }); 135 | } 136 | } 137 | 138 | module.exports = { 139 | /** 140 | * Defines a new benchmark suite 141 | * 142 | * @param {string} name The name of the suite. Required. 143 | * @param {(BenchSuite)} fn The content of the suite. Required 144 | * @returns {void} 145 | */ 146 | suite(name, fn) { 147 | if (!fn.name) { 148 | defineProperty(fn, "name", { value: name, configurable: true, writable: true }); 149 | } 150 | suitesDeclarations.push({ name, fn }); 151 | }, 152 | 153 | async run() { 154 | for (const suite of suitesDeclarations.slice().sort((x) => x.name)) { 155 | await runSuite(suite); 156 | } 157 | }, 158 | }; 159 | -------------------------------------------------------------------------------- /scripts/benchmarks/benchReport.js: -------------------------------------------------------------------------------- 1 | const colors = require("chalk"); 2 | 3 | const hasIcons = colors.supportsColor.has256; 4 | 5 | const icons = { 6 | success: colors.green(hasIcons ? "✔" : "√"), 7 | error: hasIcons ? "✖" : "×", 8 | arrow: hasIcons ? "➔" : "-", 9 | bullet: hasIcons ? "•" : "*", 10 | }; 11 | 12 | function printBenchError(suiteName, bench, error) { 13 | console.log( 14 | "\n", 15 | colors.red("-"), 16 | colors.yellowBright( 17 | `Suite ${colors.yellow.bold.italic(suiteName)}${ 18 | bench ? `Benchmark ${colors.yellow.bold.italic(bench.name)}` : "" 19 | } failed`, 20 | ), 21 | "-", 22 | colors.redBright(error), 23 | ); 24 | } 25 | 26 | module.exports = { 27 | printSuiteName(name) { 28 | console.log(`${colors.cyan(icons.bullet)} ${colors.cyan("suite")} ${colors.cyanBright(name)}`); 29 | }, 30 | 31 | printSuiteReport(report) { 32 | const cells = { name: 0, ops: 0, rme: 0, runs: 0, diff: 0 }; 33 | 34 | for (const bench of report.benchs) { 35 | cells.name = Math.max(cells.name, bench.name.length); 36 | cells.ops = Math.max(cells.ops, bench.ops.length); 37 | cells.rme = Math.max(cells.rme, bench.rme.length); 38 | cells.runs = Math.max(cells.runs, bench.runs.length); 39 | cells.diff = Math.max(cells.diff, bench.diff.length); 40 | } 41 | 42 | for (const detail of report.details) { 43 | console.log(` ${colors.gray(detail)}`); 44 | } 45 | 46 | let errors = 0; 47 | if (report.error) { 48 | ++errors; 49 | printBenchError(report.name, null, report.error); 50 | } 51 | 52 | for (const bench of report.benchs) { 53 | if (bench.error) { 54 | ++errors; 55 | printBenchError(report.name, bench, bench.error); 56 | } 57 | } 58 | 59 | if (errors) { 60 | console.log(); 61 | } 62 | 63 | for (const bench of report.benchs) { 64 | const s = []; 65 | if (bench.aborted || bench.error) { 66 | const errorKind = colors.red.bold(bench.error ? "error" : "aborted"); 67 | s.push(` ${colors.red(icons.error)} ${colors.redBright(bench.name.padEnd(cells.name))} ${errorKind}`); 68 | } else { 69 | s.push(` ${colors.green(icons.success)} ${colors.greenBright(bench.name.padEnd(cells.name))}`); 70 | s.push(` ${colors.cyanBright(bench.ops.padStart(cells.ops))}${colors.cyan(" ops/sec")}`); 71 | s.push(`±${bench.rme}%`.padStart(cells.rme + 3)); 72 | s.push(`${bench.runs} runs`.padStart(cells.runs + 6)); 73 | if (bench.fastest) { 74 | s.push(colors.green(colors.italic(" fastest"))); 75 | } else { 76 | s.push(colors.yellow(bench.diff.padStart(cells.diff + 1))); 77 | } 78 | } 79 | console.log(...s); 80 | } 81 | if (report.benchs.length) { 82 | if (report.fastest) { 83 | console.log(colors.cyan(` ${icons.arrow} Fastest is ${colors.greenBright(report.fastest)}\n`)); 84 | } else { 85 | console.log(colors.gray(` ${icons.arrow} No winner\n`)); 86 | } 87 | } else { 88 | console.log(colors.gray(` ${icons.arrow} No benchmarks\n`)); 89 | } 90 | }, 91 | 92 | get colorFlags() { 93 | return colors.supportsColor.hasBasic ? (colors.supportsColor.has256 ? "--color=256" : "--colors") : ""; 94 | }, 95 | }; 96 | -------------------------------------------------------------------------------- /scripts/benchmarks/promiseMap.js: -------------------------------------------------------------------------------- 1 | function promiseMap(collection, functor, concurrency) { 2 | return new Promise((resolve, reject) => { 3 | let running = 0; 4 | let index = 0; 5 | let rejected = false; 6 | const iterator = collection[Symbol.iterator](); 7 | 8 | function doNext() { 9 | if (rejected) { 10 | return false; 11 | } 12 | 13 | const current = iterator.next(); 14 | if (current.done) { 15 | if (running === 0 && !rejected) { 16 | resolve(); 17 | } 18 | return false; 19 | } 20 | 21 | Promise.resolve(functor(current.value, index)).then(onResolved).catch(onRejected); 22 | index++; 23 | running++; 24 | return true; 25 | } 26 | 27 | function onRejected(reason) { 28 | rejected = true; 29 | reject(reason); 30 | } 31 | 32 | function onResolved() { 33 | running--; 34 | doNext(); 35 | } 36 | 37 | for (;;) { 38 | if (!(running < concurrency && doNext())) { 39 | break; 40 | } 41 | } 42 | }); 43 | } 44 | 45 | module.exports = promiseMap; 46 | -------------------------------------------------------------------------------- /scripts/benchmarks/spinner.js: -------------------------------------------------------------------------------- 1 | const colors = require("chalk"); 2 | const readline = require("readline"); 3 | 4 | const spinner = { 5 | interval: null, 6 | printed: false, 7 | start() { 8 | if (!spinner.interval && colors.supportsColor.hasBasic && !process.env.CI) { 9 | const dot = colors.gray("."); 10 | spinner.interval = 11 | colors.supportsColor.hasBasic && 12 | setInterval(() => { 13 | spinner.printed = true; 14 | process.stderr.write(dot); 15 | }, 1000); 16 | } 17 | }, 18 | clear() { 19 | if (spinner.printed) { 20 | spinner.printed = false; 21 | readline.clearLine(process.stderr); 22 | readline.cursorTo(process.stderr, 0); 23 | } 24 | }, 25 | stop() { 26 | if (spinner.interval) { 27 | spinner.clear(); 28 | clearInterval(spinner.interval); 29 | spinner.interval = null; 30 | } 31 | }, 32 | }; 33 | 34 | module.exports = spinner; 35 | -------------------------------------------------------------------------------- /scripts/build.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const colors = require("chalk"); 4 | const path = require("path"); 5 | const fs = require("fs"); 6 | const { runMain, forkAsync } = require("./lib/utils"); 7 | 8 | const { CPP_UNITY_FILE_PATH, ROOT_FOLDER, getBinaryOutputFilePath } = require("./lib/utils"); 9 | const { unity } = require("./lib/unity"); 10 | 11 | async function development() { 12 | console.log(); 13 | const warningMessage = "Development mode is enabled. Rebuild for production before publishing."; 14 | console.warn(colors.yellowBright.underline.bold("WARNING") + colors.yellow(`: ${warningMessage}`)); 15 | console.log(); 16 | let outputText = ""; 17 | outputText += "#define ROARING_NODE_DEV 1\n\n"; 18 | outputText += `#pragma message (${JSON.stringify(warningMessage)})\n`; 19 | outputText += '#include "src/cpp/main.cpp"\n'; 20 | 21 | let oldContent; 22 | try { 23 | oldContent = fs.readFileSync(CPP_UNITY_FILE_PATH, "utf8"); 24 | } catch {} 25 | if (oldContent !== outputText) { 26 | fs.writeFileSync(path.resolve(CPP_UNITY_FILE_PATH), outputText, "utf8"); 27 | console.log(colors.yellow(`- ${path.relative(ROOT_FOLDER, CPP_UNITY_FILE_PATH)} updated`)); 28 | } else { 29 | console.log(colors.blackBright(`- ${path.relative(ROOT_FOLDER, CPP_UNITY_FILE_PATH)} is up to date`)); 30 | } 31 | } 32 | 33 | async function build() { 34 | if ( 35 | process.argv.includes("dev") || 36 | process.argv.includes("--dev") || 37 | process.argv.includes("development") || 38 | process.argv.includes("--development") 39 | ) { 40 | console.time("Development mode"); 41 | await development(); 42 | console.timeEnd("Development mode"); 43 | } else { 44 | console.time("Unity build"); 45 | const unityResult = unity(); 46 | 47 | let oldContent; 48 | try { 49 | oldContent = fs.readFileSync(CPP_UNITY_FILE_PATH, "utf8"); 50 | } catch {} 51 | if (oldContent !== unityResult.outputText) { 52 | fs.writeFileSync(path.resolve(CPP_UNITY_FILE_PATH), unityResult.outputText, "utf8"); 53 | console.log(colors.yellow(`- ${path.relative(ROOT_FOLDER, CPP_UNITY_FILE_PATH)} updated`)); 54 | } else { 55 | console.log(colors.blackBright(`- ${path.relative(ROOT_FOLDER, CPP_UNITY_FILE_PATH)} is up to date`)); 56 | } 57 | 58 | const packageJsonPath = path.resolve(ROOT_FOLDER, "package.json"); 59 | const oldPackageJson = fs.readFileSync(packageJsonPath, "utf8"); 60 | const pkg = JSON.parse(oldPackageJson); 61 | pkg.roaring_version = unityResult.roaringVersion; 62 | const newPackageJson = `${JSON.stringify(pkg, null, 2)}\n`; 63 | 64 | if (newPackageJson !== oldPackageJson) { 65 | fs.writeFileSync(packageJsonPath, newPackageJson); 66 | console.log(colors.yellow("- package.json updated")); 67 | } else { 68 | console.log(colors.blackBright("- package.json is up to date")); 69 | } 70 | 71 | console.timeEnd("Unity build"); 72 | } 73 | console.log(); 74 | 75 | process.on("exit", () => { 76 | console.log(); 77 | 78 | const BINARY_OUTPUT_FILE_PATH = getBinaryOutputFilePath(); 79 | if (fs.existsSync(BINARY_OUTPUT_FILE_PATH)) { 80 | console.log(); 81 | console.log( 82 | colors.green( 83 | `* ${path.relative(ROOT_FOLDER, BINARY_OUTPUT_FILE_PATH)} compiled. ${ 84 | fs.statSync(BINARY_OUTPUT_FILE_PATH).size 85 | } bytes`, 86 | ), 87 | ); 88 | } 89 | console.log(); 90 | }); 91 | 92 | if (!process.argv.includes("--no-compile")) { 93 | await forkAsync(path.resolve(ROOT_FOLDER, "node-pre-gyp.js"), [], { 94 | env: { 95 | ...process.env, 96 | ROARING_NODE_PRE_GYP: "custom-rebuild", 97 | }, 98 | }); 99 | await forkAsync(require.resolve("./test.js")); 100 | } 101 | } 102 | 103 | module.exports.build = build; 104 | 105 | if (require.main === module) { 106 | runMain(build, "build"); 107 | } 108 | -------------------------------------------------------------------------------- /scripts/lib/unity.js: -------------------------------------------------------------------------------- 1 | const colors = require("chalk"); 2 | const path = require("path"); 3 | 4 | const fs = require("fs"); 5 | 6 | const { CPP_SRC_FOLDER_PATH } = require("./utils"); 7 | 8 | const INCLUDE_REGEX = /^#\s*include\s+["<](.+)[">]/; 9 | const ROARING_VERSION_REGEX = /#\s*define\s+ROARING_VERSION\s+"(.+)"/; 10 | 11 | module.exports.unity = function unity() { 12 | const existsCache = new Map(); 13 | 14 | const exists = (filePath) => { 15 | let result = existsCache.get(filePath); 16 | if (result !== undefined) { 17 | return result; 18 | } 19 | result = fs.existsSync(filePath); 20 | existsCache.set(filePath, result); 21 | return result; 22 | }; 23 | 24 | const includedFiles = new Set(); 25 | let roaringVersion = null; 26 | 27 | const output = ['// This file is generated by "scripts/build.js". Do not edit it directly.', ""]; 28 | 29 | function processFile(filePath) { 30 | const content = fs.readFileSync(filePath, "utf8"); 31 | for (const line of content.split("\n")) { 32 | const includeMatch = line.match(INCLUDE_REGEX); 33 | if (includeMatch) { 34 | const includeName = includeMatch[1]; 35 | 36 | let includePath; 37 | if (includeName.startsWith("roaring/") && line.includes("<")) { 38 | includePath = path.resolve(__dirname, "../../submodules/CRoaring/include", includeName); 39 | } else { 40 | includePath = path.resolve(path.dirname(filePath), includeName); 41 | } 42 | if (exists(includePath)) { 43 | if (!includedFiles.has(includePath)) { 44 | output.push(`\n// ${line}\n`); 45 | includedFiles.add(includePath); 46 | processFile(includePath); 47 | } 48 | continue; 49 | } 50 | } else if (!roaringVersion) { 51 | const match = line.match(ROARING_VERSION_REGEX); 52 | if (match) { 53 | roaringVersion = match[1]; 54 | if (!/^[0-9]+\.[0-9]+\.[0-9]+$/.test(roaringVersion)) { 55 | throw new Error(`Invalid roaring version ${roaringVersion}`); 56 | } 57 | } 58 | } 59 | output.push(line); 60 | } 61 | } 62 | 63 | processFile(path.resolve(CPP_SRC_FOLDER_PATH, "main.cpp")); 64 | 65 | console.log(); 66 | console.log(colors.cyan(`- roaring version ${roaringVersion}`)); 67 | 68 | let outputText = output.join("\n"); 69 | 70 | // This is to fix compiling C code with C++ compiler on Windows 2019, to avoid the error 71 | // C4576: a parenthesized type followed by an initializer list is a non-standard explicit type conversion syntax 72 | outputText = outputText.replaceAll("(roaring_container_iterator_t){", "roaring_container_iterator_t{"); 73 | 74 | console.log(colors.cyanBright(`- Unity: ${includedFiles.size} files included. ${outputText.length} bytes total.`)); 75 | 76 | return { 77 | roaringVersion, 78 | outputText, 79 | }; 80 | }; 81 | -------------------------------------------------------------------------------- /scripts/lib/utils.js: -------------------------------------------------------------------------------- 1 | const colors = require("chalk"); 2 | const util = require("util"); 3 | const fs = require("fs"); 4 | const path = require("path"); 5 | const { spawn, fork } = require("child_process"); 6 | 7 | const ROOT_FOLDER = path.resolve(__dirname, "../../"); 8 | const CPP_SRC_FOLDER_PATH = path.resolve(ROOT_FOLDER, "src/cpp"); 9 | const CPP_UNITY_FILE_PATH = path.resolve(ROOT_FOLDER, "roaring-node.cpp"); 10 | 11 | let BINARY_OUTPUT_FILE_PATH; 12 | 13 | function getBinaryOutputFilePath() { 14 | if (!BINARY_OUTPUT_FILE_PATH) { 15 | BINARY_OUTPUT_FILE_PATH = require("@mapbox/node-pre-gyp/lib/pre-binding").find( 16 | path.join(ROOT_FOLDER, "package.json"), 17 | ); 18 | if (!fs.existsSync(BINARY_OUTPUT_FILE_PATH)) { 19 | BINARY_OUTPUT_FILE_PATH = path.resolve(ROOT_FOLDER, "build", "Release", "roaring.node"); 20 | } 21 | } 22 | } 23 | 24 | module.exports = { 25 | ROOT_FOLDER, 26 | CPP_SRC_FOLDER_PATH, 27 | CPP_UNITY_FILE_PATH, 28 | 29 | getBinaryOutputFilePath, 30 | 31 | runMain, 32 | spawnAsync, 33 | forkAsync, 34 | mergeDirs, 35 | }; 36 | 37 | function runMain(fn, title) { 38 | if (title) { 39 | console.log(colors.blueBright(`\n⬢ ${colors.cyanBright(title)}\n`)); 40 | } 41 | 42 | if (!fn.name) { 43 | Reflect.defineProperty(fn, "name", { 44 | value: "main", 45 | configurable: true, 46 | enumerable: false, 47 | writable: false, 48 | }); 49 | } 50 | 51 | const totalTime = () => (title ? `${colors.cyan(title)}` : "") + colors.italic(` ⌚ ${process.uptime().toFixed(2)}s`); 52 | 53 | let _processCompleted = false; 54 | 55 | const processCompleted = () => { 56 | if (!process.exitCode) { 57 | console.log(colors.greenBright("✅ OK"), totalTime()); 58 | } else { 59 | console.log(colors.redBright(`❌ Failed: exitCode${process.exitCode}`), totalTime()); 60 | } 61 | console.log(); 62 | }; 63 | 64 | process.on("exit", () => { 65 | if (!_processCompleted) { 66 | _processCompleted = true; 67 | processCompleted(); 68 | } 69 | }); 70 | 71 | const handleMainError = (e) => { 72 | if (!process.exitCode) { 73 | process.exitCode = 1; 74 | } 75 | console.log("❌ ", colors.redBright(util.inspect(e, { colors: colors.level > 0 }))); 76 | console.log(totalTime()); 77 | console.log(); 78 | }; 79 | 80 | try { 81 | const result = fn(); 82 | if (typeof result === "object" && result && typeof result.then === "function") { 83 | result.then(() => {}, handleMainError); 84 | } 85 | } catch (e) { 86 | handleMainError(e); 87 | } 88 | } 89 | 90 | function spawnAsync(command, args, options) { 91 | return new Promise((resolve, reject) => { 92 | const childProcess = spawn(command, args, { stdio: "inherit", ...options }); 93 | childProcess.on("error", (e) => { 94 | reject(e); 95 | }); 96 | childProcess.on("close", (code) => { 97 | if (code === 0) { 98 | resolve(); 99 | } else { 100 | reject(new Error(`${command} exited with code ${code}`)); 101 | } 102 | }); 103 | }); 104 | } 105 | 106 | function forkAsync(modulePath, args, options) { 107 | return new Promise((resolve, reject) => { 108 | const childProcess = fork(modulePath, args, { stdio: "inherit", ...options }); 109 | childProcess.on("error", (e) => { 110 | reject(e); 111 | }); 112 | childProcess.on("close", (code) => { 113 | if (code === 0) { 114 | resolve(); 115 | } else { 116 | reject(new Error(`${modulePath} exited with code ${code}`)); 117 | } 118 | }); 119 | }); 120 | } 121 | 122 | async function mergeDirs(src, dest) { 123 | const promises = []; 124 | await fs.promises.mkdir(dest, { recursive: true }); 125 | for (const file of fs.readdirSync(src)) { 126 | const srcFile = path.resolve(src, file); 127 | const destFile = path.resolve(dest, file); 128 | const isDir = (await fs.promises.lstat(srcFile)).isDirectory(); 129 | if (isDir) { 130 | await mergeDirs(srcFile, destFile); 131 | } else { 132 | if (fs.existsSync(destFile)) { 133 | await fs.promises.rm(destFile, { recursive: true }); 134 | } 135 | promises.push(fs.promises.rename(srcFile, destFile)); 136 | } 137 | } 138 | return Promise.all(promises); 139 | } 140 | -------------------------------------------------------------------------------- /scripts/node-pre-gyp-publish.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /* 4 | Based on https://github.com/bchr02/node-pre-gyp-github 5 | 6 | The MIT License (MIT) 7 | 8 | Copyright (c) 2015 Bill Christo 9 | 10 | Permission is hereby granted, free of charge, to any person obtaining a copy 11 | of this software and associated documentation files (the "Software"), to deal 12 | in the Software without restriction, including without limitation the rights 13 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | copies of the Software, and to permit persons to whom the Software is 15 | furnished to do so, subject to the following conditions: 16 | 17 | The above copyright notice and this permission notice shall be included in all 18 | copies or substantial portions of the Software. 19 | 20 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | SOFTWARE. 27 | 28 | */ 29 | 30 | const path = require("path"); 31 | const fs = require("fs"); 32 | const colors = require("chalk"); 33 | 34 | const { Octokit } = require("@octokit/rest"); 35 | 36 | const packageJson = require("../package.json"); 37 | const { runMain } = require("./lib/utils"); 38 | const crypto = require("crypto"); 39 | const { execSync } = require("child_process"); 40 | 41 | module.exports = { 42 | startPublishAssets, 43 | }; 44 | 45 | const ALLOWED_BRANCHES = ["publish"]; 46 | 47 | async function startPublishAssets() { 48 | const assetsByName = new Map(); 49 | /** @type {Octokit} */ 50 | let octokit; 51 | let owner; 52 | let repo; 53 | let release; 54 | 55 | let initializePromise = null; 56 | let NODE_PRE_GYP_GITHUB_TOKEN; 57 | 58 | function getGithubKey() { 59 | if (NODE_PRE_GYP_GITHUB_TOKEN) { 60 | return NODE_PRE_GYP_GITHUB_TOKEN; 61 | } 62 | 63 | NODE_PRE_GYP_GITHUB_TOKEN = process.env.NODE_PRE_GYP_GITHUB_TOKEN; 64 | if (!NODE_PRE_GYP_GITHUB_TOKEN) { 65 | try { 66 | NODE_PRE_GYP_GITHUB_TOKEN = fs 67 | .readFileSync(path.resolve(__dirname, "../.github-key"), "utf8") 68 | .replace("\n", "") 69 | .replace("\r", "") 70 | .trim(); 71 | } catch (e) {} 72 | } 73 | 74 | if (!NODE_PRE_GYP_GITHUB_TOKEN) { 75 | throw new Error("NODE_PRE_GYP_GITHUB_TOKEN environment variable not set and .github-key file not found"); 76 | } 77 | 78 | return NODE_PRE_GYP_GITHUB_TOKEN; 79 | } 80 | 81 | function checkGithubKey() { 82 | getGithubKey(); 83 | } 84 | 85 | async function initialize() { 86 | const branchName = execSync("git rev-parse --abbrev-ref HEAD") 87 | .toString() 88 | .replace(/[\n\r]/g, "") 89 | .trim(); 90 | 91 | console.log(colors.cyanBright(`branchName: ${branchName}`)); 92 | 93 | if (!ALLOWED_BRANCHES.includes(branchName)) { 94 | console.warn( 95 | colors.yellowBright( 96 | `⚠️ ${colors.underline("WARNING")}: Skipping deploy. Deploy allowed only on branch ${ALLOWED_BRANCHES.join( 97 | ", ", 98 | )}`, 99 | ), 100 | ); 101 | return false; 102 | } 103 | 104 | const ownerRepo = packageJson.repository.url.match(/https?:\/\/([^/]+)\/(.*)(?=\.git)/i); 105 | const host = `api.${ownerRepo[1]}`; 106 | [owner, repo] = ownerRepo[2].split("/"); 107 | 108 | octokit = new Octokit({ 109 | baseUrl: `https://${host}`, 110 | auth: getGithubKey(), 111 | headers: { "user-agent": packageJson.name }, 112 | request: { 113 | fetch: typeof fetch !== "undefined" ? fetch : require("node-fetch-cjs").default, 114 | }, 115 | }); 116 | 117 | const { data: existingReleases } = await octokit.rest.repos.listReleases({ owner, repo }); 118 | 119 | release = existingReleases.find((element) => element.tag_name === packageJson.version); 120 | 121 | if (!release) { 122 | release = ( 123 | await octokit.rest.repos.createRelease({ 124 | host, 125 | owner, 126 | repo, 127 | tag_name: packageJson.version, 128 | target_commitish: branchName, 129 | name: `v${packageJson.version}`, 130 | body: `${packageJson.name} ${packageJson.version}`, 131 | draft: true, 132 | prerelease: /[a-zA-Z]/.test(packageJson.version), 133 | }) 134 | ).data; 135 | console.log(colors.yellow(`Creating new release ${packageJson.version}`)); 136 | } else { 137 | console.log(colors.yellow(`Using existing release ${packageJson.version}`)); 138 | } 139 | 140 | if (release.draft) { 141 | console.log(); 142 | console.warn( 143 | colors.yellowBright( 144 | `⚠️ ${colors.underline("WARNING")}: Release ${ 145 | packageJson.version 146 | } is a draft release, YOU MUST MANUALLY PUBLISH THIS DRAFT IN GITHUB.`, 147 | ), 148 | ); 149 | console.log(); 150 | } 151 | 152 | for (const asset of (release && release.assets) || []) { 153 | assetsByName.set(asset.name, asset); 154 | } 155 | 156 | return true; 157 | } 158 | 159 | const upload = async (filePath = path.resolve(__dirname, "../build/stage")) => { 160 | if (!initializePromise) { 161 | initializePromise = initialize(); 162 | } 163 | if (!(await initializePromise)) { 164 | return; 165 | } 166 | 167 | filePath = path.resolve(filePath); 168 | 169 | const stat = await fs.promises.lstat(filePath); 170 | if (stat.isDirectory()) { 171 | const files = await fs.promises.readdir(filePath); 172 | await Promise.all(files.map((file) => upload(path.resolve(filePath, file)))); 173 | return; 174 | } 175 | 176 | const name = path.basename(filePath); 177 | 178 | const data = await fs.promises.readFile(filePath); 179 | const uploadFileHash = crypto.createHash("sha256").update(data).digest("hex"); 180 | 181 | console.log(colors.cyanBright(`* file ${name} hash: ${uploadFileHash}`)); 182 | 183 | const foundAsset = assetsByName.get(name); 184 | if (foundAsset) { 185 | const result = await octokit.request({ 186 | method: "GET", 187 | url: foundAsset.url, 188 | headers: { 189 | accept: "application/octet-stream", 190 | }, 191 | }); 192 | 193 | const downloadedFileHash = crypto.createHash("sha256").update(Buffer.from(result.data)).digest("hex"); 194 | 195 | if (downloadedFileHash === uploadFileHash) { 196 | console.log(colors.yellow(`Skipping upload of ${name} because it already exists and is the same`)); 197 | return; 198 | } 199 | 200 | if (!release.draft) { 201 | if (process.argv.includes("--overwrite")) { 202 | console.log(); 203 | console.warn( 204 | colors.yellowBright(`⚠️ WARNING: Overwriting uploaded asset ${name} in release ${packageJson.version}`), 205 | ); 206 | console.log(); 207 | } else if (process.argv.includes("--no-overwrite")) { 208 | console.log(); 209 | console.warn( 210 | colors.yellowBright(`⚠️ WARNING: Skipping uploaded asset ${name} in release ${packageJson.version}`), 211 | ); 212 | console.log(); 213 | return; 214 | } 215 | throw new Error( 216 | `${packageJson.version} in release ${packageJson.version} was already published, aborting. Run with the argument "--overwrite" or "--no-overwrite".`, 217 | ); 218 | } 219 | 220 | console.log(colors.yellow(`Deleting asset ${foundAsset.name} id:${foundAsset.id} to replace it`)); 221 | await octokit.rest.repos.deleteReleaseAsset({ 222 | owner, 223 | repo, 224 | asset_id: foundAsset.id, 225 | }); 226 | } 227 | 228 | console.log(colors.yellow(`Uploading asset ${name} ${stat.size} bytes`)); 229 | await octokit.rest.repos.uploadReleaseAsset({ 230 | owner, 231 | repo, 232 | release_id: release.id, 233 | tag_name: release.tag_name, 234 | name, 235 | data, 236 | }); 237 | }; 238 | 239 | return { 240 | checkGithubKey, 241 | upload, 242 | }; 243 | } 244 | 245 | if (require.main === module) { 246 | runMain(async () => { 247 | const { upload } = await startPublishAssets(); 248 | await upload(); 249 | }, "node-pre-gyp-publish"); 250 | } 251 | -------------------------------------------------------------------------------- /scripts/prebuild-local.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /* 4 | * This script prebuild and test the library for all supported node versions locally for the local architecture. 5 | * Of course, it does not work on Windows. 6 | */ 7 | 8 | const colors = require("chalk"); 9 | const { print: printSystemInfo } = require("./system-info"); 10 | const fs = require("fs"); 11 | const path = require("path"); 12 | 13 | const { spawnAsync, mergeDirs, runMain, ROOT_FOLDER, forkAsync } = require("./lib/utils"); 14 | 15 | const { startPublishAssets } = require("./node-pre-gyp-publish"); 16 | 17 | const NODE_VERSIONS = ["16.14.0", "18.1.0", "20.9.0", "21.1.0", "22.8.0"]; 18 | 19 | const NATIVE_DIR = path.resolve(ROOT_FOLDER, "native"); 20 | const STAGE_DIR = path.resolve(ROOT_FOLDER, "build/stage"); 21 | const STAGE_TMP_DIR = path.resolve(ROOT_FOLDER, ".tmp/stage"); 22 | const TOOLS_DIR = path.resolve(ROOT_FOLDER, ".tmp/tools"); 23 | const N_EXECUTABLE_PATH = path.resolve(TOOLS_DIR, "node_modules/.bin/n"); 24 | 25 | const rmdir = fs.promises.rm || fs.promises.rmdir; 26 | 27 | async function clean() { 28 | console.log(colors.blueBright("cleaning...")); 29 | const promises = []; 30 | if (fs.existsSync(NATIVE_DIR)) { 31 | promises.push(rmdir(NATIVE_DIR, { recursive: true })); 32 | } 33 | if (fs.existsSync(STAGE_DIR)) { 34 | promises.push(rmdir(STAGE_DIR, { recursive: true })); 35 | } 36 | if (fs.existsSync(STAGE_TMP_DIR)) { 37 | promises.push(rmdir(STAGE_TMP_DIR, { recursive: true })); 38 | } 39 | await Promise.all(promises); 40 | console.log(); 41 | } 42 | 43 | function printUsage() { 44 | let s = "usage:\n"; 45 | s += " node scripts/prebuild-local - build and test\n"; 46 | s += " node scripts/prebuild-local clean - cleans the native and build/stage directories\n"; 47 | s += " node scripts/prebuild-local package - build and test and package\n"; 48 | s += " node scripts/prebuild-local deploy - build, test, package and deploy\n"; 49 | s += "\n"; 50 | console.log(colors.yellow(s)); 51 | } 52 | 53 | async function main() { 54 | const command = process.argv[2]; 55 | 56 | if (process.argv.includes("--dev")) { 57 | throw new Error("Invalid argument --dev"); 58 | } 59 | if (process.argv.includes("--no-compile")) { 60 | throw new Error("Invalid argument --no-compile"); 61 | } 62 | 63 | console.log(colors.magentaBright("command: ", colors.italic(command) || "")); 64 | console.log(); 65 | 66 | if (command === "clean") { 67 | await clean(); 68 | return; 69 | } 70 | 71 | const isDeploy = command === "deploy"; 72 | const isPackage = isDeploy || command === "package" || command === "pack"; 73 | 74 | const publisher = isDeploy ? await startPublishAssets() : undefined; 75 | 76 | if (isDeploy) { 77 | publisher.checkGithubKey(); 78 | } 79 | 80 | if (isPackage) { 81 | await forkAsync(require.resolve("./build.js"), ["--no-compile"]); 82 | } 83 | 84 | if (command && !isDeploy && !isPackage) { 85 | printUsage(); 86 | process.exitCode = 1; 87 | return; 88 | } 89 | 90 | await clean(); 91 | 92 | console.log(colors.cyanBright(`* Building for node ${NODE_VERSIONS.join(" ")}\n`)); 93 | 94 | printSystemInfo(); 95 | 96 | console.log(colors.blueBright("- installing tools")); 97 | console.time("installing tools"); 98 | fs.mkdirSync(TOOLS_DIR, { recursive: true }); 99 | fs.writeFileSync( 100 | path.resolve(TOOLS_DIR, "package.json"), 101 | JSON.stringify({ name: "tools", private: true, dependencies: { n: "latest", "node-gyp": "latest" } }), 102 | ); 103 | await spawnAsync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-save"], { 104 | cwd: TOOLS_DIR, 105 | }); 106 | process.env.npm_config_node_gyp = path.resolve(TOOLS_DIR, "node_modules/node-gyp/bin/node-gyp.js"); 107 | if (!fs.existsSync(process.env.npm_config_node_gyp)) { 108 | process.env.npm_config_node_gyp = require.resolve("node-gyp/bin/node-gyp.js"); 109 | } 110 | console.timeEnd("installing tools"); 111 | 112 | console.log(colors.blueBright("- installing node versions")); 113 | console.time("installing node versions"); 114 | await Promise.all( 115 | NODE_VERSIONS.map((nodeVersion) => spawnAsync(N_EXECUTABLE_PATH, ["i", nodeVersion, "--download"])), 116 | ); 117 | console.timeEnd("installing node versions"); 118 | 119 | console.log(colors.blueBright("- building")); 120 | console.time("building"); 121 | for (const nodeVersion of NODE_VERSIONS) { 122 | console.log(colors.blueBright(`\n- building for node ${nodeVersion}\n`)); 123 | const args = ["run", nodeVersion, path.resolve(ROOT_FOLDER, "node-pre-gyp.js"), "rebuild"]; 124 | if (isPackage) { 125 | args.push("package", "testpackage"); 126 | } 127 | await spawnAsync(N_EXECUTABLE_PATH, args); 128 | 129 | console.log(colors.blueBright("- testing")); 130 | const testArgs = ["run", nodeVersion, require.resolve("./test.js")]; 131 | await spawnAsync(N_EXECUTABLE_PATH, testArgs); 132 | 133 | if (isDeploy) { 134 | console.log(colors.blueBright("- uploading")); 135 | await publisher.upload(); 136 | } 137 | 138 | if (isPackage) { 139 | // Move stage folder to a temporary folder so it does not get clean by node-gyp 140 | await mergeDirs(STAGE_DIR, STAGE_TMP_DIR); 141 | } 142 | } 143 | console.timeEnd("building"); 144 | 145 | if (isPackage) { 146 | // Move the directory back to build/stage 147 | await mergeDirs(STAGE_TMP_DIR, STAGE_DIR); 148 | if (fs.existsSync(STAGE_TMP_DIR)) { 149 | await rmdir(STAGE_TMP_DIR, { recursive: true }); 150 | } 151 | } 152 | } 153 | 154 | runMain(main, "prebuild-local"); 155 | -------------------------------------------------------------------------------- /scripts/precommit.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require("child_process"); 4 | const { runMain } = require("./lib/utils"); 5 | 6 | runMain(() => { 7 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 8 | if (nodeVersion >= 14) { 9 | execSync("npx lint-staged", { stdio: "inherit" }); 10 | } 11 | }, "precommit"); 12 | -------------------------------------------------------------------------------- /scripts/prepush.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require("child_process"); 4 | const fs = require("fs"); 5 | 6 | const { CPP_UNITY_FILE_PATH, runMain } = require("./lib/utils"); 7 | const { unity } = require("./lib/unity"); 8 | 9 | runMain(() => { 10 | const roaringNodeCpp = fs.readFileSync(CPP_UNITY_FILE_PATH, "utf8"); 11 | 12 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 13 | if (nodeVersion >= 14) { 14 | execSync("npx lint-staged"); 15 | } 16 | 17 | const unityResult = unity(); 18 | 19 | console.log(); 20 | 21 | if (unityResult.outputText !== roaringNodeCpp) { 22 | const chalk = require("chalk"); 23 | console.error( 24 | chalk.redBright( 25 | `${chalk.underline.bold( 26 | "ERROR", 27 | )}: ${CPP_UNITY_FILE_PATH} is outdated or not a production version. Run \`npm run build\` before committing and pushing.`, 28 | ), 29 | ); 30 | process.exitCode = 1; 31 | return; 32 | } 33 | 34 | try { 35 | require("../"); 36 | } catch { 37 | const chalk = require("chalk"); 38 | console.error( 39 | chalk.redBright( 40 | `${chalk.underline.bold( 41 | "ERROR", 42 | )}: library could not be loaded. Run \`npm run build\` before committing and pushing.`, 43 | ), 44 | ); 45 | process.exitCode = 1; 46 | return; 47 | } 48 | 49 | execSync("npm run lint", { stdio: "inherit" }); 50 | execSync("npm run test", { stdio: "inherit" }); 51 | }, "prepush"); 52 | -------------------------------------------------------------------------------- /scripts/rebuild.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { runMain } = require("./lib/utils"); 4 | const { build } = require("./build"); 5 | 6 | process.argv.push("rebuild"); 7 | 8 | runMain(build, "rebuild"); 9 | -------------------------------------------------------------------------------- /scripts/system-info.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const colors = require("chalk"); 4 | const os = require("os"); 5 | 6 | function getSystemInfo() { 7 | const cpus = os.cpus(); 8 | let physicalCpuCount; 9 | 10 | try { 11 | physicalCpuCount = require("physical-cpu-count"); 12 | } catch (e) { 13 | // Ignore error 14 | } 15 | if (!physicalCpuCount) { 16 | physicalCpuCount = Math.max(1, cpus.length / 2); 17 | } 18 | 19 | const systemInfo = { 20 | physicalCpuCount, 21 | }; 22 | 23 | systemInfo.print = function print() { 24 | console.log(); 25 | console.log( 26 | colors.whiteBright("Platform"), 27 | ":", 28 | colors.cyanBright(os.type()), 29 | colors.greenBright(os.release()), 30 | colors.yellowBright(process.arch), 31 | ); 32 | console.log(colors.whiteBright("CPU "), ":", colors.cyanBright(cpus[0].model)); 33 | console.log( 34 | colors.whiteBright("Cores "), 35 | ":", 36 | colors.cyanBright(physicalCpuCount), 37 | colors.cyan("physical"), 38 | "-", 39 | colors.greenBright(cpus.length), 40 | colors.green("logical"), 41 | ); 42 | console.log( 43 | colors.whiteBright("Memory "), 44 | ":", 45 | colors.cyanBright((os.totalmem() / 1073741824).toFixed(2)), 46 | colors.cyan("GB"), 47 | ); 48 | console.log( 49 | colors.whiteBright("NodeJS "), 50 | ":", 51 | colors.cyanBright(process.version), 52 | "-", 53 | colors.green("V8"), 54 | colors.greenBright(`v${process.versions.v8}`), 55 | ); 56 | console.log(); 57 | }; 58 | 59 | return systemInfo; 60 | } 61 | 62 | module.exports = getSystemInfo(); 63 | 64 | if (require.main === module) { 65 | module.exports.print(); 66 | } 67 | -------------------------------------------------------------------------------- /scripts/test-memory-leaks.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node --expose-gc 2 | 3 | /* eslint-disable node/no-unsupported-features/node-builtins */ 4 | /* eslint-disable no-console */ 5 | /* global gc */ 6 | 7 | const { runMain } = require("./lib/utils"); 8 | 9 | const _gc = typeof gc !== "undefined" ? gc : () => {}; 10 | 11 | async function testMemoryLeaks() { 12 | _gc(); 13 | _gc(); 14 | 15 | const { RoaringBitmap32, getRoaringUsedMemory } = require("../"); 16 | 17 | _gc(); 18 | _gc(); 19 | 20 | console.log(); 21 | console.log("RoaringUsedMemory", getRoaringUsedMemory()); 22 | console.log("RoaringBitmap32.getInstancesCount", RoaringBitmap32.getInstancesCount()); 23 | 24 | _gc(); 25 | _gc(); 26 | 27 | let a = new RoaringBitmap32([1, 2, 3]); 28 | let b = new RoaringBitmap32([1, 2]); 29 | 30 | console.time("buildup"); 31 | 32 | const operation = (i) => { 33 | switch (i % 4) { 34 | case 1: 35 | return RoaringBitmap32.and(a, b); 36 | case 2: 37 | return RoaringBitmap32.xor(a, b); 38 | case 3: 39 | return RoaringBitmap32.andNot(a, b); 40 | default: 41 | return RoaringBitmap32.or(a, b); 42 | } 43 | }; 44 | 45 | let promises = []; 46 | for (let i = 0; i < 10000; i++) { 47 | const bmp = operation(i); 48 | if (i < 5) { 49 | promises.push(bmp.toUint32ArrayAsync()); 50 | } 51 | promises.push(bmp.serializeAsync(i & 4 ? "croaring" : i & 8 ? "portable" : "unsafe_frozen_croaring")); 52 | } 53 | 54 | console.log(); 55 | console.log("RoaringUsedMemory", getRoaringUsedMemory()); 56 | console.log("RoaringBitmap32.getInstancesCount", RoaringBitmap32.getInstancesCount()); 57 | console.log(); 58 | 59 | await Promise.all(promises); 60 | promises = null; 61 | 62 | console.timeEnd("buildup"); 63 | console.log(); 64 | 65 | _gc(); 66 | _gc(); 67 | 68 | console.table(process.memoryUsage()); 69 | process.stdout.write("\nallocating"); 70 | console.time("allocation"); 71 | 72 | _gc(); 73 | _gc(); 74 | 75 | promises = []; 76 | for (let i = 0; i < 10000000; i++) { 77 | const bmp = operation(i); 78 | if (i % 100000 === 0) { 79 | process.stdout.write("."); 80 | _gc(); 81 | _gc(); 82 | promises.push(bmp.serializeAsync("croaring").then(() => undefined)); 83 | } 84 | } 85 | await Promise.all(promises); 86 | promises = null; 87 | process.stdout.write("\n"); 88 | console.timeEnd("allocation"); 89 | 90 | console.log(); 91 | console.log("RoaringUsedMemory", getRoaringUsedMemory()); 92 | console.log("RoaringBitmap32.getInstancesCount", RoaringBitmap32.getInstancesCount()); 93 | console.log(); 94 | 95 | a = null; 96 | b = null; 97 | 98 | const promise = new Promise((resolve, reject) => { 99 | for (let i = 0; i < 10; ++i) { 100 | _gc(); 101 | } 102 | setTimeout(() => { 103 | for (let i = 0; i < 10; ++i) { 104 | _gc(); 105 | } 106 | setTimeout(() => { 107 | for (let i = 0; i < 10; ++i) { 108 | _gc(); 109 | } 110 | console.table(process.memoryUsage()); 111 | console.log(); 112 | console.log("RoaringUsedMemory", getRoaringUsedMemory()); 113 | console.log("RoaringBitmap32.getInstancesCount", RoaringBitmap32.getInstancesCount()); 114 | console.log(); 115 | 116 | setTimeout(() => { 117 | if (getRoaringUsedMemory() !== 0) { 118 | reject(new Error(`Memory leak detected. ${getRoaringUsedMemory()} bytes are still allocated.`)); 119 | } 120 | 121 | if (RoaringBitmap32.getInstancesCount() !== 0) { 122 | reject( 123 | new Error(`Memory leak detected. ${RoaringBitmap32.getInstancesCount()} instances are still allocated.`), 124 | ); 125 | } 126 | resolve(); 127 | }); 128 | }, 150); 129 | }, 150); 130 | }); 131 | 132 | await promise; 133 | } 134 | 135 | runMain(testMemoryLeaks, "test-memory-leaks"); 136 | -------------------------------------------------------------------------------- /scripts/test.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const path = require("path"); 4 | const { runMain } = require("./lib/utils"); 5 | 6 | require("tsx/cjs"); 7 | 8 | runMain(() => { 9 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 10 | if (nodeVersion < 14) { 11 | const url = require("url"); 12 | if (!url.pathToFileURL) { 13 | url.pathToFileURL = (filePath) => { 14 | let pathName = path.resolve(filePath).replace(/\\/g, "/"); 15 | if (!pathName.startsWith("/")) { 16 | pathName = `/${pathName}`; 17 | } 18 | return encodeURI(`file://${pathName}`).replace(/[?#]/g, encodeURIComponent); 19 | }; 20 | } 21 | 22 | require("mocha/lib/nodejs/esm-utils.js").doImport = (v) => { 23 | if (typeof v === "object") { 24 | v = url.fileURLToPath(v); 25 | } 26 | return new Promise((resolve) => resolve(require(v))); 27 | }; 28 | } 29 | 30 | const { print: printSystemInfo } = require("./system-info.js"); 31 | 32 | process.argv.push("test/**/*.test.ts"); 33 | process.argv.push("test/*.test.ts"); 34 | 35 | printSystemInfo(); 36 | 37 | require("mocha/bin/mocha"); 38 | }, "test"); 39 | -------------------------------------------------------------------------------- /scripts/update-roaring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | git submodule update --init --recursive 6 | 7 | cd submodules/CRoaring 8 | 9 | git checkout master 10 | git pull 11 | 12 | cd ../.. 13 | -------------------------------------------------------------------------------- /src/cpp/RoaringBitmap32.h: -------------------------------------------------------------------------------- 1 | #ifndef __ROARINGBITMAP32__H__ 2 | #define __ROARINGBITMAP32__H__ 3 | 4 | #include "v8utils.h" 5 | #include "serialization-format.h" 6 | #include "WorkerError.h" 7 | 8 | using namespace roaring; 9 | using namespace roaring::api; 10 | 11 | class RoaringBitmap32; 12 | 13 | typedef roaring_bitmap_t * roaring_bitmap_t_ptr; 14 | 15 | class RoaringBitmap32 final : public ObjectWrap { 16 | public: 17 | static const constexpr uint64_t OBJECT_TOKEN = 0x21524F4152330000; 18 | 19 | static const constexpr int64_t FROZEN_COUNTER_SOFT_FROZEN = -1; 20 | static const constexpr int64_t FROZEN_COUNTER_HARD_FROZEN = -2; 21 | 22 | roaring_bitmap_t * roaring; 23 | int64_t sizeCache; 24 | int64_t _version; 25 | int64_t frozenCounter; 26 | RoaringBitmap32 * const readonlyViewOf; 27 | v8::Persistent> readonlyViewPersistent; 28 | v8::Persistent> persistent; 29 | v8utils::TypedArrayContent frozenStorage; 30 | 31 | inline bool isEmpty() const { 32 | if (this->sizeCache == 0) { 33 | return true; 34 | } 35 | if (this->readonlyViewOf != nullptr) { 36 | return this->readonlyViewOf->isEmpty(); 37 | } 38 | const roaring_bitmap_t_ptr roaring = this->roaring; 39 | bool result = roaring == nullptr || roaring_bitmap_is_empty(roaring); 40 | if (result) { 41 | const_cast(this)->sizeCache = 0; 42 | } 43 | return result; 44 | } 45 | 46 | inline size_t getSize() const { 47 | int64_t size = this->sizeCache; 48 | if (size < 0) { 49 | if (this->readonlyViewOf != nullptr) { 50 | return this->readonlyViewOf->getSize(); 51 | } 52 | const roaring_bitmap_t_ptr roaring = this->roaring; 53 | size = roaring != nullptr ? (int64_t)roaring_bitmap_get_cardinality(roaring) : 0; 54 | const_cast(this)->sizeCache = size; 55 | } 56 | return (size_t)size; 57 | } 58 | 59 | inline bool isFrozen() const { return this->frozenCounter != 0; } 60 | inline bool isFrozenHard() const { return this->frozenCounter > 0 || this->frozenCounter == FROZEN_COUNTER_HARD_FROZEN; } 61 | inline bool isFrozenForever() const { return this->frozenCounter < 0; } 62 | 63 | inline void beginFreeze() { 64 | if (this->frozenCounter >= 0) { 65 | ++this->frozenCounter; 66 | } 67 | } 68 | 69 | inline void endFreeze() { 70 | if (this->frozenCounter > 0) { 71 | --this->frozenCounter; 72 | } 73 | } 74 | 75 | inline int64_t getVersion() const { return this->_version; } 76 | 77 | inline void invalidate() { 78 | this->sizeCache = -1; 79 | ++this->_version; 80 | } 81 | 82 | inline bool roaring_bitmap_t_is_frozen(const roaring_bitmap_t * r) { 83 | return r->high_low_container.flags & ROARING_FLAG_FROZEN; 84 | } 85 | 86 | bool replaceBitmapInstance(v8::Isolate * isolate, roaring_bitmap_t * newInstance) { 87 | roaring_bitmap_t * oldInstance = this->roaring; 88 | if (oldInstance == newInstance) { 89 | return false; 90 | } 91 | if (oldInstance != nullptr) { 92 | roaring_bitmap_free(oldInstance); 93 | } 94 | if (newInstance != nullptr && roaring_bitmap_t_is_frozen(newInstance)) { 95 | this->frozenCounter = RoaringBitmap32::FROZEN_COUNTER_HARD_FROZEN; 96 | } 97 | this->roaring = newInstance; 98 | this->invalidate(); 99 | return true; 100 | } 101 | 102 | explicit RoaringBitmap32(AddonData * addonData, RoaringBitmap32 * readonlyViewOf) : 103 | ObjectWrap(addonData), 104 | roaring(readonlyViewOf->roaring), 105 | sizeCache(-1), 106 | frozenCounter(RoaringBitmap32::FROZEN_COUNTER_HARD_FROZEN), 107 | readonlyViewOf(readonlyViewOf->readonlyViewOf ? readonlyViewOf->readonlyViewOf : readonlyViewOf) { 108 | gcaware_addAllocatedMemory(sizeof(RoaringBitmap32)); 109 | } 110 | 111 | explicit RoaringBitmap32(AddonData * addonData, uint32_t capacity) : 112 | ObjectWrap(addonData), 113 | roaring(roaring_bitmap_create_with_capacity(capacity)), 114 | sizeCache(0), 115 | _version(0), 116 | frozenCounter(0), 117 | readonlyViewOf(nullptr) { 118 | ++addonData->RoaringBitmap32_instances; 119 | gcaware_addAllocatedMemory(sizeof(RoaringBitmap32)); 120 | } 121 | 122 | ~RoaringBitmap32() { 123 | gcaware_removeAllocatedMemory(sizeof(RoaringBitmap32)); 124 | if (!this->readonlyViewOf) { 125 | --this->addonData->RoaringBitmap32_instances; 126 | if (this->roaring != nullptr) { 127 | roaring_bitmap_free(this->roaring); 128 | } 129 | if (this->frozenStorage.data != nullptr && this->frozenStorage.length == std::numeric_limits::max()) { 130 | gcaware_aligned_free(this->frozenStorage.data); 131 | } 132 | } 133 | if (!this->persistent.IsEmpty()) { 134 | this->persistent.ClearWeak(); 135 | } 136 | } 137 | }; 138 | 139 | #endif 140 | -------------------------------------------------------------------------------- /src/cpp/RoaringBitmap32BufferedIterator.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_ROARING_BITMAP_32_BUFFERED_ITERATOR_ 2 | #define ROARING_NODE_ROARING_BITMAP_32_BUFFERED_ITERATOR_ 3 | 4 | #include "RoaringBitmap32.h" 5 | 6 | class RoaringBitmap32BufferedIterator final : public ObjectWrap { 7 | public: 8 | static constexpr const uint64_t OBJECT_TOKEN = 0x21524F4152490000; 9 | 10 | enum { allocatedMemoryDelta = 1024 }; 11 | 12 | roaring_uint32_iterator_t it; 13 | bool reversed; 14 | RoaringBitmap32 * bitmapInstance; 15 | int64_t bitmapVersion; 16 | v8utils::TypedArrayContent bufferContent; 17 | 18 | v8::Persistent> bitmap; 19 | v8::Persistent> persistent; 20 | 21 | explicit RoaringBitmap32BufferedIterator(AddonData * addonData, bool reversed) : 22 | ObjectWrap(addonData), reversed(reversed), bitmapInstance(nullptr) { 23 | this->it.parent = nullptr; 24 | this->it.has_value = false; 25 | gcaware_addAllocatedMemory(sizeof(RoaringBitmap32BufferedIterator)); 26 | } 27 | 28 | ~RoaringBitmap32BufferedIterator() { 29 | this->destroy(); 30 | gcaware_removeAllocatedMemory(sizeof(RoaringBitmap32BufferedIterator)); 31 | } 32 | 33 | inline uint32_t _fill() { 34 | uint32_t n; 35 | if (this->reversed) { 36 | size_t size = this->bufferContent.length; 37 | auto * data = this->bufferContent.data; 38 | for (n = 0; n < size && this->it.has_value; ++n) { 39 | data[n] = this->it.current_value; 40 | roaring_uint32_iterator_previous(&this->it); 41 | } 42 | } else { 43 | n = roaring_uint32_iterator_read(&this->it, this->bufferContent.data, this->bufferContent.length); 44 | } 45 | if (n == 0) { 46 | this->bitmapInstance = nullptr; 47 | this->bufferContent.reset(); 48 | this->bitmap.Reset(); 49 | } 50 | return n; 51 | } 52 | 53 | private: 54 | void destroy() { 55 | this->bitmap.Reset(); 56 | if (!this->persistent.IsEmpty()) { 57 | this->persistent.ClearWeak(); 58 | this->persistent.Reset(); 59 | } 60 | } 61 | }; 62 | 63 | void RoaringBitmap32BufferedIterator_fill(const v8::FunctionCallbackInfo & info) { 64 | v8::Isolate * isolate = info.GetIsolate(); 65 | 66 | RoaringBitmap32BufferedIterator * instance = 67 | ObjectWrap::TryUnwrap(info.Holder(), isolate); 68 | 69 | RoaringBitmap32 * bitmapInstance = instance ? instance->bitmapInstance : nullptr; 70 | 71 | if (bitmapInstance == nullptr) { 72 | return info.GetReturnValue().Set(0U); 73 | } 74 | 75 | if (bitmapInstance->getVersion() != instance->bitmapVersion) { 76 | return v8utils::throwError(isolate, "RoaringBitmap32 iterator - bitmap changed while iterating"); 77 | } 78 | 79 | return info.GetReturnValue().Set(instance->_fill()); 80 | } 81 | 82 | void RoaringBitmap32BufferedIterator_WeakCallback(v8::WeakCallbackInfo const & info) { 83 | RoaringBitmap32BufferedIterator * p = info.GetParameter(); 84 | if (p != nullptr) { 85 | p->~RoaringBitmap32BufferedIterator(); 86 | bare_aligned_free(p); 87 | } 88 | } 89 | 90 | void RoaringBitmap32BufferedIterator_New(const v8::FunctionCallbackInfo & info) { 91 | v8::Isolate * isolate = info.GetIsolate(); 92 | 93 | if (!info.IsConstructCall()) { 94 | return v8utils::throwTypeError(isolate, "RoaringBitmap32BufferedIterator::ctor - needs to be called with new"); 95 | } 96 | 97 | auto holder = info.Holder(); 98 | 99 | if (info.Length() < 2) { 100 | return v8utils::throwTypeError(isolate, "RoaringBitmap32BufferedIterator::ctor - needs 2 or 3 arguments"); 101 | } 102 | 103 | AddonData * addonData = AddonData::get(info); 104 | if (addonData == nullptr) { 105 | return v8utils::throwTypeError(isolate, ERROR_INVALID_OBJECT); 106 | } 107 | 108 | RoaringBitmap32 * bitmapInstance = ObjectWrap::TryUnwrap(info[0], isolate); 109 | if (bitmapInstance == nullptr) { 110 | return v8utils::throwTypeError( 111 | isolate, "RoaringBitmap32BufferedIterator::ctor - first argument must be of type RoaringBitmap32"); 112 | } 113 | 114 | bool reversed = info.Length() > 2 && info[2]->BooleanValue(isolate); 115 | 116 | auto bufferObject = info[1]; 117 | 118 | if (!bufferObject->IsUint32Array()) { 119 | return v8utils::throwTypeError( 120 | isolate, "RoaringBitmap32BufferedIterator::ctor - second argument must be of type Uint32Array"); 121 | } 122 | 123 | const v8utils::TypedArrayContent bufferContent(isolate, bufferObject); 124 | if (!bufferContent.data || bufferContent.length < 1) { 125 | return v8utils::throwTypeError(isolate, "RoaringBitmap32BufferedIterator::ctor - invalid Uint32Array buffer"); 126 | } 127 | 128 | auto * instanceMemory = bare_aligned_malloc(16, sizeof(RoaringBitmap32BufferedIterator)); 129 | auto * instance = instanceMemory ? new (instanceMemory) RoaringBitmap32BufferedIterator(addonData, reversed) : nullptr; 130 | if (instance == nullptr) { 131 | return v8utils::throwError(isolate, "RoaringBitmap32BufferedIterator::ctor - allocation failed"); 132 | } 133 | 134 | int indices[2] = {0, 1}; 135 | void * values[2] = {instance, (void *)(RoaringBitmap32BufferedIterator::OBJECT_TOKEN)}; 136 | holder->SetAlignedPointerInInternalFields(2, indices, values); 137 | 138 | info.GetReturnValue().Set(holder); 139 | 140 | instance->persistent.Reset(isolate, holder); 141 | instance->persistent.SetWeak(instance, RoaringBitmap32BufferedIterator_WeakCallback, v8::WeakCallbackType::kParameter); 142 | 143 | auto context = isolate->GetCurrentContext(); 144 | 145 | instance->bitmapInstance = bitmapInstance; 146 | instance->bitmapVersion = bitmapInstance->getVersion(); 147 | 148 | auto a0Maybe = info[0]->ToObject(context); 149 | v8::Local a0; 150 | if (!a0Maybe.ToLocal(&a0)) { 151 | return v8utils::throwError(isolate, "RoaringBitmap32BufferedIterator::ctor - allocation failed"); 152 | } 153 | instance->bitmap.Reset(isolate, a0); 154 | 155 | auto a1Maybe = bufferObject->ToObject(context); 156 | v8::Local a1; 157 | if (!a1Maybe.ToLocal(&a1)) { 158 | return v8utils::throwError(isolate, "RoaringBitmap32BufferedIterator::ctor - allocation failed"); 159 | } 160 | 161 | instance->bufferContent.set(isolate, bufferObject); 162 | 163 | if (reversed) { 164 | roaring_iterator_init_last(bitmapInstance->roaring, &instance->it); 165 | } else { 166 | roaring_iterator_init(bitmapInstance->roaring, &instance->it); 167 | } 168 | 169 | uint32_t n = instance->_fill(); 170 | 171 | if (holder->Set(context, addonData->strings.n.Get(isolate), v8::Uint32::NewFromUnsigned(isolate, n)).IsNothing()) { 172 | return v8utils::throwError(isolate, "RoaringBitmap32BufferedIterator::ctor - instantiation failed"); 173 | } 174 | } 175 | 176 | void RoaringBitmap32BufferedIterator_Init(v8::Local exports, AddonData * addonData) { 177 | v8::Isolate * isolate = v8::Isolate::GetCurrent(); 178 | 179 | auto className = NEW_LITERAL_V8_STRING(isolate, "RoaringBitmap32BufferedIterator", v8::NewStringType::kInternalized); 180 | 181 | v8::Local ctor = 182 | v8::FunctionTemplate::New(isolate, RoaringBitmap32BufferedIterator_New, addonData->external.Get(isolate)); 183 | 184 | ctor->SetClassName(className); 185 | ctor->InstanceTemplate()->SetInternalFieldCount(2); 186 | 187 | addonData->RoaringBitmap32BufferedIterator_constructorTemplate.Set(isolate, ctor); 188 | 189 | NODE_SET_PROTOTYPE_METHOD(ctor, "fill", RoaringBitmap32BufferedIterator_fill); 190 | 191 | auto ctorFunctionMaybe = ctor->GetFunction(isolate->GetCurrentContext()); 192 | v8::Local ctorFunction; 193 | 194 | if (!ctorFunctionMaybe.ToLocal(&ctorFunction)) { 195 | return v8utils::throwError(isolate, "Failed to instantiate RoaringBitmap32BufferedIterator"); 196 | } 197 | 198 | addonData->RoaringBitmap32BufferedIterator_constructor.Set(isolate, ctorFunction); 199 | v8utils::defineHiddenField(isolate, exports, "RoaringBitmap32BufferedIterator", ctorFunction); 200 | } 201 | 202 | #endif // ROARING_NODE_ROARING_BITMAP_32_BUFFERED_ITERATOR_ 203 | -------------------------------------------------------------------------------- /src/cpp/WorkerError.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_WORKER_ERROR_ 2 | #define ROARING_NODE_WORKER_ERROR_ 3 | 4 | #include "includes.h" 5 | #include "v8utils.h" 6 | 7 | struct WorkerError { 8 | const char * msg; 9 | const char * syscall; 10 | int errorno; 11 | std::string path; 12 | 13 | explicit WorkerError() : msg(nullptr), syscall(nullptr), errorno(0) {} 14 | 15 | explicit WorkerError(const char * msg) : msg(msg), syscall(nullptr), errorno(0) {} 16 | 17 | explicit WorkerError(int errorno, const char * syscall, const std::string & path) : 18 | msg(nullptr), syscall(syscall), errorno(errorno ? errorno : 5), path(path) {} 19 | 20 | inline bool hasError() const { return (msg != nullptr && msg[0] != '\0') || errorno != 0; } 21 | 22 | static WorkerError from_errno(const char * syscall, const std::string & path) { 23 | int errorno = errno; 24 | errno = 0; 25 | return WorkerError(errorno, syscall, path); 26 | } 27 | 28 | v8::Local newV8Error(v8::Isolate * isolate) const { 29 | v8::EscapableHandleScope handle_scope(isolate); 30 | v8::Local output; 31 | if (this->errorno) { 32 | output = node::ErrnoException( 33 | isolate, this->errorno, this->syscall, this->msg && this->msg[0] ? this->msg : nullptr, this->path.c_str()); 34 | } else { 35 | const char * msg = this->msg && this->msg[0] ? this->msg : "Invalid operation"; 36 | v8::MaybeLocal message = v8::String::NewFromUtf8(isolate, msg, v8::NewStringType::kInternalized); 37 | output = v8::Exception::Error(message.IsEmpty() ? v8::String::Empty(isolate) : message.ToLocalChecked()); 38 | } 39 | return handle_scope.Escape(output); 40 | } 41 | }; 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /src/cpp/addon-data.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_ADDON_DATA_ 2 | #define ROARING_NODE_ADDON_DATA_ 3 | 4 | #include "includes.h" 5 | #include "addon-strings.h" 6 | 7 | template 8 | inline void ignoreMaybeResult(v8::Maybe) {} 9 | 10 | template 11 | inline void ignoreMaybeResult(v8::MaybeLocal) {} 12 | 13 | class AddonData final { 14 | public: 15 | v8::Isolate * isolate; 16 | 17 | AddonDataStrings strings; 18 | 19 | v8::Eternal Buffer; 20 | v8::Eternal Uint32Array; 21 | v8::Eternal Uint32Array_from; 22 | v8::Eternal Buffer_from; 23 | 24 | std::atomic RoaringBitmap32_instances; 25 | 26 | v8::Eternal RoaringBitmap32_constructorTemplate; 27 | v8::Eternal RoaringBitmap32_constructor; 28 | 29 | v8::Eternal RoaringBitmap32BufferedIterator_constructorTemplate; 30 | v8::Eternal RoaringBitmap32BufferedIterator_constructor; 31 | 32 | v8::Eternal external; 33 | 34 | inline AddonData() : isolate(nullptr), RoaringBitmap32_instances(0) {} 35 | 36 | static inline AddonData * get(const v8::FunctionCallbackInfo & info) { 37 | v8::Local data = info.Data(); 38 | if (data.IsEmpty()) { 39 | return nullptr; 40 | } 41 | v8::Local external = data.As(); 42 | if (external.IsEmpty()) { 43 | return nullptr; 44 | } 45 | return reinterpret_cast(external->Value()); 46 | } 47 | 48 | inline void initialize(v8::Isolate * isolate) { 49 | this->isolate = isolate; 50 | 51 | external.Set(isolate, v8::External::New(isolate, this)); 52 | 53 | this->strings.initialize(isolate); 54 | 55 | auto context = isolate->GetCurrentContext(); 56 | 57 | auto global = context->Global(); 58 | 59 | auto uint32Array = global->Get(context, NEW_LITERAL_V8_STRING(isolate, "Uint32Array", v8::NewStringType::kInternalized)) 60 | .ToLocalChecked() 61 | ->ToObject(context) 62 | .ToLocalChecked(); 63 | 64 | auto buffer = global->Get(context, NEW_LITERAL_V8_STRING(isolate, "Buffer", v8::NewStringType::kInternalized)) 65 | .ToLocalChecked() 66 | .As(); 67 | 68 | this->Buffer.Set(isolate, buffer); 69 | 70 | this->Buffer_from.Set( 71 | isolate, 72 | buffer->Get(context, NEW_LITERAL_V8_STRING(isolate, "from", v8::NewStringType::kInternalized)) 73 | .ToLocalChecked() 74 | .As()); 75 | 76 | this->Uint32Array.Set(isolate, uint32Array); 77 | 78 | this->Uint32Array_from.Set( 79 | isolate, 80 | v8::Local::Cast( 81 | uint32Array->Get(context, NEW_LITERAL_V8_STRING(isolate, "from", v8::NewStringType::kInternalized)) 82 | .ToLocalChecked())); 83 | } 84 | }; 85 | 86 | inline void AddonData_setMethod( 87 | v8::Local recv, const char * name, v8::FunctionCallback callback, AddonData * addonData) { 88 | v8::Isolate * isolate = v8::Isolate::GetCurrent(); 89 | v8::HandleScope handle_scope(isolate); 90 | v8::Local context = isolate->GetCurrentContext(); 91 | v8::Local t = v8::FunctionTemplate::New(isolate, callback, addonData->external.Get(isolate)); 92 | v8::Local fn = t->GetFunction(context).ToLocalChecked(); 93 | v8::Local fn_name = v8::String::NewFromUtf8(isolate, name, v8::NewStringType::kInternalized).ToLocalChecked(); 94 | fn->SetName(fn_name); 95 | ignoreMaybeResult(recv->Set(context, fn_name, fn)); 96 | } 97 | 98 | #endif 99 | -------------------------------------------------------------------------------- /src/cpp/addon-strings.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_ADDON_STRINGS_ 2 | #define ROARING_NODE_ADDON_STRINGS_ 3 | 4 | #include "includes.h" 5 | 6 | const char * const ERROR_FROZEN = "This bitmap is frozen and cannot be modified"; 7 | const char * const ERROR_INVALID_OBJECT = "Invalid RoaringBitmap32 object"; 8 | 9 | class AddonDataStrings final { 10 | public: 11 | v8::Eternal n; 12 | v8::Eternal readonly; 13 | v8::Eternal RoaringBitmap32; 14 | v8::Eternal symbol_rnshared; 15 | 16 | v8::Eternal OperationFailed; 17 | v8::Eternal Comma; 18 | 19 | inline void initialize(v8::Isolate * isolate) { 20 | literal(isolate, this->n, "n"); 21 | literal(isolate, this->readonly, "readonly"); 22 | literal(isolate, this->RoaringBitmap32, "RoaringBitmap32"); 23 | literal(isolate, this->Comma, ","); 24 | 25 | literal(isolate, this->OperationFailed, "Operation failed"); 26 | 27 | symbol_rnshared.Set( 28 | isolate, v8::Symbol::ForApi(isolate, NEW_LITERAL_V8_STRING(isolate, "rnshared", v8::NewStringType::kInternalized))); 29 | } 30 | 31 | private: 32 | template 33 | static void literal(v8::Isolate * isolate, v8::Eternal & result, const char (&literal)[N]) { 34 | result.Set(isolate, NEW_LITERAL_V8_STRING(isolate, literal, v8::NewStringType::kInternalized)); 35 | } 36 | }; 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /src/cpp/aligned-buffers.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_ALIGNED_BUFFERS_ 2 | #define ROARING_NODE_ALIGNED_BUFFERS_ 3 | 4 | #include "v8utils.h" 5 | #include "memory.h" 6 | 7 | void _bufferAlignedAlloc(const v8::FunctionCallbackInfo & info, bool unsafe, bool shared) { 8 | v8::Isolate * isolate = info.GetIsolate(); 9 | v8::HandleScope scope(isolate); 10 | 11 | int64_t size; 12 | int32_t alignment = 32; 13 | if ( 14 | info.Length() < 1 || !info[0]->IsNumber() || !info[0]->IntegerValue(isolate->GetCurrentContext()).To(&size) || 15 | size < 0) { 16 | return v8utils::throwTypeError(isolate, "Buffer size must be a positive integer"); 17 | } 18 | 19 | if (info.Length() >= 2 && !info[1]->IsUndefined()) { 20 | if (!info[1]->IsNumber() || !info[1]->Int32Value(isolate->GetCurrentContext()).To(&alignment) || alignment <= 0) { 21 | return v8utils::throwTypeError(isolate, "Buffer alignment must be a positive integer"); 22 | } 23 | } 24 | 25 | if (size < 0) { 26 | size = 0; 27 | } 28 | 29 | if ((uint64_t)size > node::Buffer::kMaxLength || (uint64_t)size + alignment >= node::Buffer::kMaxLength) { 30 | return v8utils::throwTypeError(isolate, "Buffer size is too large"); 31 | } 32 | 33 | if (alignment > 1024) { 34 | return v8utils::throwTypeError(isolate, "Buffer alignment is too large"); 35 | } 36 | 37 | void * ptr = bare_aligned_malloc(alignment, size); 38 | if (!ptr) { 39 | return v8utils::throwError(isolate, "Buffer memory allocation failed"); 40 | } 41 | 42 | if (!unsafe) { 43 | memset(ptr, 0, size); 44 | } 45 | 46 | if (shared) { 47 | AddonData * addonData = AddonData::get(info); 48 | if (addonData == nullptr) { 49 | bare_aligned_free(ptr); 50 | return v8utils::throwError(isolate, "Invalid call"); 51 | } 52 | 53 | auto backingStore = v8::SharedArrayBuffer::NewBackingStore(ptr, (size_t)size, bare_aligned_free_callback2, nullptr); 54 | if (!backingStore) { 55 | bare_aligned_free(ptr); 56 | return v8utils::throwError(isolate, "Buffer creation failed"); 57 | } 58 | auto sharedBuf = v8::SharedArrayBuffer::New(isolate, std::move(backingStore)); 59 | v8::Local bufferObj; 60 | if (sharedBuf.IsEmpty() || !v8utils::bufferFromArrayBuffer(isolate, addonData, sharedBuf, 0, size, bufferObj)) { 61 | bare_aligned_free(ptr); 62 | return v8utils::throwError(isolate, "Buffer creation failed"); 63 | } 64 | info.GetReturnValue().Set(bufferObj); 65 | return; 66 | } 67 | 68 | v8::MaybeLocal bufferMaybe; 69 | bufferMaybe = node::Buffer::New(isolate, (char *)ptr, size, bare_aligned_free_callback, nullptr); 70 | v8::Local bufferValue; 71 | if (!bufferMaybe.ToLocal(&bufferValue) || bufferValue.IsEmpty()) { 72 | bare_aligned_free(ptr); 73 | return v8utils::throwError(isolate, "Buffer creation failed"); 74 | } 75 | 76 | info.GetReturnValue().Set(bufferValue); 77 | } 78 | 79 | void bufferAlignedAlloc(const v8::FunctionCallbackInfo & info) { _bufferAlignedAlloc(info, false, false); } 80 | 81 | void bufferAlignedAllocUnsafe(const v8::FunctionCallbackInfo & info) { _bufferAlignedAlloc(info, true, false); } 82 | 83 | void bufferAlignedAllocShared(const v8::FunctionCallbackInfo & info) { _bufferAlignedAlloc(info, false, true); } 84 | 85 | void bufferAlignedAllocSharedUnsafe(const v8::FunctionCallbackInfo & info) { 86 | _bufferAlignedAlloc(info, true, true); 87 | } 88 | 89 | void isBufferAligned(const v8::FunctionCallbackInfo & info) { 90 | v8::Isolate * isolate = info.GetIsolate(); 91 | v8::HandleScope scope(isolate); 92 | 93 | if (info.Length() < 1) { 94 | return info.GetReturnValue().Set(false); 95 | } 96 | 97 | // Second parameter must be a positive integer 98 | int32_t alignment = 32; 99 | if (info.Length() >= 2 && !info[1]->IsUndefined()) { 100 | if (!info[1]->IsNumber() || !info[1]->Int32Value(isolate->GetCurrentContext()).To(&alignment) || alignment < 0) { 101 | return info.GetReturnValue().Set(false); 102 | } 103 | } 104 | 105 | if (alignment == 0) { 106 | return info.GetReturnValue().Set(true); 107 | } 108 | 109 | v8utils::TypedArrayContent content(isolate, info[0]); 110 | info.GetReturnValue().Set(is_pointer_aligned(content.data, alignment)); 111 | } 112 | 113 | void AlignedBuffers_Init(v8::Local exports, AddonData * addonData) { 114 | AddonData_setMethod(exports, "bufferAlignedAlloc", bufferAlignedAlloc, addonData); 115 | AddonData_setMethod(exports, "bufferAlignedAllocUnsafe", bufferAlignedAllocUnsafe, addonData); 116 | AddonData_setMethod(exports, "bufferAlignedAllocShared", bufferAlignedAllocShared, addonData); 117 | AddonData_setMethod(exports, "bufferAlignedAllocSharedUnsafe", bufferAlignedAllocSharedUnsafe, addonData); 118 | AddonData_setMethod(exports, "isBufferAligned", isBufferAligned, addonData); 119 | } 120 | 121 | #endif // ROARING_NODE_ALIGNED_BUFFERS_ 122 | -------------------------------------------------------------------------------- /src/cpp/croaring.cpp: -------------------------------------------------------------------------------- 1 | #define printf(...) ((void)0) 2 | #define fprintf(...) ((void)0) 3 | 4 | #if defined(__clang__) 5 | # pragma clang diagnostic push 6 | # pragma clang diagnostic ignored "-Wunused-variable" 7 | # pragma clang diagnostic ignored "-Wunused-but-set-variable" 8 | # pragma clang diagnostic ignored "-Wunused-but-set-parameter" 9 | # pragma clang diagnostic ignored "-Wmissing-field-initializers" 10 | # pragma clang diagnostic ignored "-Wunused-function" 11 | #elif defined(__GNUC__) 12 | # pragma GCC diagnostic push 13 | # pragma GCC diagnostic ignored "-Wunused-variable" 14 | # pragma GCC diagnostic ignored "-Wunused-but-set-variable" 15 | # pragma GCC diagnostic ignored "-Wunused-but-set-parameter" 16 | # pragma GCC diagnostic ignored "-Wmissing-field-initializers" 17 | # pragma GCC diagnostic ignored "-Wunused-function" 18 | #elif defined(_MSC_VER) 19 | # pragma warning(push) 20 | # pragma warning(disable: 4244) // possible loss of data 21 | #endif 22 | 23 | #ifdef small 24 | # undef small // on windows this seems to be defined to something... 25 | #endif 26 | 27 | #include "../../submodules/CRoaring/src/isadetection.c" 28 | #include "../../submodules/CRoaring/src/array_util.c" 29 | #include "../../submodules/CRoaring/src/bitset_util.c" 30 | #include "../../submodules/CRoaring/src/bitset.c" 31 | #include "../../submodules/CRoaring/src/containers/array.c" 32 | #include "../../submodules/CRoaring/src/containers/bitset.c" 33 | #include "../../submodules/CRoaring/src/containers/containers.c" 34 | #include "../../submodules/CRoaring/src/containers/convert.c" 35 | #include "../../submodules/CRoaring/src/containers/mixed_intersection.c" 36 | #include "../../submodules/CRoaring/src/containers/mixed_union.c" 37 | #include "../../submodules/CRoaring/src/containers/mixed_equal.c" 38 | #include "../../submodules/CRoaring/src/containers/mixed_subset.c" 39 | #include "../../submodules/CRoaring/src/containers/mixed_negation.c" 40 | #include "../../submodules/CRoaring/src/containers/mixed_xor.c" 41 | #include "../../submodules/CRoaring/src/containers/mixed_andnot.c" 42 | #include "../../submodules/CRoaring/src/containers/run.c" 43 | #include "../../submodules/CRoaring/src/memory.c" 44 | #include "../../submodules/CRoaring/src/roaring.c" 45 | #include "../../submodules/CRoaring/src/roaring_priority_queue.c" 46 | #include "../../submodules/CRoaring/src/roaring_array.c" 47 | 48 | #if defined(__clang__) 49 | # pragma clang diagnostic pop 50 | #elif defined(__GNUC__) 51 | # pragma GCC diagnostic pop 52 | #elif defined(_MSC_VER) 53 | # pragma warning(pop) 54 | #endif 55 | 56 | #undef printf 57 | #undef fprintf 58 | -------------------------------------------------------------------------------- /src/cpp/croaring.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_CROARING_ 2 | #define ROARING_NODE_CROARING_ 3 | 4 | #include "includes.h" 5 | #include "memory.h" 6 | 7 | #include "../../submodules/CRoaring/include/roaring/roaring.h" 8 | 9 | void croaringMemoryInitialize() { 10 | static std::atomic _roaringBitMemoryInitialized{false}; 11 | static std::mutex _roaringBitMemoryInitializedMutex; 12 | 13 | if (!_roaringBitMemoryInitialized) { 14 | roaring_memory_t roaringMemory; 15 | roaringMemory.malloc = gcaware_malloc; 16 | roaringMemory.realloc = gcaware_realloc; 17 | roaringMemory.calloc = gcaware_calloc; 18 | roaringMemory.free = gcaware_free; 19 | roaringMemory.aligned_malloc = gcaware_aligned_malloc; 20 | roaringMemory.aligned_free = gcaware_aligned_free; 21 | std::lock_guard guard(_roaringBitMemoryInitializedMutex); 22 | if (_roaringBitMemoryInitialized) { 23 | roaring_init_memory_hook(roaringMemory); 24 | _roaringBitMemoryInitialized = true; 25 | } 26 | } 27 | } 28 | 29 | #endif // ROARING_NODE_CROARING_ 30 | -------------------------------------------------------------------------------- /src/cpp/includes.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_INCLUDES_ 2 | #define ROARING_NODE_INCLUDES_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #if defined(__APPLE__) 19 | # include 20 | #else 21 | # include 22 | #endif 23 | 24 | #ifdef _MSC_VER 25 | # define atomicIncrement32(ptr) InterlockedIncrement(ptr) 26 | # define atomicDecrement32(ptr) InterlockedDecrement(ptr) 27 | #else 28 | # define atomicIncrement32(ptr) __sync_add_and_fetch(ptr, 1) 29 | # define atomicDecrement32(ptr) __sync_sub_and_fetch(ptr, 1) 30 | #endif 31 | 32 | #include "croaring.h" 33 | 34 | #if NODE_MAJOR_VERSION > 14 35 | # define NEW_LITERAL_V8_STRING(isolate, str, type) v8::String::NewFromUtf8Literal(isolate, str, type) 36 | #else 37 | # define NEW_LITERAL_V8_STRING(isolate, str, type) v8::String::NewFromUtf8(isolate, str, type).ToLocalChecked() 38 | #endif 39 | 40 | typedef const char * const_char_ptr_t; 41 | 42 | #endif // ROARING_NODE_INCLUDES_ 43 | -------------------------------------------------------------------------------- /src/cpp/main.cpp: -------------------------------------------------------------------------------- 1 | #include "aligned-buffers.h" 2 | #include "RoaringBitmap32-main.h" 3 | #include "RoaringBitmap32BufferedIterator.h" 4 | 5 | using namespace v8; 6 | 7 | #define PREPROCESSOR_CONCAT(a, b) PREPROCESSOR_CONCAT_HELPER(a, b) 8 | #define PREPROCESSOR_CONCAT_HELPER(a, b) a##b 9 | 10 | #define MODULE_WORKER_ENABLED(module_name, registration) \ 11 | extern "C" NODE_MODULE_EXPORT void PREPROCESSOR_CONCAT(node_register_module_v, NODE_MODULE_VERSION)( \ 12 | v8::Local exports, v8::Local module, v8::Local context) { \ 13 | registration(exports); \ 14 | } 15 | 16 | void AddonData_DeleteInstance(void * addonData) { 17 | if (thread_local_isolate == reinterpret_cast(addonData)->isolate) { 18 | thread_local_isolate = nullptr; 19 | } 20 | 21 | delete (AddonData *)addonData; 22 | } 23 | 24 | void InitRoaringNode(Local exports) { 25 | v8::Isolate * isolate = v8::Isolate::GetCurrent(); 26 | 27 | thread_local_isolate = isolate; 28 | 29 | v8::HandleScope scope(isolate); 30 | 31 | AddonData * addonData = new AddonData(); 32 | 33 | node::AddEnvironmentCleanupHook(isolate, AddonData_DeleteInstance, addonData); 34 | 35 | addonData->initialize(isolate); 36 | 37 | AlignedBuffers_Init(exports, addonData); 38 | RoaringBitmap32_Init(exports, addonData); 39 | RoaringBitmap32BufferedIterator_Init(exports, addonData); 40 | 41 | AddonData_setMethod(exports, "getRoaringUsedMemory", getRoaringUsedMemory, addonData); 42 | 43 | v8utils::defineHiddenField(isolate, exports, "default", exports); 44 | } 45 | 46 | MODULE_WORKER_ENABLED(roaring, InitRoaringNode); 47 | 48 | #include "croaring.cpp" 49 | -------------------------------------------------------------------------------- /src/cpp/memory.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_MEMORY_ 2 | #define ROARING_NODE_MEMORY_ 3 | 4 | #include "includes.h" 5 | 6 | /** portable version of posix_memalign */ 7 | void * bare_aligned_malloc(size_t alignment, size_t size) { 8 | void * p; 9 | #ifdef _MSC_VER 10 | p = _aligned_malloc(size, alignment); 11 | #elif defined(__MINGW32__) || defined(__MINGW64__) 12 | p = __mingw_aligned_malloc(size, alignment); 13 | #else 14 | // somehow, if this is used before including "x86intrin.h", it creates an 15 | // implicit defined warning. 16 | if (posix_memalign(&p, alignment, size) != 0) return NULL; 17 | #endif 18 | return p; 19 | } 20 | 21 | /** portable version of free fo aligned allocs */ 22 | void bare_aligned_free(void * memblock) { 23 | if (memblock != nullptr) { 24 | #ifdef _MSC_VER 25 | _aligned_free(memblock); 26 | #elif defined(__MINGW32__) || defined(__MINGW64__) 27 | __mingw_aligned_free(memblock); 28 | #else 29 | free(memblock); 30 | #endif 31 | } 32 | } 33 | 34 | /** portable version of malloc_size */ 35 | inline size_t bare_malloc_size(const void * ptr) { 36 | #if defined(__APPLE__) 37 | return malloc_size((void *)ptr); 38 | #elif defined(_WIN32) || defined(__MINGW32__) || defined(__MINGW64__) 39 | return _msize((void *)ptr); 40 | #else 41 | return malloc_usable_size((void *)ptr); 42 | #endif 43 | } 44 | 45 | /** portable version of malloc_size for memory allocated with bare_aligned_malloc */ 46 | inline size_t bare_aligned_malloc_size(const void * ptr) { 47 | #if defined(__APPLE__) 48 | return malloc_size((void *)ptr); 49 | #elif defined(_WIN32) 50 | return _aligned_msize((void *)ptr, 32, 0); 51 | #else 52 | return malloc_usable_size((void *)ptr); 53 | #endif 54 | } 55 | 56 | std::atomic gcaware_totalMemCounter{0}; 57 | 58 | int64_t gcaware_totalMem() { return gcaware_totalMemCounter; } 59 | 60 | thread_local v8::Isolate * thread_local_isolate = nullptr; 61 | 62 | inline void _gcaware_adjustAllocatedMemory(int64_t size) { 63 | if (size != 0) { 64 | v8::Isolate * isolate = v8::Isolate::GetCurrent(); 65 | if (isolate == nullptr) { 66 | isolate = thread_local_isolate; 67 | } 68 | if (isolate != nullptr) { 69 | isolate->AdjustAmountOfExternalAllocatedMemory(size); 70 | } 71 | gcaware_totalMemCounter += size; 72 | } 73 | } 74 | 75 | inline void gcaware_addAllocatedMemory(size_t size) { _gcaware_adjustAllocatedMemory((int64_t)size); } 76 | 77 | inline void gcaware_removeAllocatedMemory(size_t size) { _gcaware_adjustAllocatedMemory(-(int64_t)size); } 78 | 79 | void * gcaware_malloc(size_t size) { 80 | void * memory = malloc(size); 81 | if (memory != nullptr) { 82 | gcaware_addAllocatedMemory(bare_malloc_size(memory)); 83 | } 84 | return memory; 85 | } 86 | 87 | void * gcaware_realloc(void * memory, size_t size) { 88 | size_t oldSize = memory != nullptr ? bare_malloc_size(memory) : 0; 89 | memory = realloc(memory, size); 90 | if (memory != nullptr) { 91 | gcaware_removeAllocatedMemory(oldSize); 92 | gcaware_addAllocatedMemory(bare_malloc_size(memory)); 93 | } 94 | return memory; 95 | } 96 | 97 | void * gcaware_calloc(size_t count, size_t size) { 98 | void * memory = calloc(count, size); 99 | if (memory != nullptr) { 100 | gcaware_addAllocatedMemory(bare_malloc_size(memory)); 101 | } 102 | return memory; 103 | } 104 | 105 | void gcaware_free(void * memory) { 106 | if (memory != nullptr) { 107 | gcaware_removeAllocatedMemory(bare_malloc_size(memory)); 108 | free(memory); 109 | } 110 | } 111 | 112 | void * gcaware_aligned_malloc(size_t alignment, size_t size) { 113 | void * memory = bare_aligned_malloc(alignment, size); 114 | if (memory != nullptr) { 115 | gcaware_addAllocatedMemory(bare_aligned_malloc_size(memory)); 116 | } 117 | return memory; 118 | } 119 | 120 | void gcaware_aligned_free(void * memory) { 121 | if (memory != nullptr) { 122 | gcaware_removeAllocatedMemory(bare_aligned_malloc_size(memory)); 123 | bare_aligned_free(memory); 124 | } 125 | } 126 | 127 | void bare_aligned_free_callback(char * data, void * hint) { bare_aligned_free(data); } 128 | 129 | void bare_aligned_free_callback2(void * data, size_t length, void * deleter_data) { bare_aligned_free(data); } 130 | 131 | inline bool is_pointer_aligned(const void * ptr, std::uintptr_t alignment) noexcept { 132 | auto iptr = reinterpret_cast(ptr); 133 | return !(iptr % alignment); 134 | } 135 | 136 | void getRoaringUsedMemory(const v8::FunctionCallbackInfo & info) { 137 | info.GetReturnValue().Set((double)gcaware_totalMem()); 138 | } 139 | 140 | #endif // ROARING_NODE_MEMORY_ 141 | -------------------------------------------------------------------------------- /src/cpp/mmap.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_MMAP_ 2 | #define ROARING_NODE_MMAP_ 3 | 4 | #if defined(_WIN32) || defined(__MINGW32__) || defined(__MINGW64__) 5 | 6 | /* mmap() replacement for Windows 7 | * 8 | * Author: Mike Frysinger 9 | * Placed into the public domain 10 | */ 11 | 12 | /* References: 13 | * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx 14 | * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx 15 | * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx 16 | * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx 17 | */ 18 | 19 | # include 20 | # include 21 | # include 22 | 23 | # define PROT_READ 0x1 24 | # define PROT_WRITE 0x2 25 | /* This flag is only available in WinXP+ */ 26 | # ifdef FILE_MAP_EXECUTE 27 | # define PROT_EXEC 0x4 28 | # else 29 | # define PROT_EXEC 0x0 30 | # define FILE_MAP_EXECUTE 0 31 | # endif 32 | 33 | # define MAP_SHARED 0x01 34 | # define MAP_PRIVATE 0x02 35 | # define MAP_ANONYMOUS 0x20 36 | # define MAP_ANON MAP_ANONYMOUS 37 | # define MAP_FAILED ((void *)-1) 38 | 39 | # ifdef __USE_FILE_OFFSET64 40 | # define DWORD_HI(x) (x >> 32) 41 | # define DWORD_LO(x) ((x)&0xffffffff) 42 | # else 43 | # define DWORD_HI(x) (0) 44 | # define DWORD_LO(x) (x) 45 | # endif 46 | 47 | static void * mmap(void * start, size_t length, int prot, int flags, int fd, off_t offset) { 48 | if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) return MAP_FAILED; 49 | if (fd == -1) { 50 | if (!(flags & MAP_ANON) || offset) return MAP_FAILED; 51 | } else if (flags & MAP_ANON) 52 | return MAP_FAILED; 53 | 54 | DWORD flProtect; 55 | if (prot & PROT_WRITE) { 56 | if (prot & PROT_EXEC) 57 | flProtect = PAGE_EXECUTE_READWRITE; 58 | else 59 | flProtect = PAGE_READWRITE; 60 | } else if (prot & PROT_EXEC) { 61 | if (prot & PROT_READ) 62 | flProtect = PAGE_EXECUTE_READ; 63 | else if (prot & PROT_EXEC) 64 | flProtect = PAGE_EXECUTE; 65 | } else 66 | flProtect = PAGE_READONLY; 67 | 68 | off_t end = length + offset; 69 | HANDLE mmap_fd, h; 70 | if (fd == -1) 71 | mmap_fd = INVALID_HANDLE_VALUE; 72 | else 73 | mmap_fd = (HANDLE)_get_osfhandle(fd); 74 | h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL); 75 | if (h == NULL) return MAP_FAILED; 76 | 77 | DWORD dwDesiredAccess; 78 | if (prot & PROT_WRITE) 79 | dwDesiredAccess = FILE_MAP_WRITE; 80 | else 81 | dwDesiredAccess = FILE_MAP_READ; 82 | if (prot & PROT_EXEC) dwDesiredAccess |= FILE_MAP_EXECUTE; 83 | if (flags & MAP_PRIVATE) dwDesiredAccess |= FILE_MAP_COPY; 84 | void * ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length); 85 | if (ret == NULL) { 86 | CloseHandle(h); 87 | ret = MAP_FAILED; 88 | } 89 | return ret; 90 | } 91 | 92 | static void munmap(void * addr, size_t length) { 93 | UnmapViewOfFile(addr); 94 | /* ruh-ro, we leaked handle from CreateFileMapping() ... */ 95 | } 96 | 97 | # undef DWORD_HI 98 | # undef DWORD_LO 99 | 100 | #else 101 | 102 | # include 103 | # include 104 | 105 | #endif 106 | #endif 107 | -------------------------------------------------------------------------------- /src/cpp/object-wrap.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_OBJECT_WRAP_ 2 | #define ROARING_NODE_OBJECT_WRAP_ 3 | 4 | #include "addon-data.h" 5 | 6 | class ObjectWrap { 7 | public: 8 | AddonData * const addonData; 9 | 10 | template 11 | static T * TryUnwrap(const v8::Local & value, v8::Isolate * isolate) { 12 | v8::Local obj; 13 | if (!value->IsObject()) { 14 | return nullptr; 15 | } 16 | 17 | if (!value->ToObject(isolate->GetCurrentContext()).ToLocal(&obj)) { 18 | return nullptr; 19 | } 20 | 21 | if (obj->InternalFieldCount() != 2) { 22 | return nullptr; 23 | } 24 | 25 | if ((uintptr_t)obj->GetAlignedPointerFromInternalField(1) != T::OBJECT_TOKEN) { 26 | return nullptr; 27 | } 28 | 29 | return (T *)(obj->GetAlignedPointerFromInternalField(0)); 30 | } 31 | 32 | template 33 | static T * TryUnwrap(const v8::FunctionCallbackInfo & info, int argumentIndex) { 34 | return info.Length() <= argumentIndex ? nullptr : ObjectWrap::TryUnwrap(info[argumentIndex], info.GetIsolate()); 35 | } 36 | 37 | protected: 38 | explicit ObjectWrap(AddonData * addonData) : addonData(addonData) {} 39 | }; 40 | 41 | #endif // ROARING_NODE_OBJECT_WRAP_ 42 | -------------------------------------------------------------------------------- /src/cpp/serialization-csv.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_SERIALIZATION_CSV_ 2 | #define ROARING_NODE_SERIALIZATION_CSV_ 3 | 4 | #include "includes.h" 5 | #include 6 | #include "mmap.h" 7 | #include "serialization-format.h" 8 | #include "WorkerError.h" 9 | 10 | struct CsvFileDescriptorSerializer final { 11 | public: 12 | static int iterate(const roaring::api::roaring_bitmap_t * r, int fd, FileSerializationFormat format) { 13 | char separator; 14 | switch (format) { 15 | case FileSerializationFormat::newline_separated_values: separator = '\n'; break; 16 | case FileSerializationFormat::comma_separated_values: separator = ','; break; 17 | case FileSerializationFormat::tab_separated_values: separator = '\t'; break; 18 | case FileSerializationFormat::json_array: separator = ','; break; 19 | default: return EINVAL; 20 | } 21 | 22 | CsvFileDescriptorSerializer writer(fd, separator); 23 | if (format == FileSerializationFormat::json_array) { 24 | writer.appendChar('['); 25 | } 26 | 27 | if (r) { 28 | roaring_iterate(r, roaringIteratorFn, &writer); 29 | } 30 | 31 | if (format == FileSerializationFormat::newline_separated_values) { 32 | writer.appendChar('\n'); 33 | } else if (format == FileSerializationFormat::json_array) { 34 | writer.appendChar(']'); 35 | } 36 | 37 | if (!writer.flush()) { 38 | int errorno = errno; 39 | errno = 0; 40 | return errorno ? errorno : EIO; 41 | } 42 | 43 | return 0; 44 | } 45 | 46 | private: 47 | const constexpr static size_t BUFFER_SIZE = 131072; 48 | 49 | char * buf; 50 | size_t bufPos; 51 | int fd; 52 | bool needsSeparator; 53 | char separator; 54 | 55 | CsvFileDescriptorSerializer(int fd, char separator) : 56 | buf((char *)gcaware_aligned_malloc(32, BUFFER_SIZE)), bufPos(0), fd(fd), needsSeparator(false), separator(separator) {} 57 | 58 | ~CsvFileDescriptorSerializer() { gcaware_aligned_free(this->buf); } 59 | 60 | bool flush() { 61 | if (this->bufPos == 0) { 62 | return true; 63 | } 64 | if (!this->buf) { 65 | return false; 66 | } 67 | ssize_t written = write(this->fd, this->buf, this->bufPos); 68 | if (written < 0) { 69 | gcaware_aligned_free(this->buf); 70 | this->buf = nullptr; 71 | return false; 72 | } 73 | this->bufPos = 0; 74 | return true; 75 | } 76 | 77 | bool appendChar(char c) { 78 | if (this->bufPos + 1 >= BUFFER_SIZE) { 79 | if (!this->flush()) { 80 | return false; 81 | } 82 | } 83 | if (!this->buf) { 84 | return false; 85 | } 86 | this->buf[this->bufPos++] = c; 87 | return true; 88 | } 89 | 90 | bool appendValue(uint32_t value) { 91 | if (this->bufPos + 15 >= BUFFER_SIZE) { 92 | if (!this->flush()) { 93 | return false; 94 | } 95 | } 96 | if (!this->buf) { 97 | return false; 98 | } 99 | if (this->needsSeparator) { 100 | this->buf[this->bufPos++] = this->separator; 101 | } 102 | this->needsSeparator = true; 103 | 104 | char * str = this->buf + this->bufPos; 105 | int32_t i, j; 106 | char c; 107 | 108 | /* uint to decimal */ 109 | i = 0; 110 | do { 111 | uint32_t remainder = value % 10; 112 | str[i++] = (char)(remainder + 48); 113 | value = value / 10; 114 | } while (value != 0); 115 | 116 | this->bufPos += i; 117 | 118 | /* reverse string */ 119 | for (j = 0, i--; j < i; j++, i--) { 120 | c = str[i]; 121 | str[i] = str[j]; 122 | str[j] = c; 123 | } 124 | 125 | return true; 126 | } 127 | 128 | static bool roaringIteratorFn(uint32_t value, void * param) { 129 | return ((CsvFileDescriptorSerializer *)param)->appendValue(value); 130 | } 131 | }; 132 | 133 | WorkerError deserializeRoaringCsvFile( 134 | roaring::api::roaring_bitmap_t * r, int fd, const char * input, size_t input_size, const std::string & filePath) { 135 | const constexpr static size_t BUFFER_SIZE = 131072; 136 | 137 | char * buf; 138 | ssize_t readBytes; 139 | if (input == nullptr) { 140 | buf = (char *)gcaware_aligned_malloc(32, BUFFER_SIZE); 141 | if (!buf) { 142 | return WorkerError("Failed to allocate memory for text deserialization"); 143 | } 144 | } else { 145 | buf = (char *)input; 146 | readBytes = (ssize_t)input_size; 147 | if (readBytes < 0) { 148 | return WorkerError("Input too big"); 149 | } 150 | if (readBytes == 0) { 151 | return WorkerError(); 152 | } 153 | } 154 | 155 | roaring_bulk_context_t context; 156 | memset(&context, 0, sizeof(context)); 157 | uint64_t value = 0; 158 | 159 | bool hasValue = false; 160 | bool isNegative = false; 161 | for (;;) { 162 | if (input == nullptr) { 163 | readBytes = read(fd, buf, BUFFER_SIZE); 164 | if (readBytes <= 0) { 165 | if (readBytes < 0) { 166 | WorkerError err = WorkerError::from_errno("read", filePath); 167 | gcaware_aligned_free(buf); 168 | return err; 169 | } 170 | break; 171 | } 172 | } 173 | 174 | for (ssize_t i = 0; i < readBytes; i++) { 175 | char c = buf[i]; 176 | if (c >= '0' && c <= '9') { 177 | if (value <= 0xffffffff) { 178 | hasValue = true; 179 | value = value * 10 + (c - '0'); 180 | } 181 | } else { 182 | if (hasValue) { 183 | hasValue = false; 184 | if (!isNegative && value <= 0xffffffff) { 185 | roaring_bitmap_add_bulk(r, &context, value); 186 | } 187 | } 188 | value = 0; 189 | isNegative = c == '-'; 190 | } 191 | } 192 | 193 | if (input != nullptr) { 194 | break; 195 | } 196 | } 197 | 198 | if (!isNegative && hasValue && value <= 0xffffffff) { 199 | roaring_bitmap_add_bulk(r, &context, value); 200 | } 201 | if (input == nullptr) { 202 | gcaware_aligned_free(buf); 203 | } 204 | 205 | return WorkerError(); 206 | } 207 | 208 | #endif 209 | -------------------------------------------------------------------------------- /src/cpp/serialization-format.h: -------------------------------------------------------------------------------- 1 | #ifndef ROARING_NODE_SERIALIZATION_FORMAT_ 2 | #define ROARING_NODE_SERIALIZATION_FORMAT_ 3 | 4 | #include "includes.h" 5 | 6 | const uint32_t MAX_SERIALIZATION_ARRAY_SIZE_IN_BYTES = 0x00FFFFFF; 7 | 8 | enum class SerializationFormat { 9 | INVALID = -1, 10 | croaring = 0, 11 | portable = 1, 12 | unsafe_frozen_croaring = 2, 13 | uint32_array = 4, 14 | }; 15 | 16 | enum class FileSerializationFormat { 17 | INVALID = -1, 18 | croaring = 0, 19 | portable = 1, 20 | unsafe_frozen_croaring = 2, 21 | uint32_array = 4, 22 | 23 | comma_separated_values = 10, 24 | tab_separated_values = 11, 25 | newline_separated_values = 12, 26 | json_array = 20 27 | }; 28 | 29 | enum class DeserializationFormat { 30 | INVALID = -1, 31 | croaring = 0, 32 | portable = 1, 33 | unsafe_frozen_croaring = 2, 34 | unsafe_frozen_portable = 3, 35 | uint32_array = 4, 36 | 37 | comma_separated_values = 10, 38 | tab_separated_values = 11, 39 | newline_separated_values = 12, 40 | json_array = 20 41 | }; 42 | 43 | enum class FileDeserializationFormat { 44 | INVALID = -1, 45 | croaring = 0, 46 | portable = 1, 47 | unsafe_frozen_croaring = 2, 48 | unsafe_frozen_portable = 3, 49 | uint32_array = 4, 50 | 51 | comma_separated_values = 10, 52 | tab_separated_values = 11, 53 | newline_separated_values = 12, 54 | json_array = 20 55 | }; 56 | 57 | enum class FrozenViewFormat { 58 | INVALID = -1, 59 | unsafe_frozen_croaring = 2, 60 | unsafe_frozen_portable = 3, 61 | }; 62 | 63 | SerializationFormat tryParseSerializationFormat(const v8::Local & value, v8::Isolate * isolate) { 64 | if (!isolate || value.IsEmpty()) { 65 | return SerializationFormat::INVALID; 66 | } 67 | if (value->IsBoolean()) { 68 | if (value->IsTrue()) { 69 | return SerializationFormat::portable; 70 | } 71 | if (value->IsFalse()) { 72 | return SerializationFormat::croaring; 73 | } 74 | } else if (value->IsString()) { 75 | v8::String::Utf8Value formatString(isolate, value); 76 | if (strcmp(*formatString, "croaring") == 0) { 77 | return SerializationFormat::croaring; 78 | } 79 | if (strcmp(*formatString, "portable") == 0) { 80 | return SerializationFormat::portable; 81 | } 82 | if (strcmp(*formatString, "unsafe_frozen_croaring") == 0) { 83 | return SerializationFormat::unsafe_frozen_croaring; 84 | } 85 | if (strcmp(*formatString, "uint32_array") == 0) { 86 | return SerializationFormat::uint32_array; 87 | } 88 | } 89 | return SerializationFormat::INVALID; 90 | } 91 | 92 | FileSerializationFormat tryParseFileSerializationFormat(const v8::Local & value, v8::Isolate * isolate) { 93 | SerializationFormat sf = tryParseSerializationFormat(value, isolate); 94 | if (sf != SerializationFormat::INVALID) { 95 | return static_cast(sf); 96 | } 97 | if (!isolate || value.IsEmpty()) { 98 | return FileSerializationFormat::INVALID; 99 | } 100 | if (value->IsString()) { 101 | v8::String::Utf8Value formatString(isolate, value); 102 | if (strcmp(*formatString, "comma_separated_values") == 0) { 103 | return FileSerializationFormat::comma_separated_values; 104 | } 105 | if (strcmp(*formatString, "tab_separated_values") == 0) { 106 | return FileSerializationFormat::tab_separated_values; 107 | } 108 | if (strcmp(*formatString, "newline_separated_values") == 0) { 109 | return FileSerializationFormat::newline_separated_values; 110 | } 111 | if (strcmp(*formatString, "json_array") == 0) { 112 | return FileSerializationFormat::json_array; 113 | } 114 | } 115 | return FileSerializationFormat::INVALID; 116 | } 117 | 118 | DeserializationFormat tryParseDeserializationFormat(const v8::Local & value, v8::Isolate * isolate) { 119 | if (!isolate || value.IsEmpty()) { 120 | return DeserializationFormat::INVALID; 121 | } 122 | if (value->IsBoolean()) { 123 | if (value->IsTrue()) { 124 | return DeserializationFormat::portable; 125 | } 126 | if (value->IsFalse()) { 127 | return DeserializationFormat::croaring; 128 | } 129 | } else if (value->IsString()) { 130 | v8::String::Utf8Value formatString(isolate, value); 131 | if (strcmp(*formatString, "croaring") == 0) { 132 | return DeserializationFormat::croaring; 133 | } 134 | if (strcmp(*formatString, "portable") == 0) { 135 | return DeserializationFormat::portable; 136 | } 137 | if (strcmp(*formatString, "unsafe_frozen_croaring") == 0) { 138 | return DeserializationFormat::unsafe_frozen_croaring; 139 | } 140 | if (strcmp(*formatString, "unsafe_frozen_portable") == 0) { 141 | return DeserializationFormat::unsafe_frozen_portable; 142 | } 143 | if (strcmp(*formatString, "uint32_array") == 0) { 144 | return DeserializationFormat::uint32_array; 145 | } 146 | if (strcmp(*formatString, "comma_separated_values") == 0) { 147 | return DeserializationFormat::comma_separated_values; 148 | } 149 | if (strcmp(*formatString, "tab_separated_values") == 0) { 150 | return DeserializationFormat::tab_separated_values; 151 | } 152 | if (strcmp(*formatString, "newline_separated_values") == 0) { 153 | return DeserializationFormat::newline_separated_values; 154 | } 155 | if (strcmp(*formatString, "json_array") == 0) { 156 | return DeserializationFormat::json_array; 157 | } 158 | } 159 | return DeserializationFormat::INVALID; 160 | } 161 | 162 | FileDeserializationFormat tryParseFileDeserializationFormat(const v8::Local & value, v8::Isolate * isolate) { 163 | return (FileDeserializationFormat)tryParseDeserializationFormat(value, isolate); 164 | } 165 | 166 | FrozenViewFormat tryParseFrozenViewFormat(const v8::Local & value, v8::Isolate * isolate) { 167 | if (!isolate || value.IsEmpty()) { 168 | return FrozenViewFormat::INVALID; 169 | } 170 | if (value->IsString()) { 171 | v8::String::Utf8Value formatString(isolate, value); 172 | if (strcmp(*formatString, "unsafe_frozen_croaring") == 0) { 173 | return FrozenViewFormat::unsafe_frozen_croaring; 174 | } 175 | if (strcmp(*formatString, "unsafe_frozen_portable") == 0) { 176 | return FrozenViewFormat::unsafe_frozen_portable; 177 | } 178 | } 179 | return FrozenViewFormat::INVALID; 180 | } 181 | 182 | #endif // ROARING_NODE_SERIALIZATION_FORMAT_ 183 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.deserializeParallelAsync.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import { expect, use as chaiUse } from "chai"; 3 | 4 | chaiUse(require("chai-as-promised")); 5 | 6 | describe("RoaringBitmap32 deserializeParallelAsync", () => { 7 | describe("async/await", () => { 8 | describe("one empty buffer", () => { 9 | it("deserializes an empty buffer (non portable)", async () => { 10 | const bitmap = await RoaringBitmap32.deserializeParallelAsync([Buffer.from([])], false); 11 | expect(bitmap).to.have.lengthOf(1); 12 | expect(bitmap[0]).to.be.instanceOf(RoaringBitmap32); 13 | expect(bitmap[0].size).eq(0); 14 | }); 15 | 16 | it("deserializes an empty buffer (portable)", async () => { 17 | const bitmap = await RoaringBitmap32.deserializeParallelAsync([Buffer.from([])], true); 18 | expect(bitmap).to.have.lengthOf(1); 19 | expect(bitmap[0]).to.be.instanceOf(RoaringBitmap32); 20 | expect(bitmap[0].size).eq(0); 21 | }); 22 | }); 23 | 24 | describe("multiple empty buffers", () => { 25 | it("deserializes an empty buffer (non portable)", async () => { 26 | const bitmap = await RoaringBitmap32.deserializeParallelAsync( 27 | [Buffer.from([]), Buffer.from([]), Buffer.from([])], 28 | false, 29 | ); 30 | expect(bitmap).to.have.lengthOf(3); 31 | for (let i = 0; i < 3; ++i) { 32 | expect(bitmap[i]).to.be.instanceOf(RoaringBitmap32); 33 | expect(bitmap[i].size).eq(0); 34 | } 35 | }); 36 | 37 | it("deserializes an empty buffer (portable)", async () => { 38 | const bitmap = await RoaringBitmap32.deserializeParallelAsync( 39 | [Buffer.from([]), Buffer.from([]), Buffer.from([])], 40 | true, 41 | ); 42 | expect(bitmap).to.have.lengthOf(3); 43 | for (let i = 0; i < 3; ++i) { 44 | expect(bitmap[i]).to.be.instanceOf(RoaringBitmap32); 45 | expect(bitmap[i].size).eq(0); 46 | } 47 | }); 48 | }); 49 | 50 | it("deserializes multiple buffers (non portable)", async () => { 51 | const sources = []; 52 | for (let i = 0; i < 10; ++i) { 53 | const array = [i, i + 10, i * 10000, i * 999999]; 54 | for (let j = 0; j < i; ++j) { 55 | array.push(j); 56 | } 57 | sources.push(new RoaringBitmap32(array)); 58 | } 59 | 60 | const result = await RoaringBitmap32.deserializeParallelAsync( 61 | sources.map((x) => x.serialize(false)), 62 | false, 63 | ); 64 | expect(result).to.have.lengthOf(sources.length); 65 | for (let i = 0; i < sources.length; ++i) { 66 | expect(result[i].toArray()).deep.equal(sources[i].toArray()); 67 | } 68 | }); 69 | 70 | it("deserializes multiple buffers (portable)", async () => { 71 | const sources = []; 72 | for (let i = 0; i < 10; ++i) { 73 | const array = [i, i + 10, i * 10000, i * 999999]; 74 | for (let j = 0; j < i; ++j) { 75 | array.push(j); 76 | } 77 | sources.push(new RoaringBitmap32(array)); 78 | } 79 | 80 | const result = await RoaringBitmap32.deserializeParallelAsync( 81 | sources.map((x) => x.serialize(true)), 82 | true, 83 | ); 84 | expect(result).to.have.lengthOf(sources.length); 85 | for (let i = 0; i < sources.length; ++i) { 86 | expect(result[i].toArray()).deep.equal(sources[i].toArray()); 87 | } 88 | }); 89 | }); 90 | }); 91 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.fromAsync.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import { expect, use as chaiUse } from "chai"; 3 | 4 | chaiUse(require("chai-as-promised")); 5 | 6 | describe("RoaringBitmap32 fromAsync", () => { 7 | describe("awayt/async", () => { 8 | describe("empty", () => { 9 | it("creates an empty bitmap with undefined", async () => { 10 | const bitmap = await RoaringBitmap32.fromArrayAsync(undefined as any); 11 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 12 | expect(bitmap.isEmpty).eq(true); 13 | }); 14 | 15 | it("creates an empty bitmap with null", async () => { 16 | const bitmap = await RoaringBitmap32.fromArrayAsync(null as any); 17 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 18 | expect(bitmap.isEmpty).eq(true); 19 | }); 20 | 21 | it("creates an empty bitmap with an empty Uint32Array", async () => { 22 | const bitmap = await RoaringBitmap32.fromArrayAsync(new Uint32Array(0)); 23 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 24 | expect(bitmap.isEmpty).eq(true); 25 | }); 26 | 27 | it("creates an empty bitmap with an empty Int32Array", async () => { 28 | const bitmap = await RoaringBitmap32.fromArrayAsync(new Int32Array(0)); 29 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 30 | expect(bitmap.isEmpty).eq(true); 31 | }); 32 | 33 | it("creates an empty bitmap with an empty array", async () => { 34 | const bitmap = await RoaringBitmap32.fromArrayAsync([]); 35 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 36 | expect(bitmap.isEmpty).eq(true); 37 | }); 38 | }); 39 | 40 | describe("with data", () => { 41 | it("creates a bitmap with an Uint32Array", async () => { 42 | const values = [1, 2, 3, 4, 5]; 43 | const bitmap = await RoaringBitmap32.fromArrayAsync(new Uint32Array(values)); 44 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 45 | expect(bitmap.toArray()).deep.equal(values); 46 | }); 47 | 48 | it("creates a bitmap with an Int32Array", async () => { 49 | const values = [1, 2, 3, 4, 5]; 50 | const bitmap = await RoaringBitmap32.fromArrayAsync(new Int32Array(values)); 51 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 52 | expect(bitmap.toArray()).deep.equal(values); 53 | }); 54 | 55 | it("creates a bitmap with an array", async () => { 56 | const values = [1, 2, 3, 0x7fffffff, 0xffffffff]; 57 | const bitmap = await RoaringBitmap32.fromArrayAsync(values); 58 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 59 | expect(bitmap.toArray()).deep.equal(values); 60 | }); 61 | }); 62 | }); 63 | }); 64 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.frozen.test.ts: -------------------------------------------------------------------------------- 1 | import { FrozenViewFormat } from "../.."; 2 | import RoaringBitmap32 from "../../RoaringBitmap32"; 3 | import { expect } from "chai"; 4 | 5 | const ERROR_FROZEN = "This bitmap is frozen and cannot be modified"; 6 | 7 | describe("RoaringBitmap32 frozen", () => { 8 | describe("FrozenViewFormat", () => { 9 | it("should have the right values", () => { 10 | expect(FrozenViewFormat.unsafe_frozen_croaring).eq("unsafe_frozen_croaring"); 11 | expect(FrozenViewFormat.unsafe_frozen_portable).eq("unsafe_frozen_portable"); 12 | 13 | expect(Object.values(FrozenViewFormat)).to.deep.eq(["unsafe_frozen_croaring", "unsafe_frozen_portable"]); 14 | 15 | expect(RoaringBitmap32.FrozenViewFormat).to.eq(FrozenViewFormat); 16 | 17 | expect(new RoaringBitmap32().FrozenViewFormat).to.eq(FrozenViewFormat); 18 | }); 19 | }); 20 | 21 | describe("freeze", () => { 22 | it("set isFrozen to true, return this, can be called multiple times", () => { 23 | const bitmap = new RoaringBitmap32(); 24 | expect(bitmap.isFrozen).to.be.false; 25 | expect(bitmap.freeze()).to.equal(bitmap); 26 | expect(bitmap.isFrozen).to.be.true; 27 | 28 | expect(bitmap.freeze()).to.equal(bitmap); 29 | expect(bitmap.isFrozen).to.be.true; 30 | }); 31 | 32 | it("throws when calling writable methods", () => { 33 | const bitmap = new RoaringBitmap32(); 34 | bitmap.freeze(); 35 | 36 | expect(() => bitmap.copyFrom([1])).to.throw(ERROR_FROZEN); 37 | expect(() => bitmap.add(1)).to.throw(ERROR_FROZEN); 38 | expect(() => bitmap.tryAdd(1)).to.throw(ERROR_FROZEN); 39 | expect(() => bitmap.addMany([1])).to.throw(ERROR_FROZEN); 40 | expect(() => bitmap.remove(1)).to.throw(ERROR_FROZEN); 41 | expect(() => bitmap.removeMany([1])).to.throw(ERROR_FROZEN); 42 | expect(() => bitmap.delete(1)).to.throw(ERROR_FROZEN); 43 | expect(() => bitmap.clear()).to.throw(ERROR_FROZEN); 44 | expect(() => bitmap.orInPlace([1])).to.throw(ERROR_FROZEN); 45 | expect(() => bitmap.andNotInPlace([1])).to.throw(ERROR_FROZEN); 46 | expect(() => bitmap.andInPlace([1])).to.throw(ERROR_FROZEN); 47 | expect(() => bitmap.xorInPlace([1])).to.throw(ERROR_FROZEN); 48 | expect(() => bitmap.flipRange(0, 10)).to.throw(ERROR_FROZEN); 49 | expect(() => bitmap.addRange(0, 9)).to.throw(ERROR_FROZEN); 50 | expect(() => bitmap.removeRange(0, 8)).to.throw(ERROR_FROZEN); 51 | 52 | const bitmap1 = new RoaringBitmap32(); 53 | const bitmap2 = new RoaringBitmap32(); 54 | bitmap1.freeze(); 55 | expect(() => RoaringBitmap32.swap(bitmap1, bitmap2)).to.throw(ERROR_FROZEN); 56 | expect(() => RoaringBitmap32.swap(bitmap2, bitmap1)).to.throw(ERROR_FROZEN); 57 | }); 58 | }); 59 | 60 | describe("serializeAsync", () => { 61 | it("Is temporarily frozen while using serializeAsync", async () => { 62 | const bitmap = new RoaringBitmap32([1, 2, 3]); 63 | expect(bitmap.isFrozen).to.be.false; 64 | const promise = bitmap.serializeAsync("croaring"); 65 | expect(bitmap.isFrozen).to.be.true; 66 | await promise; 67 | expect(bitmap.isFrozen).to.be.false; 68 | bitmap.add(4); 69 | }); 70 | 71 | it('keep it frozen after calling "serializeAsync"', async () => { 72 | const bitmap = new RoaringBitmap32([1, 2, 3]); 73 | bitmap.freeze(); 74 | expect(bitmap.isFrozen).to.be.true; 75 | const promise = bitmap.serializeAsync("croaring"); 76 | expect(bitmap.isFrozen).to.be.true; 77 | await promise; 78 | expect(bitmap.isFrozen).to.be.true; 79 | expect(() => bitmap.add(4)).to.throw(ERROR_FROZEN); 80 | }); 81 | }); 82 | 83 | describe("unsafeFrozenView", () => { 84 | it("throws if the buffer is an invalid type", () => { 85 | expect(() => RoaringBitmap32.unsafeFrozenView({} as any, "unsafe_frozen_croaring")).to.throw(); 86 | expect(() => RoaringBitmap32.unsafeFrozenView([] as any, "unsafe_frozen_croaring")).to.throw(); 87 | expect(() => RoaringBitmap32.unsafeFrozenView(null as any, "unsafe_frozen_croaring")).to.throw(); 88 | expect(() => RoaringBitmap32.unsafeFrozenView(undefined as any, "unsafe_frozen_croaring")).to.throw(); 89 | expect(() => RoaringBitmap32.unsafeFrozenView(new Uint32Array(1), "unsafe_frozen_croaring")).to.throw(); 90 | }); 91 | 92 | it("throws if format is invalid", () => { 93 | expect(() => RoaringBitmap32.unsafeFrozenView(new Uint8Array(10), "a" as any)).to.throw(); 94 | expect(() => RoaringBitmap32.unsafeFrozenView(new Uint8Array(10), "" as any)).to.throw(); 95 | expect(() => RoaringBitmap32.unsafeFrozenView(new Uint8Array(10), undefined as any)).to.throw(); 96 | expect(() => RoaringBitmap32.unsafeFrozenView(new Uint8Array(10), null as any)).to.throw(); 97 | }); 98 | 99 | it("can create a view from a serialized bitmap, and the view is frozen", () => { 100 | const values = [1, 2, 3, 100, 8772837, 0x7ffffff1, 0x7fffffff]; 101 | const bitmap = new RoaringBitmap32(values); 102 | const serialized = bitmap.serialize("unsafe_frozen_croaring"); 103 | expect(RoaringBitmap32.isBufferAligned(serialized)).to.be.true; 104 | const view = RoaringBitmap32.unsafeFrozenView(serialized, "unsafe_frozen_croaring"); 105 | expect(view.isFrozen).to.be.true; 106 | expect(view.toArray()).to.deep.equal(values); 107 | expect(() => view.add(4)).to.throw(ERROR_FROZEN); 108 | expect(() => view.runOptimize()).to.throw(ERROR_FROZEN); 109 | expect(() => view.shrinkToFit()).to.throw(ERROR_FROZEN); 110 | expect(() => view.removeRunCompression()).to.throw(ERROR_FROZEN); 111 | 112 | const copy = view.clone(); 113 | expect(copy.isFrozen).to.be.false; 114 | expect(copy.toArray()).to.deep.equal(values); 115 | expect(copy.tryAdd(4)).to.be.true; 116 | }); 117 | 118 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 119 | if (nodeVersion >= 12) { 120 | it("can create a view from a serialized bitmap in a SharedArrayBuffer, and the view is frozen", () => { 121 | const values = [1, 2, 3, 100, 8772837, 0x7ffffff1, 0x7fffffff]; 122 | const bitmap = new RoaringBitmap32(values); 123 | const sharedBuffer = RoaringBitmap32.bufferAlignedAllocShared( 124 | bitmap.getSerializationSizeInBytes("unsafe_frozen_croaring"), 125 | ); 126 | expect(sharedBuffer.buffer).to.be.instanceOf(SharedArrayBuffer); 127 | const serialized = bitmap.serialize("unsafe_frozen_croaring", sharedBuffer); 128 | expect(serialized).to.eq(sharedBuffer); 129 | 130 | const view = RoaringBitmap32.unsafeFrozenView(sharedBuffer, "unsafe_frozen_croaring"); 131 | expect(view.isFrozen).to.be.true; 132 | expect(view.toArray()).to.deep.equal(values); 133 | 134 | const copy = view.clone(); 135 | expect(copy.isFrozen).to.be.false; 136 | expect(copy.toArray()).to.deep.equal(values); 137 | expect(copy.tryAdd(4)).to.be.true; 138 | }); 139 | } 140 | }); 141 | 142 | describe("asReadonlyView", () => { 143 | it("offers a readonly view", () => { 144 | const bitmap = new RoaringBitmap32([1, 2, 3]); 145 | const readonlyView = bitmap.asReadonlyView() as RoaringBitmap32; 146 | expect(readonlyView).to.not.eq(bitmap); 147 | expect(readonlyView.isFrozen).to.be.true; 148 | expect(readonlyView.asReadonlyView()).eq(readonlyView); 149 | expect(bitmap.asReadonlyView()).to.equal(readonlyView); 150 | 151 | expect(() => readonlyView.copyFrom([1])).to.throw(ERROR_FROZEN); 152 | expect(() => readonlyView.add(1)).to.throw(ERROR_FROZEN); 153 | expect(() => readonlyView.tryAdd(1)).to.throw(ERROR_FROZEN); 154 | expect(() => readonlyView.addMany([1])).to.throw(ERROR_FROZEN); 155 | expect(() => readonlyView.remove(1)).to.throw(ERROR_FROZEN); 156 | expect(() => readonlyView.removeMany([1])).to.throw(ERROR_FROZEN); 157 | expect(() => readonlyView.delete(1)).to.throw(ERROR_FROZEN); 158 | expect(() => readonlyView.clear()).to.throw(ERROR_FROZEN); 159 | expect(() => readonlyView.orInPlace([1])).to.throw(ERROR_FROZEN); 160 | expect(() => readonlyView.andNotInPlace([1])).to.throw(ERROR_FROZEN); 161 | expect(() => readonlyView.andInPlace([1])).to.throw(ERROR_FROZEN); 162 | expect(() => readonlyView.xorInPlace([1])).to.throw(ERROR_FROZEN); 163 | expect(() => readonlyView.flipRange(0, 10)).to.throw(ERROR_FROZEN); 164 | expect(() => readonlyView.addRange(0, 9)).to.throw(ERROR_FROZEN); 165 | expect(() => readonlyView.removeRange(0, 8)).to.throw(ERROR_FROZEN); 166 | expect(() => readonlyView.removeRunCompression()).to.throw(ERROR_FROZEN); 167 | expect(() => readonlyView.runOptimize()).to.throw(ERROR_FROZEN); 168 | expect(() => readonlyView.shrinkToFit()).to.throw(ERROR_FROZEN); 169 | 170 | bitmap.add(100); 171 | expect(readonlyView.toArray()).to.deep.equal([1, 2, 3, 100]); 172 | expect(readonlyView.size).to.equal(4); 173 | bitmap.add(200); 174 | expect(readonlyView.toArray()).to.deep.equal([1, 2, 3, 100, 200]); 175 | expect(readonlyView.size).to.equal(5); 176 | expect(readonlyView.size).to.equal(5); 177 | 178 | bitmap.freeze(); 179 | expect(readonlyView.isFrozen).to.be.true; 180 | expect(bitmap.asReadonlyView()).eq(readonlyView); 181 | }); 182 | }); 183 | 184 | it("returns this if the bitmap is frozen", () => { 185 | const bitmap = new RoaringBitmap32(); 186 | bitmap.freeze(); 187 | expect(bitmap.asReadonlyView()).to.equal(bitmap); 188 | }); 189 | }); 190 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.import.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import { expect } from "chai"; 3 | 4 | describe("RoaringBitmap32 import", () => { 5 | it('exports itself with a "default" property', () => { 6 | const required = require("../../RoaringBitmap32"); 7 | expect(!!required).to.be.true; 8 | expect(required === required.default).to.be.true; 9 | }); 10 | 11 | it("supports typescript \"import RoaringBitmap32 from 'roaring/RoaringBitmap32'\" syntax", () => { 12 | expect(RoaringBitmap32 === require("../../RoaringBitmap32")).eq(true); 13 | }); 14 | 15 | it("is a class", () => { 16 | expect(typeof RoaringBitmap32).eq("function"); 17 | expect(RoaringBitmap32.prototype.constructor).eq(RoaringBitmap32); 18 | }); 19 | 20 | it("can be called as a normal function", () => { 21 | const bitmap = (RoaringBitmap32 as any as () => any)(); 22 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.serialization-file.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import { expect, use as chaiUse } from "chai"; 3 | import path from "path"; 4 | import fs from "fs"; 5 | import type { FileSerializationDeserializationFormatType } from "../.."; 6 | import { FileDeserializationFormat, FileSerializationFormat } from "../.."; 7 | 8 | const tmpDir = path.resolve(__dirname, "..", "..", ".tmp", "tests"); 9 | 10 | chaiUse(require("chai-as-promised")); 11 | 12 | describe("RoaringBitmap32 file serialization", () => { 13 | before(() => { 14 | if (!fs.existsSync(tmpDir)) { 15 | fs.mkdirSync(tmpDir, { recursive: true }); 16 | } 17 | }); 18 | 19 | describe("FileSerializationFormat", () => { 20 | it("should have the right values", () => { 21 | expect(FileSerializationFormat.croaring).eq("croaring"); 22 | expect(FileSerializationFormat.portable).eq("portable"); 23 | expect(FileSerializationFormat.uint32_array).eq("uint32_array"); 24 | expect(FileSerializationFormat.unsafe_frozen_croaring).eq("unsafe_frozen_croaring"); 25 | expect(FileSerializationFormat.comma_separated_values).eq("comma_separated_values"); 26 | expect(FileSerializationFormat.tab_separated_values).eq("tab_separated_values"); 27 | expect(FileSerializationFormat.newline_separated_values).eq("newline_separated_values"); 28 | expect(FileSerializationFormat.json_array).eq("json_array"); 29 | 30 | expect(Object.values(FileSerializationFormat)).to.deep.eq([ 31 | "croaring", 32 | "portable", 33 | "unsafe_frozen_croaring", 34 | "uint32_array", 35 | "comma_separated_values", 36 | "tab_separated_values", 37 | "newline_separated_values", 38 | "json_array", 39 | ]); 40 | 41 | expect(RoaringBitmap32.FileSerializationFormat).to.eq(FileSerializationFormat); 42 | 43 | expect(new RoaringBitmap32().FileSerializationFormat).to.eq(FileSerializationFormat); 44 | }); 45 | }); 46 | 47 | describe("FileDeserializationFormat", () => { 48 | it("should have the right values", () => { 49 | expect(FileDeserializationFormat.croaring).eq("croaring"); 50 | expect(FileDeserializationFormat.portable).eq("portable"); 51 | expect(FileDeserializationFormat.unsafe_frozen_croaring).eq("unsafe_frozen_croaring"); 52 | expect(FileDeserializationFormat.unsafe_frozen_portable).eq("unsafe_frozen_portable"); 53 | expect(FileDeserializationFormat.comma_separated_values).eq("comma_separated_values"); 54 | expect(FileDeserializationFormat.tab_separated_values).eq("tab_separated_values"); 55 | expect(FileDeserializationFormat.newline_separated_values).eq("newline_separated_values"); 56 | expect(FileDeserializationFormat.json_array).eq("json_array"); 57 | 58 | expect(Object.values(FileDeserializationFormat)).to.deep.eq([ 59 | "croaring", 60 | "portable", 61 | "unsafe_frozen_croaring", 62 | "unsafe_frozen_portable", 63 | "uint32_array", 64 | "comma_separated_values", 65 | "tab_separated_values", 66 | "newline_separated_values", 67 | "json_array", 68 | ]); 69 | 70 | expect(RoaringBitmap32.FileDeserializationFormat).to.eq(FileDeserializationFormat); 71 | 72 | expect(new RoaringBitmap32().FileDeserializationFormat).to.eq(FileDeserializationFormat); 73 | }); 74 | }); 75 | 76 | it("serialize and deserialize empty bitmaps in various formats", async () => { 77 | const formats: FileSerializationDeserializationFormatType[] = [ 78 | "portable", 79 | "croaring", 80 | "unsafe_frozen_croaring", 81 | "uint32_array", 82 | "comma_separated_values", 83 | "tab_separated_values", 84 | "newline_separated_values", 85 | "json_array", 86 | ]; 87 | for (const format of formats) { 88 | const tmpFilePath = path.resolve(tmpDir, `test-ϴ-${format}.bin`); 89 | await new RoaringBitmap32().serializeFileAsync(tmpFilePath, format); 90 | expect((await RoaringBitmap32.deserializeFileAsync(tmpFilePath, format)).toArray()).to.deep.equal([]); 91 | } 92 | }); 93 | 94 | it("serialize and deserialize in various formats", async () => { 95 | for (const format of ["portable", "croaring", "unsafe_frozen_croaring", "uint32_array"] as const) { 96 | const tmpFilePath = path.resolve(tmpDir, `test-1-${format}.bin`); 97 | const data = [1, 2, 3, 100, 0xfffff, 0xffffffff]; 98 | await new RoaringBitmap32(data).serializeFileAsync(tmpFilePath, format); 99 | expect((await RoaringBitmap32.deserializeFileAsync(tmpFilePath, format)).toArray()).to.deep.equal(data); 100 | } 101 | }); 102 | 103 | it("serializeFileAsync truncates file if it already exists", async () => { 104 | const tmpFilePath = path.resolve(tmpDir, `test-truncate.bin`); 105 | await fs.promises.writeFile(tmpFilePath, Buffer.alloc(10000)); 106 | await new RoaringBitmap32([1, 2, 3]).serializeFileAsync(tmpFilePath, "portable"); 107 | expect((await RoaringBitmap32.deserializeFileAsync(tmpFilePath, "portable")).toArray()).to.deep.equal([1, 2, 3]); 108 | }); 109 | 110 | it("throws ENOENT if file does not exist", async () => { 111 | const tmpFilePath = path.resolve(tmpDir, `test-ENOENT.bin`); 112 | let error: any; 113 | try { 114 | await RoaringBitmap32.deserializeFileAsync(tmpFilePath, "portable"); 115 | } catch (e) { 116 | error = e; 117 | } 118 | expect(error).to.be.an.instanceOf(Error); 119 | expect(error.message).to.match(/^ENOENT, No such file or directory/); 120 | expect(error.code).to.equal("ENOENT"); 121 | expect(error.syscall).to.equal("open"); 122 | expect(error.path).to.equal(tmpFilePath); 123 | }); 124 | 125 | it("serializes to comma_separated_values", async () => { 126 | const tmpFilePath = path.resolve(tmpDir, `test-csv.csv`); 127 | const bmp = new RoaringBitmap32([1, 2, 3, 100, 14120, 3481983]); 128 | bmp.addRange(0x100, 0x120); 129 | await bmp.serializeFileAsync(tmpFilePath, "comma_separated_values"); 130 | const text = await fs.promises.readFile(tmpFilePath, "utf8"); 131 | expect(text).to.equal( 132 | "1,2,3,100,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,14120,3481983", 133 | ); 134 | }); 135 | 136 | it("serializes to newline_separated_values", async () => { 137 | const tmpFilePath = path.resolve(tmpDir, `test-nlf.csv`); 138 | const bmp = new RoaringBitmap32([1, 2, 3, 100, 14120, 3481983]); 139 | bmp.addRange(0x100, 0x120); 140 | await bmp.serializeFileAsync(tmpFilePath, "newline_separated_values"); 141 | const text = await fs.promises.readFile(tmpFilePath, "utf8"); 142 | expect(text).to.equal( 143 | "1,2,3,100,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,14120,3481983,".replace( 144 | /,/g, 145 | "\n", 146 | ), 147 | ); 148 | }); 149 | 150 | it("serializes to tab_separated_values", async () => { 151 | const tmpFilePath = path.resolve(tmpDir, `test-ntab.csv`); 152 | const bmp = new RoaringBitmap32([1, 2, 3, 100, 14120, 3481983]); 153 | bmp.addRange(0x100, 0x120); 154 | await bmp.serializeFileAsync(tmpFilePath, "tab_separated_values"); 155 | const text = await fs.promises.readFile(tmpFilePath, "utf8"); 156 | expect(text).to.equal( 157 | "1,2,3,100,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,14120,3481983".replace( 158 | /,/g, 159 | "\t", 160 | ), 161 | ); 162 | }); 163 | 164 | it("serializes to an empty json array", async () => { 165 | const tmpFilePath = path.resolve(tmpDir, `test-json-array-empty.csv`); 166 | await new RoaringBitmap32().serializeFileAsync(tmpFilePath, "json_array"); 167 | expect(await fs.promises.readFile(tmpFilePath, "utf8")).to.equal("[]"); 168 | }); 169 | 170 | it("serializes to a json array", async () => { 171 | const tmpFilePath = path.resolve(tmpDir, `test-json-array.csv`); 172 | const bmp = new RoaringBitmap32([1, 2, 3, 100, 14120, 3481983]); 173 | bmp.addRange(0x100, 0x120); 174 | await bmp.serializeFileAsync(tmpFilePath, "json_array"); 175 | const text = await fs.promises.readFile(tmpFilePath, "utf8"); 176 | expect(text).to.equal( 177 | "[1,2,3,100,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,14120,3481983]", 178 | ); 179 | }); 180 | }); 181 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap32.worker-threads.test.ts: -------------------------------------------------------------------------------- 1 | import { resolve as pathResolve } from "path"; 2 | 3 | // Check that node is 12 or higher 4 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 5 | if (nodeVersion > 12) { 6 | describe("RoaringBitmap32 worker-threads", () => { 7 | it("can be used and works inside a worker thread", () => { 8 | // eslint-disable-next-line node/no-unsupported-features/node-builtins 9 | const { Worker } = require("worker_threads"); 10 | const worker = new Worker(pathResolve(__dirname, "worker-thread-test.js")); 11 | return new Promise((resolve, reject) => { 12 | worker.on("message", (message: any) => { 13 | if (message === "ok") { 14 | resolve(); 15 | } else { 16 | reject(new Error(message)); 17 | } 18 | }); 19 | worker.on("error", reject); 20 | worker.on("exit", (code: any) => { 21 | if (code !== 0) { 22 | reject(new Error(`Worker stopped with exit code ${code}`)); 23 | } 24 | }); 25 | }); 26 | }); 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/RoaringBitmap33.deserializeAsync.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import { expect } from "chai"; 3 | 4 | describe("RoaringBitmap32 deserializeAsync", () => { 5 | describe("async/await", () => { 6 | describe("empty buffer", () => { 7 | it("deserializes an empty buffer (non portable)", async () => { 8 | const bitmap = await RoaringBitmap32.deserializeAsync(Buffer.from([]), false); 9 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 10 | expect(bitmap.size).eq(0); 11 | }); 12 | 13 | it("deserializes an empty buffer (portable)", async () => { 14 | const bitmap = await RoaringBitmap32.deserializeAsync(Buffer.from([]), true); 15 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 16 | expect(bitmap.size).eq(0); 17 | }); 18 | }); 19 | 20 | describe("empty bitmap", () => { 21 | it("deserializes an empty bitmap (non portable)", async () => { 22 | const bitmap = await RoaringBitmap32.deserializeAsync(Buffer.from([1, 0, 0, 0, 0]), false); 23 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 24 | expect(bitmap.size).eq(0); 25 | }); 26 | 27 | it("deserializes an empty bitmap (portable)", async () => { 28 | const bitmap = await RoaringBitmap32.deserializeAsync(Buffer.from([58, 48, 0, 0, 0, 0, 0, 0]), true); 29 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 30 | expect(bitmap.size).eq(0); 31 | }); 32 | }); 33 | 34 | it("deserializes simple bitmap", async () => { 35 | const values = [1, 2, 100, 101, 105, 109, 0x7fffffff, 0xfffffffe, 0xffffffff]; 36 | 37 | const bufferNonPortable = new RoaringBitmap32(values).serialize(false); 38 | const bufferPortable = new RoaringBitmap32(values).serialize(true); 39 | 40 | const promises: Promise[] = []; 41 | for (let i = 0; i < 10; ++i) { 42 | promises.push(RoaringBitmap32.deserializeAsync(bufferNonPortable, false)); 43 | promises.push(RoaringBitmap32.deserializeAsync(bufferPortable, true)); 44 | } 45 | 46 | for (const bitmap of await Promise.all(promises)) { 47 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 48 | expect(bitmap.toArray()).deep.equal(values); 49 | } 50 | }); 51 | 52 | it("deserializes larger bitmap", async () => { 53 | const bitmap = new RoaringBitmap32(); 54 | let rnd = 112043213; 55 | for (let i = 0; i < 1000; ++i) { 56 | rnd = ((rnd + i * 25253) * 3924461) >>> 0; 57 | bitmap.add(rnd); 58 | } 59 | const bufferNonPortable = bitmap.serialize(false); 60 | const bufferPortable = bitmap.serialize(true); 61 | 62 | const promises: Promise[] = []; 63 | for (let i = 0; i < 5; ++i) { 64 | promises.push(RoaringBitmap32.deserializeAsync(bufferNonPortable, false)); 65 | promises.push(RoaringBitmap32.deserializeAsync(bufferPortable, true)); 66 | } 67 | 68 | const resolved = await Promise.all(promises); 69 | 70 | for (const b of resolved) { 71 | expect(b).to.be.instanceOf(RoaringBitmap32); 72 | expect(b.isEqual(bitmap)).eq(true); 73 | } 74 | }); 75 | 76 | it("propagates deserialization errors", async () => { 77 | const wrongBuffer = Buffer.from([99, 98, 97]); 78 | const promise = RoaringBitmap32.deserializeAsync(wrongBuffer, false); 79 | let error: any; 80 | try { 81 | await promise; 82 | } catch (e) { 83 | error = e; 84 | } 85 | expect(error.message).eq("RoaringBitmap32 deserialization - invalid portable header byte"); 86 | }); 87 | }); 88 | 89 | describe("callback", () => { 90 | describe("empty buffer", () => { 91 | it("deserializes an empty buffer (non portable)", () => { 92 | return new Promise((resolve, reject) => { 93 | expect( 94 | RoaringBitmap32.deserializeAsync(Buffer.from([]), false, (error, bitmap) => { 95 | if (error) { 96 | reject(error); 97 | return; 98 | } 99 | try { 100 | expect(error).to.be.null; 101 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 102 | expect(bitmap!.size).eq(0); 103 | resolve(); 104 | } catch (e) { 105 | reject(e); 106 | } 107 | }), 108 | ).to.be.undefined; 109 | }); 110 | }); 111 | 112 | it("deserializes an empty buffer (portable)", () => { 113 | return new Promise((resolve, reject) => { 114 | expect( 115 | RoaringBitmap32.deserializeAsync(Buffer.from([]), true, (error, bitmap) => { 116 | if (error) { 117 | reject(error); 118 | return; 119 | } 120 | try { 121 | expect(error).to.be.null; 122 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 123 | expect(bitmap!.size).eq(0); 124 | resolve(); 125 | } catch (e) { 126 | reject(e); 127 | } 128 | }), 129 | ).to.be.undefined; 130 | }); 131 | }); 132 | }); 133 | 134 | describe("empty bitmap", () => { 135 | it("deserializes an empty bitmap (non portable)", () => { 136 | return new Promise((resolve, reject) => { 137 | expect( 138 | RoaringBitmap32.deserializeAsync(Buffer.from([1, 0, 0, 0, 0]), false, (error, bitmap) => { 139 | if (error) { 140 | reject(error); 141 | return; 142 | } 143 | try { 144 | expect(error).to.be.null; 145 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 146 | expect(bitmap!.size).eq(0); 147 | resolve(); 148 | } catch (e) { 149 | reject(e); 150 | } 151 | }), 152 | ).to.be.undefined; 153 | }); 154 | }); 155 | 156 | it("deserializes an empty bitmap (portable)", () => { 157 | return new Promise((resolve, reject) => { 158 | expect( 159 | RoaringBitmap32.deserializeAsync(Buffer.from([58, 48, 0, 0, 0, 0, 0, 0]), true, (error, bitmap) => { 160 | if (error) { 161 | reject(error); 162 | return; 163 | } 164 | try { 165 | expect(error).to.be.null; 166 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 167 | expect(bitmap!.size).eq(0); 168 | resolve(); 169 | } catch (e) { 170 | reject(e); 171 | } 172 | }), 173 | ).to.be.undefined; 174 | }); 175 | }); 176 | }); 177 | 178 | it("deserializes simple bitmap", () => { 179 | return new Promise((resolve, reject) => { 180 | const values = [1, 2, 100, 101, 105, 109, 0x7fffffff, 0xfffffffe, 0xffffffff]; 181 | 182 | const bufferNonPortable = new RoaringBitmap32(values).serialize(false); 183 | 184 | expect( 185 | RoaringBitmap32.deserializeAsync(bufferNonPortable, false, (error, bitmap) => { 186 | if (error) { 187 | reject(error); 188 | return; 189 | } 190 | try { 191 | expect(error).to.be.null; 192 | expect(bitmap).to.be.instanceOf(RoaringBitmap32); 193 | expect(bitmap!.toArray()).deep.equal(values); 194 | resolve(); 195 | } catch (e) { 196 | reject(e); 197 | } 198 | }), 199 | ).to.be.undefined; 200 | }); 201 | }); 202 | 203 | it("propagates deserialization errors", () => { 204 | return new Promise((resolve, reject) => { 205 | const wrongBuffer = Buffer.from([99, 98, 97]); 206 | expect( 207 | RoaringBitmap32.deserializeAsync(wrongBuffer, false, (error, bitmap) => { 208 | try { 209 | expect(bitmap).to.be.undefined; 210 | expect(error!.message).eq("RoaringBitmap32 deserialization - invalid portable header byte"); 211 | resolve(); 212 | } catch (e) { 213 | reject(e); 214 | } 215 | }), 216 | ).to.be.undefined; 217 | }); 218 | }); 219 | }); 220 | }); 221 | -------------------------------------------------------------------------------- /test/RoaringBitmap32/worker-thread-test.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line node/no-unsupported-features/node-builtins 2 | const { parentPort } = require("worker_threads"); 3 | const { expect } = require("chai"); 4 | 5 | process.on("uncaughtException", (err) => { 6 | // eslint-disable-next-line no-console 7 | console.error("worker thread uncaughtException", err); 8 | if (parentPort) { 9 | parentPort.postMessage(err); 10 | } 11 | }); 12 | 13 | async function main() { 14 | const { RoaringBitmap32 } = require("../.."); 15 | 16 | const bitmap = new RoaringBitmap32(); 17 | bitmap.add(1); 18 | bitmap.add(2); 19 | bitmap.add(100); 20 | bitmap.add(101); 21 | 22 | bitmap.addRange(105, 109); 23 | bitmap.addMany([0x7fffffff, 0xfffffffe, 0xffffffff]); 24 | 25 | expect(bitmap.toArray()).deep.equal([1, 2, 100, 101, 105, 106, 107, 108, 0x7fffffff, 0xfffffffe, 0xffffffff]); 26 | 27 | const serialized = await bitmap.serializeAsync("portable"); 28 | 29 | const bitmap2 = await RoaringBitmap32.deserializeAsync(serialized, "portable"); 30 | expect(bitmap2.toArray()).deep.equal([1, 2, 100, 101, 105, 106, 107, 108, 0x7fffffff, 0xfffffffe, 0xffffffff]); 31 | 32 | bitmap2.add(121); 33 | 34 | const bitmap3 = RoaringBitmap32.orMany([bitmap, bitmap2]); 35 | 36 | expect(Array.from(bitmap3)).deep.equal([1, 2, 100, 101, 105, 106, 107, 108, 121, 0x7fffffff, 0xfffffffe, 0xffffffff]); 37 | } 38 | 39 | main() 40 | .then(() => { 41 | if (parentPort) { 42 | parentPort.postMessage("ok"); 43 | } else { 44 | // eslint-disable-next-line no-console 45 | console.log("ok"); 46 | } 47 | }) 48 | .catch((e) => { 49 | // eslint-disable-next-line no-console 50 | console.error("worker thread error", e); 51 | if (parentPort) { 52 | parentPort.postMessage(e); 53 | } 54 | }); 55 | -------------------------------------------------------------------------------- /test/RoaringBitmap32Iterator/RoaringBitmap32Iterator.import.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32Iterator from "../../RoaringBitmap32Iterator"; 2 | import { expect } from "chai"; 3 | 4 | describe("RoaringBitmap32Iterator import", () => { 5 | it('exports itself with a "default" property', () => { 6 | const required = require("../../RoaringBitmap32Iterator"); 7 | expect(!!required).to.be.true; 8 | expect(required === required.default).to.be.true; 9 | }); 10 | 11 | it("supports typescript \"import RoaringBitmap32Iterator from 'roaring/RoaringBitmap32Iterator'\" syntax", () => { 12 | expect(RoaringBitmap32Iterator === require("../../RoaringBitmap32Iterator")).eq(true); 13 | }); 14 | 15 | it("is a class", () => { 16 | expect(typeof RoaringBitmap32Iterator).eq("function"); 17 | expect(RoaringBitmap32Iterator.prototype.constructor).eq(RoaringBitmap32Iterator); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /test/RoaringBitmap32Iterator/RoaringBitmap32Iterator.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32 from "../../RoaringBitmap32"; 2 | import RoaringBitmap32Iterator from "../../RoaringBitmap32Iterator"; 3 | import { expect } from "chai"; 4 | 5 | describe("RoaringBitmap32Iterator", () => { 6 | describe("constructor", () => { 7 | it("is a class", () => { 8 | expect(typeof RoaringBitmap32).eq("function"); 9 | }); 10 | 11 | it("creates an empty iterator with no arguments", () => { 12 | const iter = new RoaringBitmap32Iterator(); 13 | expect(iter).to.be.instanceOf(RoaringBitmap32Iterator); 14 | }); 15 | 16 | it("creates an iterator with a RoaringBitmap32", () => { 17 | const bitmap = new RoaringBitmap32([3, 4, 5]); 18 | const iter = new RoaringBitmap32Iterator(bitmap); 19 | expect(iter).to.be.instanceOf(RoaringBitmap32Iterator); 20 | }); 21 | 22 | it("throws an exception if called with a non RoaringBitmap32", () => { 23 | expect(() => new RoaringBitmap32Iterator(123 as any)).to.throw(Error); 24 | expect(() => new RoaringBitmap32Iterator([123] as any)).to.throw(Error); 25 | }); 26 | }); 27 | 28 | describe("next", () => { 29 | it("is a function", () => { 30 | const iter = new RoaringBitmap32Iterator(); 31 | expect(typeof iter.next).eq("function"); 32 | }); 33 | 34 | it("returns an empty result if iterator is created without arguments", () => { 35 | const iter = new RoaringBitmap32Iterator(); 36 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 37 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 38 | }); 39 | 40 | it("returns an empty result if iterator is created with an empty RoaringBitmap32", () => { 41 | const iter = new RoaringBitmap32Iterator(new RoaringBitmap32()); 42 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 43 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 44 | }); 45 | 46 | it("allows iterating a small array", () => { 47 | const iter = new RoaringBitmap32Iterator(new RoaringBitmap32([123, 456, 999, 1000])); 48 | expect(iter.next()).deep.equal({ value: 123, done: false }); 49 | expect(iter.next()).deep.equal({ value: 456, done: false }); 50 | expect(iter.next()).deep.equal({ value: 999, done: false }); 51 | expect(iter.next()).deep.equal({ value: 1000, done: false }); 52 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 53 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 54 | expect(iter.next()).deep.equal({ value: undefined, done: true }); 55 | }); 56 | }); 57 | 58 | describe("Symbol.iterator", () => { 59 | it("is a function", () => { 60 | const iter = new RoaringBitmap32Iterator(); 61 | expect(typeof iter[Symbol.iterator]).deep.equal("function"); 62 | }); 63 | 64 | it("returns this", () => { 65 | const iter = new RoaringBitmap32Iterator(); 66 | expect(iter[Symbol.iterator]()).eq(iter); 67 | }); 68 | 69 | it("allows foreach (empty)", () => { 70 | const iter = new RoaringBitmap32Iterator(); 71 | expect(iter.next()).deep.equal({ done: true, value: undefined }); 72 | }); 73 | 74 | it("allows foreach (small array)", () => { 75 | const iter = new RoaringBitmap32Iterator(new RoaringBitmap32([123, 456, 789])); 76 | const values = []; 77 | for (const x of iter) { 78 | values.push(x); 79 | } 80 | expect(values).deep.equal([123, 456, 789]); 81 | }); 82 | 83 | it("allows Array.from", () => { 84 | const iter = new RoaringBitmap32Iterator(new RoaringBitmap32([123, 456, 789])); 85 | const values = Array.from(iter); 86 | expect(values).deep.equal([123, 456, 789]); 87 | }); 88 | }); 89 | 90 | describe("buffer (number)", () => { 91 | it("iterates, buffer 1, bitmap 0", () => { 92 | const bitmap = new RoaringBitmap32(); 93 | const iterator = new RoaringBitmap32Iterator(bitmap, 1); 94 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 95 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 96 | }); 97 | 98 | it("iterates, buffer 2, bitmap 0", () => { 99 | const bitmap = new RoaringBitmap32(); 100 | const iterator = new RoaringBitmap32Iterator(bitmap, 2); 101 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 102 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 103 | }); 104 | 105 | it("iterates, buffer 1, bitmap 1", () => { 106 | const bitmap = new RoaringBitmap32([5]); 107 | const iterator = new RoaringBitmap32Iterator(bitmap, 1); 108 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 109 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 110 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 111 | }); 112 | 113 | it("iterates, buffer 2, bitmap 1", () => { 114 | const bitmap = new RoaringBitmap32([5]); 115 | const iterator = new RoaringBitmap32Iterator(bitmap, 2); 116 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 117 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 118 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 119 | }); 120 | 121 | it("iterates, buffer 1, bitmap 3", () => { 122 | const bitmap = new RoaringBitmap32([5, 7, 9]); 123 | const iterator = new RoaringBitmap32Iterator(bitmap, 1); 124 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 125 | expect(iterator.next()).deep.equal({ value: 7, done: false }); 126 | expect(iterator.next()).deep.equal({ value: 9, done: false }); 127 | }); 128 | 129 | it("iterates, buffer 2, bitmap 3", () => { 130 | const bitmap = new RoaringBitmap32([5, 7, 9]); 131 | const iterator = new RoaringBitmap32Iterator(bitmap, 2); 132 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 133 | expect(iterator.next()).deep.equal({ value: 7, done: false }); 134 | expect(iterator.next()).deep.equal({ value: 9, done: false }); 135 | }); 136 | }); 137 | 138 | describe("buffer (Uint32Array)", () => { 139 | it("iterates, buffer 1, bitmap 0", () => { 140 | const bitmap = new RoaringBitmap32(); 141 | const buffer = new Uint32Array(1); 142 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 143 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 144 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 145 | expect(Array.from(buffer)).deep.equal([0]); 146 | }); 147 | 148 | it("iterates, buffer 2, bitmap 0", () => { 149 | const bitmap = new RoaringBitmap32(); 150 | const buffer = new Uint32Array(2); 151 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 152 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 153 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 154 | expect(Array.from(buffer)).deep.equal([0, 0]); 155 | }); 156 | 157 | it("iterates, buffer 1, bitmap 1", () => { 158 | const bitmap = new RoaringBitmap32([5]); 159 | const buffer = new Uint32Array(1); 160 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 161 | expect(buffer[0]).eq(0); 162 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 163 | expect(buffer[0]).eq(5); 164 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 165 | expect(buffer[0]).eq(5); 166 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 167 | expect(buffer[0]).eq(5); 168 | }); 169 | 170 | it("iterates, buffer 2, bitmap 1", () => { 171 | const bitmap = new RoaringBitmap32([5]); 172 | const buffer = new Uint32Array(2); 173 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 174 | expect(Array.from(buffer)).deep.equal([0, 0]); 175 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 176 | expect(Array.from(buffer)).deep.equal([5, 0]); 177 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 178 | expect(Array.from(buffer)).deep.equal([5, 0]); 179 | expect(iterator.next()).deep.equal({ value: undefined, done: true }); 180 | expect(Array.from(buffer)).deep.equal([5, 0]); 181 | }); 182 | 183 | it("iterates, buffer 1, bitmap 3", () => { 184 | const bitmap = new RoaringBitmap32([5, 7, 9]); 185 | const buffer = new Uint32Array(1); 186 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 187 | expect(buffer[0]).eq(0); 188 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 189 | expect(buffer[0]).eq(5); 190 | expect(iterator.next()).deep.equal({ value: 7, done: false }); 191 | expect(buffer[0]).eq(7); 192 | expect(iterator.next()).deep.equal({ value: 9, done: false }); 193 | expect(buffer[0]).eq(9); 194 | }); 195 | 196 | it("iterates, buffer 2, bitmap 3", () => { 197 | const bitmap = new RoaringBitmap32([5, 7, 9]); 198 | const buffer = new Uint32Array(2); 199 | const iterator = new RoaringBitmap32Iterator(bitmap, buffer); 200 | expect(Array.from(buffer)).deep.equal([0, 0]); 201 | expect(iterator.next()).deep.equal({ value: 5, done: false }); 202 | expect(Array.from(buffer)).deep.equal([5, 7]); 203 | expect(iterator.next()).deep.equal({ value: 7, done: false }); 204 | expect(Array.from(buffer)).deep.equal([5, 7]); 205 | expect(iterator.next()).deep.equal({ value: 9, done: false }); 206 | expect(buffer[0]).eq(9); 207 | }); 208 | }); 209 | 210 | it("throws if the bitmap is changed while iterating", () => { 211 | const bitmap = new RoaringBitmap32(); 212 | bitmap.addRange(0, 1050); 213 | 214 | function doNothing(_v: any) {} 215 | 216 | let error: any; 217 | try { 218 | let n = 0; 219 | for (const v of new RoaringBitmap32Iterator(bitmap, 256)) { 220 | if (n++ === 0) { 221 | bitmap.add(999999); 222 | } else { 223 | doNothing(v); 224 | } 225 | } 226 | } catch (e) { 227 | error = e; 228 | } 229 | expect(error.message).eq("RoaringBitmap32 iterator - bitmap changed while iterating"); 230 | }); 231 | 232 | describe("RoaringBitmap32 iterable", () => { 233 | it("returns a RoaringBitmap32Iterator", () => { 234 | const bitmap = new RoaringBitmap32(); 235 | const iterator = bitmap[Symbol.iterator](); 236 | expect(iterator).to.be.instanceOf(RoaringBitmap32Iterator); 237 | expect(typeof iterator.next).eq("function"); 238 | }); 239 | 240 | it("has both [Symbol.iterator] and iterator", () => { 241 | const bitmap = new RoaringBitmap32(); 242 | expect(bitmap.iterator).eq(bitmap[Symbol.iterator]); 243 | }); 244 | 245 | it("returns an empty iterator for an empty bitmap", () => { 246 | const bitmap = new RoaringBitmap32(); 247 | const iterator = bitmap[Symbol.iterator](); 248 | expect(iterator.next()).deep.equal({ 249 | done: true, 250 | value: undefined, 251 | }); 252 | expect(iterator.next()).deep.equal({ 253 | done: true, 254 | value: undefined, 255 | }); 256 | }); 257 | it("iterates a non empty bitmap", () => { 258 | const bitmap = new RoaringBitmap32([0xffffffff, 3]); 259 | const iterator = bitmap[Symbol.iterator](); 260 | expect(iterator.next()).deep.equal({ 261 | done: false, 262 | value: 3, 263 | }); 264 | expect(iterator.next()).deep.equal({ 265 | done: false, 266 | value: 0xffffffff, 267 | }); 268 | expect(iterator.next()).deep.equal({ 269 | done: true, 270 | value: undefined, 271 | }); 272 | }); 273 | }); 274 | }); 275 | -------------------------------------------------------------------------------- /test/RoaringBitmap32ReverseIterator/RoaringBitmap32ReverseIterator.import.test.ts: -------------------------------------------------------------------------------- 1 | import RoaringBitmap32ReverseIterator from "../../RoaringBitmap32ReverseIterator"; 2 | import { expect } from "chai"; 3 | 4 | describe("RoaringBitmap32ReverseIterator import", () => { 5 | it('exports itself with a "default" property', () => { 6 | const required = require("../../RoaringBitmap32ReverseIterator"); 7 | expect(!!required).to.be.true; 8 | expect(required === required.default).to.be.true; 9 | }); 10 | 11 | it("supports typescript \"import RoaringBitmap32ReverseIterator from 'roaring/RoaringBitmap32ReverseIterator'\" syntax", () => { 12 | expect(RoaringBitmap32ReverseIterator === require("../../RoaringBitmap32ReverseIterator")).eq(true); 13 | }); 14 | 15 | it("is a class", () => { 16 | expect(typeof RoaringBitmap32ReverseIterator).eq("function"); 17 | expect(RoaringBitmap32ReverseIterator.prototype.constructor).eq(RoaringBitmap32ReverseIterator); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /test/aligned-buffers.test.ts: -------------------------------------------------------------------------------- 1 | import { 2 | asBuffer, 3 | RoaringBitmap32, 4 | bufferAlignedAlloc, 5 | bufferAlignedAllocUnsafe, 6 | isBufferAligned, 7 | ensureBufferAligned, 8 | bufferAlignedAllocShared, 9 | bufferAlignedAllocSharedUnsafe, 10 | } from ".."; 11 | import { expect } from "chai"; 12 | 13 | describe("asBuffer", () => { 14 | it("returns a buffer", () => { 15 | const input = Buffer.alloc(3); 16 | expect(input).to.eq(input); 17 | }); 18 | 19 | it("wraps an ArrayBuffer", () => { 20 | const input = new ArrayBuffer(3); 21 | const output = asBuffer(input); 22 | expect(output).to.be.instanceOf(Buffer); 23 | expect(output.buffer).to.eq(input); 24 | }); 25 | 26 | it("wraps a SharedArrayBuffer", () => { 27 | const input = new SharedArrayBuffer(3); 28 | const output = asBuffer(input); 29 | expect(output).to.be.instanceOf(Buffer); 30 | expect(output.buffer).to.eq(input); 31 | }); 32 | 33 | it("wraps an arraybuffer view", () => { 34 | for (const ctor of [ 35 | Uint8Array, 36 | Uint16Array, 37 | Uint32Array, 38 | Int8Array, 39 | Int16Array, 40 | Int32Array, 41 | Float32Array, 42 | Float64Array, 43 | Uint8ClampedArray, 44 | ]) { 45 | const input = new ctor(3); 46 | const output = asBuffer(input); 47 | expect(output).to.be.instanceOf(Buffer); 48 | expect(output.buffer).to.eq(input.buffer); 49 | expect(output.byteOffset).to.eq(0); 50 | expect(output.byteLength).to.eq(input.byteLength); 51 | } 52 | }); 53 | 54 | it("wraps an arraybuffer view with offset", () => { 55 | for (const ctor of [ 56 | Uint8Array, 57 | Uint16Array, 58 | Uint32Array, 59 | Int8Array, 60 | Int16Array, 61 | Int32Array, 62 | Float32Array, 63 | Float64Array, 64 | Uint8ClampedArray, 65 | ]) { 66 | const sourceArrayBuffer = new ArrayBuffer(20 * ctor.BYTES_PER_ELEMENT); 67 | const input = new ctor(sourceArrayBuffer, 8, 8); 68 | const output = asBuffer(input); 69 | expect(output).to.be.instanceOf(Buffer); 70 | expect(output.buffer).to.eq(input.buffer); 71 | expect(output.byteOffset).to.eq(input.byteOffset); 72 | expect(output.byteLength).to.eq(input.byteLength); 73 | } 74 | }); 75 | }); 76 | 77 | describe("bufferAlignedAlloc", () => { 78 | it("exposes the bufferAlignedAlloc function", () => { 79 | expect(bufferAlignedAlloc).to.be.a("function"); 80 | expect(RoaringBitmap32.bufferAlignedAlloc).to.eq(bufferAlignedAlloc); 81 | }); 82 | 83 | it("throws if first argument (size) is invalid", () => { 84 | expect(() => bufferAlignedAlloc(-1)).to.throw(); 85 | expect(() => bufferAlignedAlloc("x" as any)).to.throw(); 86 | expect(() => bufferAlignedAlloc({} as any)).to.throw(); 87 | expect(() => bufferAlignedAlloc(null as any)).to.throw(); 88 | }); 89 | 90 | it("can allocate an empty buffer", () => { 91 | const buffer = bufferAlignedAlloc(0); 92 | expect(buffer).to.be.instanceOf(Buffer); 93 | expect(buffer.buffer).to.be.instanceOf(ArrayBuffer); 94 | expect(buffer.length).to.eq(0); 95 | expect(isBufferAligned(buffer)).to.eq(true); 96 | }); 97 | 98 | it("can allocate a buffer of a given size", () => { 99 | const buffer = bufferAlignedAlloc(10, 128); 100 | expect(buffer).to.be.instanceOf(Buffer); 101 | expect(buffer.buffer).to.be.instanceOf(ArrayBuffer); 102 | expect(buffer.length).to.eq(10); 103 | expect(isBufferAligned(buffer, 128)).to.eq(true); 104 | }); 105 | 106 | it("is initialized with zeroes", () => { 107 | const buffer = bufferAlignedAlloc(2340); 108 | expect(buffer).to.be.instanceOf(Buffer); 109 | expect(buffer.buffer).to.be.instanceOf(ArrayBuffer); 110 | expect(buffer.length).to.eq(2340); 111 | for (let i = 0; i < 2340; ++i) { 112 | expect(buffer[i]).to.eq(0); 113 | } 114 | expect(isBufferAligned(buffer)).to.eq(true); 115 | expect(RoaringBitmap32.isBufferAligned(buffer, 32)).to.eq(true); 116 | }); 117 | }); 118 | 119 | describe("bufferAlignedAllocUnsafe", () => { 120 | it("exposes the bufferAlignedAllocUnsafe function", () => { 121 | expect(bufferAlignedAllocUnsafe).to.be.a("function"); 122 | expect(RoaringBitmap32.bufferAlignedAllocUnsafe).to.eq(bufferAlignedAllocUnsafe); 123 | }); 124 | 125 | it("throws if first argument (size) is invalid", () => { 126 | expect(() => bufferAlignedAllocUnsafe(-1)).to.throw(); 127 | expect(() => bufferAlignedAllocUnsafe(null as any)).to.throw(); 128 | expect(() => bufferAlignedAllocUnsafe("x" as any)).to.throw(); 129 | expect(() => bufferAlignedAllocUnsafe({} as any)).to.throw(); 130 | }); 131 | 132 | it("can allocate an empty buffer", () => { 133 | const buffer = bufferAlignedAllocUnsafe(0, 512); 134 | expect(buffer).to.be.instanceOf(Buffer); 135 | expect(buffer.length).to.eq(0); 136 | expect(isBufferAligned(buffer, 512)).to.eq(true); 137 | }); 138 | 139 | it("can allocate a buffer of a given size", () => { 140 | const buffer = bufferAlignedAllocUnsafe(10); 141 | expect(buffer).to.be.instanceOf(Buffer); 142 | expect(buffer.length).to.eq(10); 143 | expect(isBufferAligned(buffer)).to.eq(true); 144 | expect(RoaringBitmap32.isBufferAligned(buffer, 32)).to.eq(true); 145 | expect(RoaringBitmap32.isBufferAligned(buffer.buffer)).to.eq(true); 146 | }); 147 | }); 148 | 149 | describe("ensureBufferAligned", () => { 150 | it("exposes the ensureBufferAligned function", () => { 151 | expect(ensureBufferAligned).to.be.a("function"); 152 | expect(RoaringBitmap32.ensureBufferAligned).to.eq(ensureBufferAligned); 153 | }); 154 | 155 | it("returns the same buffer if it is already aligned", () => { 156 | const buffer = bufferAlignedAlloc(30); 157 | expect(ensureBufferAligned(buffer, 32)).to.eq(buffer); 158 | }); 159 | 160 | it("returns a new buffer if the buffer is not aligned", () => { 161 | const unalignedBuffer = Buffer.from(bufferAlignedAlloc(31).buffer, 1, 27); 162 | const result = ensureBufferAligned(unalignedBuffer); 163 | expect(result).to.not.eq(unalignedBuffer); 164 | expect(result).to.be.instanceOf(Buffer); 165 | expect(result.length).to.eq(27); 166 | expect(isBufferAligned(result)).to.eq(true); 167 | }); 168 | 169 | it("returns a new buffer if the buffer is not aligned, with a custom alignment", () => { 170 | const unalignedBuffer = Buffer.from(bufferAlignedAlloc(31).buffer, 1, 30); 171 | const result = ensureBufferAligned(unalignedBuffer, 256); 172 | expect(result).to.not.eq(unalignedBuffer); 173 | expect(result).to.be.instanceOf(Buffer); 174 | expect(result.length).to.eq(30); 175 | expect(isBufferAligned(result, 256)).to.eq(true); 176 | }); 177 | 178 | it("works with SharedArrayBuffer", () => { 179 | const sab = new SharedArrayBuffer(32); 180 | const unalignedBuffer = Buffer.from(sab, 1, 30); 181 | expect(unalignedBuffer.buffer).to.eq(sab); 182 | const result = ensureBufferAligned(unalignedBuffer); 183 | expect(result).to.not.eq(unalignedBuffer); 184 | expect(result).to.be.instanceOf(Buffer); 185 | expect(result.buffer).to.be.instanceOf(SharedArrayBuffer); 186 | expect(result.length).to.eq(30); 187 | expect(isBufferAligned(result)).to.eq(true); 188 | }); 189 | }); 190 | 191 | const nodeVersion = parseInt(process.versions.node.split(".")[0], 10); 192 | if (nodeVersion >= 12) { 193 | describe("bufferAlignedAllocShared", () => { 194 | it("exposes the bufferAlignedAllocShared function", () => { 195 | expect(bufferAlignedAllocShared).to.be.a("function"); 196 | expect(RoaringBitmap32.bufferAlignedAllocShared).to.eq(bufferAlignedAllocShared); 197 | }); 198 | 199 | it("throws if first argument (size) is invalid", () => { 200 | expect(() => bufferAlignedAllocShared(-1)).to.throw(); 201 | expect(() => bufferAlignedAllocShared(null as any)).to.throw(); 202 | expect(() => bufferAlignedAllocShared("x" as any)).to.throw(); 203 | expect(() => bufferAlignedAllocShared({} as any)).to.throw(); 204 | }); 205 | 206 | it("can allocate an empty buffer", () => { 207 | const buffer = bufferAlignedAllocShared(0, 512); 208 | expect(buffer).to.be.instanceOf(Buffer); 209 | expect(buffer.buffer).to.be.instanceOf(SharedArrayBuffer); 210 | expect(buffer.length).to.eq(0); 211 | expect(isBufferAligned(buffer, 512)).to.eq(true); 212 | }); 213 | 214 | it("can allocate a buffer of a given size", () => { 215 | const buffer = bufferAlignedAllocShared(10); 216 | expect(buffer).to.be.instanceOf(Buffer); 217 | expect(buffer.buffer).to.be.instanceOf(SharedArrayBuffer); 218 | expect(buffer.length).to.eq(10); 219 | expect(isBufferAligned(buffer)).to.eq(true); 220 | expect(RoaringBitmap32.isBufferAligned(buffer, 32)).to.eq(true); 221 | expect(RoaringBitmap32.isBufferAligned(buffer.buffer)).to.eq(true); 222 | }); 223 | }); 224 | 225 | describe("bufferAlignedAllocSharedUnsafe", () => { 226 | it("exposes the bufferAlignedAllocSharedUnsafe function", () => { 227 | expect(bufferAlignedAllocSharedUnsafe).to.be.a("function"); 228 | expect(RoaringBitmap32.bufferAlignedAllocSharedUnsafe).to.eq(bufferAlignedAllocSharedUnsafe); 229 | }); 230 | 231 | it("throws if first argument (size) is invalid", () => { 232 | expect(() => bufferAlignedAllocSharedUnsafe(-1)).to.throw(); 233 | expect(() => bufferAlignedAllocSharedUnsafe(null as any)).to.throw(); 234 | expect(() => bufferAlignedAllocSharedUnsafe("x" as any)).to.throw(); 235 | expect(() => bufferAlignedAllocSharedUnsafe({} as any)).to.throw(); 236 | }); 237 | 238 | it("can allocate an empty buffer", () => { 239 | const buffer = bufferAlignedAllocSharedUnsafe(0, 512); 240 | expect(buffer).to.be.instanceOf(Buffer); 241 | expect(buffer.buffer).to.be.instanceOf(SharedArrayBuffer); 242 | expect(buffer.length).to.eq(0); 243 | expect(isBufferAligned(buffer, 512)).to.eq(true); 244 | }); 245 | 246 | it("can allocate a buffer of a given size", () => { 247 | const buffer = bufferAlignedAllocSharedUnsafe(10); 248 | expect(buffer).to.be.instanceOf(Buffer); 249 | expect(buffer.buffer).to.be.instanceOf(SharedArrayBuffer); 250 | expect(buffer.length).to.eq(10); 251 | expect(isBufferAligned(buffer)).to.eq(true); 252 | expect(RoaringBitmap32.isBufferAligned(buffer, 32)).to.eq(true); 253 | expect(RoaringBitmap32.isBufferAligned(buffer.buffer)).to.eq(true); 254 | }); 255 | }); 256 | } 257 | -------------------------------------------------------------------------------- /test/roaring.test.ts: -------------------------------------------------------------------------------- 1 | import roaring from ".."; 2 | import { expect } from "chai"; 3 | 4 | import RoaringBitmap32 from "../RoaringBitmap32"; 5 | import RoaringBitmap32Iterator from "../RoaringBitmap32Iterator"; 6 | 7 | describe("roaring", () => { 8 | it("is an object", () => { 9 | expect(typeof roaring).eq("object"); 10 | }); 11 | 12 | it('exports itself with a "default" property', () => { 13 | const required = require("../"); 14 | expect(!!required).to.be.true; 15 | expect(required === roaring).to.be.true; 16 | expect(required === required.default).to.be.true; 17 | }); 18 | 19 | it("has RoaringBitmap32", () => { 20 | expect(typeof roaring.RoaringBitmap32).eq("function"); 21 | expect(roaring.RoaringBitmap32 === RoaringBitmap32).to.be.true; 22 | }); 23 | 24 | it("has RoaringBitmap32Iterator", () => { 25 | expect(typeof roaring.RoaringBitmap32Iterator).eq("function"); 26 | expect(roaring.RoaringBitmap32Iterator === RoaringBitmap32Iterator).to.be.true; 27 | }); 28 | 29 | it("has CRoaringVersion", () => { 30 | expect(typeof roaring.CRoaringVersion).eq("string"); 31 | const values = roaring.CRoaringVersion.split("."); 32 | expect(values).to.have.lengthOf(3); 33 | for (let i = 0; i < 3; ++i) { 34 | expect(Number.isInteger(Number.parseInt(values[i], 10))).eq(true); 35 | } 36 | }); 37 | 38 | it("has roaring PackageVersion", () => { 39 | expect(typeof roaring.PackageVersion).eq("string"); 40 | const values = roaring.CRoaringVersion.split("."); 41 | expect(values).to.have.lengthOf(3); 42 | for (let i = 0; i < 3; ++i) { 43 | expect(Number.isInteger(Number.parseInt(values[i], 10))).eq(true); 44 | } 45 | }); 46 | }); 47 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowJs": false, 4 | "checkJs": false, 5 | "noEmit": true, 6 | "stripInternal": true, 7 | "allowSyntheticDefaultImports": true, 8 | "esModuleInterop": true, 9 | "allowUnreachableCode": false, 10 | "allowUnusedLabels": false, 11 | "alwaysStrict": true, 12 | "declaration": true, 13 | "emitDecoratorMetadata": false, 14 | "experimentalDecorators": false, 15 | "forceConsistentCasingInFileNames": true, 16 | "importHelpers": false, 17 | "lib": ["esnext"], 18 | "module": "commonjs", 19 | "moduleResolution": "node", 20 | "newLine": "LF", 21 | "noFallthroughCasesInSwitch": true, 22 | "noImplicitAny": true, 23 | "noImplicitReturns": true, 24 | "noImplicitThis": true, 25 | "noStrictGenericChecks": false, 26 | "noUnusedLocals": false, 27 | "noUnusedParameters": false, 28 | "outDir": ".", 29 | "pretty": true, 30 | "removeComments": false, 31 | "strict": true, 32 | "skipLibCheck": false, 33 | "skipDefaultLibCheck": false, 34 | "strictFunctionTypes": true, 35 | "strictNullChecks": true, 36 | "strictPropertyInitialization": true, 37 | "target": "ES2022", 38 | "isolatedModules": true 39 | }, 40 | "exclude": ["node_modules", "node_modules/**/*", ".eslintcache/**/*", "temp/**/*", "build/**/*"], 41 | "typeAcquisition": { 42 | "enable": true 43 | }, 44 | "typedocOptions": { 45 | "out": "docs", 46 | "entryPoints": "src/index.ts", 47 | "exclude": ["test/**/*", "node_modules/**/*", "temp/**/*", "build/**/*", "docs/**/*"] 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "declaration": false, 5 | "noEmit": true 6 | }, 7 | "exclude": ["node_modules/**/*", ".eslintcache/**/*", "temp/**/*", "build/**/*"] 8 | } 9 | --------------------------------------------------------------------------------