├── doc ├── benchmarks │ ├── images │ │ ├── concat.png │ │ ├── list_iterate.png │ │ ├── list_lookup.png │ │ ├── list_construct.png │ │ ├── concat_time_all_rrb.png │ │ ├── list_construct_all_but_vavr.png │ │ ├── list_iterate_all_but_core_rrb_vector.png │ │ └── concat_time_all_rrb_but_core_rrb_vector.png │ ├── data │ │ ├── list_lookup.csv │ │ ├── list_iterate.csv │ │ ├── list_construct.csv │ │ └── concat.csv │ └── benchmarks.md ├── use-transducers │ └── README.md ├── crrbv-27 │ ├── use-shift-increment-2.patch │ ├── proposed-fix-needs-thought-and-testing.patch │ └── description.md ├── rrb-tree-notes.md └── hash-details.md ├── .github └── workflows │ ├── snapshot.yml │ ├── doc-build.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── src ├── parameterized │ └── clojure │ │ └── clojure │ │ └── core │ │ ├── rrb_vector │ │ ├── fork_join.clj │ │ ├── protocols.clj │ │ ├── parameters.clj │ │ ├── interop.clj │ │ ├── nodes.clj │ │ └── debug_platform_dependent.clj │ │ └── rrb_vector.clj ├── test │ ├── cljs │ │ └── clojure │ │ │ └── core │ │ │ └── rrb_vector │ │ │ ├── test_cljs_only.cljs │ │ │ ├── test_cljs.cljs │ │ │ ├── test_utils.cljs │ │ │ └── long_test.cljs │ ├── clojure │ │ └── clojure │ │ │ └── core │ │ │ └── rrb_vector │ │ │ ├── test_cljs.clj │ │ │ ├── test_utils.clj │ │ │ ├── long_test.clj │ │ │ └── test_clj_only.clj │ └── resources │ │ └── clojure │ │ └── core │ │ └── rrb_vector │ │ └── cljs_testsuite.clj ├── main │ ├── cljs │ │ └── clojure │ │ │ └── core │ │ │ ├── rrb_vector │ │ │ ├── protocols.cljs │ │ │ ├── interop.cljs │ │ │ ├── macros.clj │ │ │ ├── debug_platform_dependent.cljs │ │ │ ├── trees.cljs │ │ │ ├── transients.cljs │ │ │ └── nodes.cljs │ │ │ └── rrb_vector.cljs │ └── clojure │ │ └── clojure │ │ └── core │ │ ├── rrb_vector │ │ ├── fork_join.clj │ │ ├── protocols.clj │ │ ├── parameters.clj │ │ ├── interop.clj │ │ └── nodes.clj │ │ └── rrb_vector.clj └── test_local │ └── clojure │ └── clojure │ └── core │ └── rrb_vector_check.clj ├── CONTRIBUTING.md ├── script ├── mvn-run-tests ├── test ├── jdo ├── sdo └── replace-params ├── project.clj ├── pom.xml ├── CHANGES.md ├── deps.edn └── LICENSE /doc/benchmarks/images/concat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/concat.png -------------------------------------------------------------------------------- /doc/benchmarks/images/list_iterate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/list_iterate.png -------------------------------------------------------------------------------- /doc/benchmarks/images/list_lookup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/list_lookup.png -------------------------------------------------------------------------------- /doc/benchmarks/images/list_construct.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/list_construct.png -------------------------------------------------------------------------------- /doc/benchmarks/images/concat_time_all_rrb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/concat_time_all_rrb.png -------------------------------------------------------------------------------- /doc/benchmarks/images/list_construct_all_but_vavr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/list_construct_all_but_vavr.png -------------------------------------------------------------------------------- /doc/benchmarks/images/list_iterate_all_but_core_rrb_vector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/list_iterate_all_but_core_rrb_vector.png -------------------------------------------------------------------------------- /doc/benchmarks/images/concat_time_all_rrb_but_core_rrb_vector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/clojure/core.rrb-vector/master/doc/benchmarks/images/concat_time_all_rrb_but_core_rrb_vector.png -------------------------------------------------------------------------------- /.github/workflows/snapshot.yml: -------------------------------------------------------------------------------- 1 | name: Snapshot on demand 2 | 3 | on: [workflow_dispatch] 4 | 5 | jobs: 6 | call-snapshot: 7 | uses: clojure/build.ci/.github/workflows/snapshot.yml@master 8 | secrets: inherit 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /lib 3 | /classes 4 | /checkouts 5 | *.jar 6 | *.class 7 | .lein-deps-sum 8 | .lein-failures 9 | .lein-plugins 10 | .lein-repl-history 11 | /.repl 12 | /out 13 | /repl 14 | .\#* 15 | /.nrepl-port 16 | .idea 17 | *.iml 18 | /.cpcache 19 | -------------------------------------------------------------------------------- /.github/workflows/doc-build.yml: -------------------------------------------------------------------------------- 1 | name: Build API Docs 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | call-doc-build-workflow: 8 | uses: clojure/build.ci/.github/workflows/doc-build.yml@master 9 | with: 10 | project: clojure/core.rrb-vector 11 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/fork_join.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.fork-join 2 | (:require [clojure.core.reducers :as r])) 3 | 4 | (def pool @#'r/pool) 5 | (def task @#'r/fjtask) 6 | (def invoke @#'r/fjinvoke) 7 | (def fork @#'r/fjfork) 8 | (def join @#'r/fjjoin) 9 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.protocols) 2 | 3 | (defprotocol PSpliceableVector 4 | (splicev [v1 v2])) 5 | 6 | (defprotocol PSliceableVector 7 | (slicev [v start end])) 8 | 9 | (defprotocol PTransientDebugAccess 10 | (debugGetRoot [v]) 11 | (debugGetShift [v]) 12 | (debugGetTail [v]) 13 | (debugGetCnt [v])) 14 | -------------------------------------------------------------------------------- /src/test/cljs/clojure/core/rrb_vector/test_cljs_only.cljs: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.test-cljs-only 2 | (:require [clojure.test :as test :refer [deftest testing is are]] 3 | [clojure.core.rrb-vector.test-utils :as u] 4 | [clojure.core.rrb-vector :as fv] 5 | [clojure.core.rrb-vector.debug :as dv] 6 | [clojure.core.rrb-vector.debug-platform-dependent :as dpd])) 7 | 8 | (dv/set-debug-opts! dv/full-debug-opts) 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | This is a [Clojure contrib] project. 2 | 3 | Under the Clojure contrib [guidelines], this project cannot accept 4 | pull requests. All patches must be submitted via [JIRA]. 5 | 6 | See [Contributing] on the Clojure website for 7 | more information on how to contribute. 8 | 9 | [Clojure contrib]: https://clojure.org/community/contrib_libs 10 | [Contributing]: https://clojure.org/community/contributing 11 | [JIRA]: https://clojure.atlassian.net/browse/CRRBV 12 | [guidelines]: https://clojure.org/community/contrib_howto 13 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release on demand 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | releaseVersion: 7 | description: "Version to release" 8 | required: true 9 | snapshotVersion: 10 | description: "Snapshot version after release" 11 | required: true 12 | 13 | jobs: 14 | call-release: 15 | uses: clojure/build.ci/.github/workflows/release.yml@master 16 | with: 17 | releaseVersion: ${{ github.event.inputs.releaseVersion }} 18 | snapshotVersion: ${{ github.event.inputs.snapshotVersion }} 19 | secrets: inherit -------------------------------------------------------------------------------- /doc/use-transducers/README.md: -------------------------------------------------------------------------------- 1 | The patch use-transducers.md was developed around July or August 2019, 2 | but not yet included in the production core.rrb-vector code, because 3 | of a desire to continue to make core.rrb-vector compatible with 4 | Clojure 1.5.1 and later, whereas transducers were not implemented in 5 | Clojure until version 1.7.0. 6 | 7 | We should re-examine this patch when we are ready to require Clojure 8 | 1.7.0 or later as a minimum supported version for the core.rrb-vector 9 | library. I may include some performance measurements with and without 10 | these changes, to show how much they can improve the performance of 11 | some operations. 12 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/protocols.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.protocols) 10 | 11 | (defprotocol PSpliceableVector 12 | (-splicev [v1 v2])) 13 | 14 | (defprotocol PSliceableVector 15 | (-slicev [v start end])) 16 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [push] 4 | 5 | jobs: 6 | test: 7 | strategy: 8 | matrix: 9 | os: [ubuntu-latest] # macOS-latest, windows-latest] 10 | java-version: ["8", "11"] # NOTE: tests fail on Java 17 as they depend on Nashorn 11 | clojure-version: ["1.9.0", "1.10.3", "1.11.1"] 12 | runs-on: ${{ matrix.os }} 13 | steps: 14 | - uses: actions/checkout@v3 15 | - name: Set up Java 16 | uses: actions/setup-java@v3 17 | with: 18 | java-version: ${{ matrix.java-version }} 19 | distribution: 'temurin' 20 | cache: 'maven' 21 | - name: Build with Maven 22 | run: mvn -ntp -B -Dclojure.version=${{ matrix.clojure-version }} clean test 23 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector/fork_join.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.fork-join 10 | (:require [clojure.core.reducers :as r])) 11 | 12 | (def pool @#'r/pool) 13 | (def task @#'r/fjtask) 14 | (def invoke @#'r/fjinvoke) 15 | (def fork @#'r/fjfork) 16 | (def join @#'r/fjjoin) 17 | -------------------------------------------------------------------------------- /script/mvn-run-tests: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Example of a command run by the build.clojure.org Jenkins machine to 4 | # run Clojure/Java and ClojureScript tests for data.xml library: 5 | 6 | # /var/lib/jenkins/tools/hudson.tasks.Maven_MavenInstallation/Maven_3.2.5/bin/mvn "-Djdk=Oracle 11 EA" -DCLOJURE_VERSION=1.7.0 -Dclojure.version=1.7.0 clean test 7 | 8 | prog_name=`basename $0` 9 | 10 | usage() { 11 | 1>&2 echo "usage: $prog_name " 12 | 1>&2 echo "" 13 | 1>&2 echo "Examples:" 14 | 1>&2 echo "" 15 | 1>&2 echo " $prog_name 1.7.0" 16 | 1>&2 echo " $prog_name 1.10.1" 17 | } 18 | 19 | if [ $# -ne 1 ] 20 | then 21 | usage 22 | exit 1 23 | fi 24 | 25 | CLOJURE_VERSION="$1" 26 | 27 | set -x 28 | mvn -DCLOJURE_VERSION=${CLOJURE_VERSION} -Dclojure.version=${CLOJURE_VERSION} clean test 29 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector/protocols.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.protocols) 10 | 11 | (defprotocol PSpliceableVector 12 | (splicev [v1 v2])) 13 | 14 | (defprotocol PSliceableVector 15 | (slicev [v start end])) 16 | 17 | (defprotocol PTransientDebugAccess 18 | (debugGetRoot [v]) 19 | (debugGetShift [v]) 20 | (debugGetTail [v]) 21 | (debugGetCnt [v])) 22 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.parameters) 2 | 3 | ;; The values in comments before each def are the value of that 4 | ;; parameter: 5 | 6 | ;; * when the shift-increment is 5 7 | ;; * when the shift-increment is 3 8 | ;; * when the shift-increment is 2 9 | 10 | ;; 5 3 2 11 | (def shift-increment 5) 12 | 13 | ;; 10 6 4 14 | (def shift-increment-times-2 (* 2 shift-increment)) 15 | 16 | ;; 32 8 4 17 | (def max-branches (bit-shift-left 1 shift-increment)) 18 | 19 | ;; 0x1f 0x7 0x3 20 | (def branch-mask (dec max-branches)) 21 | 22 | ;; 31 7 3 23 | (def max-branches-minus-1 (dec max-branches)) 24 | 25 | ;; 30 6 2 26 | (def max-branches-minus-2 (- max-branches 2)) 27 | 28 | ;; 33 9 5 29 | (def non-regular-array-len (inc max-branches)) 30 | 31 | ;; 1024 64 16 32 | (def max-branches-squared (* max-branches max-branches)) 33 | -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # See README.md for some sample install instructions for Ubuntu 18.04 4 | # Linux and macOS. If you use those instructions, then the following environment variable settings should work: 5 | 6 | # export NODEJS_CMD="node" 7 | # export SPIDERMONKEY_CMD="js52" 8 | 9 | if [ "${NODEJS_CMD}" = "" -a "${SPIDERMONKEY_CMD}" = "" ]; then 10 | echo "Neither NODEJS_CMD nor SPIDERMONKEY_CMD is set, cannot run tests" 11 | exit 1 12 | fi 13 | 14 | rm -rf out 15 | mkdir -p out 16 | lein with-profile +cljs cljsbuild once test 17 | echo "Launching test runner..." 18 | 19 | if [ "${NODEJS_CMD}" != "" ]; then 20 | echo "Testing with Node.js:" 21 | "${NODEJS_CMD}" -e 'require("./out/test"); clojure.core.rrb_vector.test_cljs.run()' 22 | fi 23 | 24 | if [ "${SPIDERMONKEY_CMD}" != "" ]; then 25 | echo "Testing with SpiderMonkey:" 26 | "${SPIDERMONKEY_CMD}" -f out/test.js "--execute=clojure.core.rrb_vector.test_cljs.run()" 27 | fi 28 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/interop.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.interop 2 | (:require [clojure.core.rrb-vector.protocols 3 | :refer [PSliceableVector slicev 4 | PSpliceableVector splicev]] 5 | [clojure.core.rrb-vector.rrbt :refer [as-rrbt]]) 6 | (:import (clojure.core Vec) 7 | (clojure.lang PersistentVector APersistentVector$SubVector) 8 | (clojure.core.rrb_vector.rrbt Vector))) 9 | 10 | (extend-protocol PSliceableVector 11 | Vec 12 | (slicev [v start end] 13 | (slicev (as-rrbt v) start end)) 14 | 15 | PersistentVector 16 | (slicev [v start end] 17 | (slicev (as-rrbt v) start end)) 18 | 19 | APersistentVector$SubVector 20 | (slicev [v start end] 21 | (slicev (as-rrbt v) start end))) 22 | 23 | (extend-protocol PSpliceableVector 24 | Vec 25 | (splicev [v1 v2] 26 | (splicev (as-rrbt v1) v2)) 27 | 28 | PersistentVector 29 | (splicev [v1 v2] 30 | (splicev (as-rrbt v1) v2)) 31 | 32 | APersistentVector$SubVector 33 | (splicev [v1 v2] 34 | (splicev (as-rrbt v1) v2))) 35 | -------------------------------------------------------------------------------- /doc/crrbv-27/use-shift-increment-2.patch: -------------------------------------------------------------------------------- 1 | diff --git a/deps.edn b/deps.edn 2 | index 775cc3e..f03df7a 100644 3 | --- a/deps.edn 4 | +++ b/deps.edn 5 | @@ -5,8 +5,8 @@ 6 | ;; want to test with modifications to it: 7 | ;; org.clojure/clojurescript {:local/root "/Users/jafinger/clj/clojurescript"} 8 | 9 | -{:paths ["src/main/clojure" "src/main/cljs" "src/main/cljc"] 10 | - ;;:paths ["src/parameterized/clojure" "src/main/cljs" "src/main/cljc"] 11 | +{;;:paths ["src/main/clojure" "src/main/cljs" "src/main/cljc"] 12 | + :paths ["src/parameterized/clojure" "src/main/cljs" "src/main/cljc"] 13 | :aliases 14 | {;; Common alias to use for all Clojure/Java commands 15 | :clj {:jvm-opts ["-XX:-OmitStackTraceInFastThrow"]} 16 | diff --git a/src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj b/src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj 17 | index 2cd5004..c8a58ea 100644 18 | --- a/src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj 19 | +++ b/src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj 20 | @@ -8,7 +8,7 @@ 21 | ;; * when the shift-increment is 2 22 | 23 | ;; 5 3 2 24 | -(def shift-increment 5) 25 | +(def shift-increment 2) 26 | 27 | ;; 10 6 4 28 | (def shift-increment-times-2 (* 2 shift-increment)) 29 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/interop.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.interop 10 | (:require [clojure.core.rrb-vector.protocols 11 | :refer [PSliceableVector -slicev 12 | PSpliceableVector -splicev]] 13 | [clojure.core.rrb-vector.rrbt :refer [-as-rrbt]])) 14 | 15 | (extend-protocol PSliceableVector 16 | cljs.core/PersistentVector 17 | (-slicev [v start end] 18 | (-slicev (-as-rrbt v) start end)) 19 | 20 | cljs.core/Subvec 21 | (-slicev [v start end] 22 | (-slicev (-as-rrbt v) start end))) 23 | 24 | (extend-protocol PSpliceableVector 25 | cljs.core/PersistentVector 26 | (-splicev [v1 v2] 27 | (-splicev (-as-rrbt v1) v2)) 28 | 29 | cljs.core/Subvec 30 | (-splicev [v1 v2] 31 | (-splicev (-as-rrbt v1) v2))) 32 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/macros.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.macros 10 | (:refer-clojure :exclude [assert])) 11 | 12 | (def ^:const elide-assertions? true) 13 | (def ^:const elide-debug-printouts? true) 14 | 15 | (defmacro assert [& args] 16 | (if-not elide-assertions? 17 | `(clojure.core/assert ~@args))) 18 | 19 | (defmacro dbg [& args] 20 | (if-not elide-debug-printouts? 21 | `(prn ~@args))) 22 | 23 | (defmacro dbg- [& args]) 24 | 25 | (defmacro ^:private gen-vector-method [& params] 26 | (let [arr (gensym "arr__")] 27 | `(let [~arr (cljs.core/make-array ~(count params))] 28 | ~@(map-indexed (fn [i param] 29 | `(cljs.core/aset ~arr ~i ~param)) 30 | params) 31 | (clojure.core.rrb-vector.rrbt/Vector. 32 | ~(count params) 5 cljs.core/PersistentVector.EMPTY_NODE ~arr nil nil)))) 33 | -------------------------------------------------------------------------------- /script/jdo: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Run some task using Clojure/Java 4 | 5 | if [ $# -eq 0 ] 6 | then 7 | # Default if nothing else is specified is a REPL plus listening 8 | # for Socket REPL 9 | TASK="socket" 10 | elif [ $# -eq 1 ] 11 | then 12 | TASK="$1" 13 | else 14 | 1>&2 echo "usage: `basename $0` arg1" 15 | exit 1 16 | fi 17 | 18 | set -x 19 | 20 | case ${TASK} in 21 | sock*) 22 | # Run REPL, with option to listen for Socket REPL connection, and 23 | # test paths in classpath. 24 | exec clj -A:clj:clj-test:clj-socket ;; 25 | test*) 26 | # Run 'short' tests 27 | exec clojure -A:clj:clj-test:clj-runt ;; 28 | chec*) 29 | # Run 'short' tests with extra checks enabled 30 | exec clojure -A:clj:clj-test:clj-extrachecks-runt ;; 31 | long*) 32 | # Run long/generative tests 33 | exec clojure -A:clj:clj-test:clj-runlongtests ;; 34 | coll*) 35 | # Run collection-check generative tests 36 | exec clojure -A:clj:clj-test:clj-check:clj-runcheck ;; 37 | perf*) 38 | # Run performance tests 39 | exec clojure -A:clj:clj-test:clj-runperf ;; 40 | focu*) 41 | # Run whatever the current 'focus' tests are 42 | exec clojure -A:clj:clj-test:clj-check:clj-runfocus ;; 43 | east*|lint*) 44 | # Run Eastwood 45 | exec clojure -A:clj:clj-test:clj-check:eastwood ;; 46 | *) 47 | 1>&2 echo "unknown task name: ${TASK}" 48 | exit 1 ;; 49 | esac 50 | -------------------------------------------------------------------------------- /script/sdo: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Run some task using ClojureScript 4 | 5 | if [ $# -eq 0 ] 6 | then 7 | # Default if nothing else is specified is a REPL plus listening 8 | # for Socket REPL 9 | TASK="socket" 10 | elif [ $# -eq 1 ] 11 | then 12 | TASK="$1" 13 | else 14 | 1>&2 echo "usage: `basename $0` arg1" 15 | exit 1 16 | fi 17 | 18 | set -x 19 | 20 | case ${TASK} in 21 | sock*) 22 | # Run REPL, with option to listen for Socket REPL connection, and 23 | # test paths in classpath. 24 | exec clj -A:cljs:cljs-test:cljs-socket ;; 25 | test*) 26 | # Run 'short' tests 27 | exec clojure -A:cljs:cljs-test:cljs-runt ;; 28 | chec*) 29 | # Run 'short' tests with extra checks enabled 30 | exec clojure -A:cljs:cljs-test:cljs-extrachecks-runt ;; 31 | long*) 32 | # Run long/generative tests 33 | exec clojure -A:cljs:cljs-test:cljs-runlongtests ;; 34 | coll*) 35 | # Run collection-check generative tests 36 | exec clojure -A:cljs:cljs-test:cljs-check:cljs-runcheck ;; 37 | perf*) 38 | # Run performance tests 39 | exec clojure -A:cljs:cljs-test:cljs-runperf ;; 40 | focu*) 41 | # Run whatever the current 'focus' tests are 42 | exec clojure -A:cljs:cljs-test:cljs-check:cljs-runfocus ;; 43 | east*|lint*) 44 | 1>&2 echo "Eastwood not supported for ClojureScript" ;; 45 | *) 46 | 1>&2 echo "unknown task name: ${TASK}" 47 | exit 1 ;; 48 | esac 49 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector/parameters.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.parameters) 10 | 11 | ;; This namespace exists primarily so that the parameterized version 12 | ;; of this code, and the 'production' version of this code, can be 13 | ;; more similar to each other, by requiring this namespace from most 14 | ;; of the other namespaces. 15 | 16 | ;; Even though the values below are not used in most of the production 17 | ;; code, they can serve a little bit as documentation of these 18 | ;; parameter values. 19 | 20 | (def shift-increment 5) 21 | 22 | (def shift-increment-times-2 (* 2 shift-increment)) 23 | (def max-branches (bit-shift-left 1 shift-increment)) 24 | (def branch-mask (dec max-branches)) 25 | (def max-branches-minus-1 (dec max-branches)) 26 | (def max-branches-minus-2 (- max-branches 2)) 27 | (def non-regular-array-len (inc max-branches)) 28 | (def max-branches-squared (* max-branches max-branches)) 29 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector/interop.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.interop 10 | (:require [clojure.core.rrb-vector.protocols 11 | :refer [PSliceableVector slicev 12 | PSpliceableVector splicev]] 13 | [clojure.core.rrb-vector.rrbt :refer [as-rrbt]]) 14 | (:import (clojure.core Vec) 15 | (clojure.lang PersistentVector APersistentVector$SubVector) 16 | (clojure.core.rrb_vector.rrbt Vector))) 17 | 18 | (extend-protocol PSliceableVector 19 | Vec 20 | (slicev [v start end] 21 | (slicev (as-rrbt v) start end)) 22 | 23 | PersistentVector 24 | (slicev [v start end] 25 | (slicev (as-rrbt v) start end)) 26 | 27 | APersistentVector$SubVector 28 | (slicev [v start end] 29 | (slicev (as-rrbt v) start end))) 30 | 31 | (extend-protocol PSpliceableVector 32 | Vec 33 | (splicev [v1 v2] 34 | (splicev (as-rrbt v1) v2)) 35 | 36 | PersistentVector 37 | (splicev [v1 v2] 38 | (splicev (as-rrbt v1) v2)) 39 | 40 | APersistentVector$SubVector 41 | (splicev [v1 v2] 42 | (splicev (as-rrbt v1) v2))) 43 | -------------------------------------------------------------------------------- /src/test_local/clojure/clojure/core/rrb_vector_check.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector-check 2 | (:require [clojure.test :as test :refer [deftest testing is are]] 3 | [clojure.core.rrb-vector.test-utils :as u] 4 | [clojure.core.rrb-vector :as fv] 5 | [clojure.test.check.generators :as gen] 6 | [collection-check.core :refer [assert-vector-like]]) 7 | (:use clojure.test)) 8 | 9 | ;; On my 2015 MacBook Pro with JDK 11, a few num-tests values and 10 | ;; approximate run time of assert-vector-like test on Clojure 11 | ;; implementation: 12 | 13 | ;; 1,000: 16 sec 14 | ;; 10,000: 120 sec 15 | 16 | (def short-num-tests 1000) 17 | (def medium-num-tests 10000) 18 | (def long-num-tests 100000) 19 | ;;(def num-tests short-num-tests) 20 | (def num-tests medium-num-tests) 21 | ;;(def num-tests long-num-tests) 22 | 23 | ;; collection-check.core/assert-vector-like calls test.chuck/checking. 24 | ;; The README for the test.chuck library says that test.chuck/checking 25 | ;; is intended to be called directly within a `deftest` form, with no 26 | ;; need for any `is` or `are` calls, because test.chuck/checking makes 27 | ;; calls to those macros inside itself. 28 | ;; 29 | ;; I have confirmed, by intentionally making the function fv/vector 30 | ;; return incorrect values in some cases, that this deftest does fail 31 | ;; as it should, given the direct call to function assert-vector-like. 32 | 33 | (deftest collection-check 34 | (println "Before assert-vector-like with num-tests=" num-tests) 35 | (assert-vector-like num-tests (fv/vector) gen/int) 36 | (println "After assert-vector-like with num-tests=" num-tests) 37 | (is (every? nil? (.-array ^clojure.lang.PersistentVector$Node 38 | (.-root ^clojure.lang.PersistentVector (vector)))))) 39 | -------------------------------------------------------------------------------- /doc/benchmarks/data/list_lookup.csv: -------------------------------------------------------------------------------- 1 | size,bifurcan.List,java.ArrayList,clojure.PersistentVector,vavr.Vector,scala.Vector,paguro.RrbTree,bifurcan.LinearList,clojure.core.rrb-vector 2 | 10,1.7,1.4,4.8,1.8,1.9,1.9,1.6,6.2 3 | 17,1.5882353,1.2941177,4.7647057,1.5882353,1.8235294,1.6470588,1.5294118,6.2352943 4 | 31,1.483871,1.2580645,4.709677,1.483871,1.7419355,1.516129,1.451613,6.129032 5 | 56,3.0535715,1.3214285,6.375,3.125,2.357143,1.5178572,1.4464285,9.928572 6 | 100,3.63,1.27,7.68,3.05,2.56,4.14,1.35,10.83 7 | 177,3.4745762,1.2542373,7.3502827,2.9096045,2.6271186,4.220339,1.3050847,10.915255 8 | 316,3.506329,1.2468355,7.844937,2.943038,2.7246835,4.243671,1.272152,10.876582 9 | 562,3.608541,1.2419928,7.640569,2.8701067,2.8790035,4.0373664,1.2562277,10.770463 10 | 1000,3.614,1.236,7.5,3.024,2.936,3.876,1.257,10.76 11 | 1778,7.7165356,1.2485939,9.169854,6.038808,4.84027,5.8762655,1.3582677,13.452756 12 | 3162,7.7179003,1.245098,9.3390255,6.066414,5.9930425,6.0588236,1.4111322,13.652435 13 | 5623,7.695892,1.2532456,9.659434,6.1563225,6.353904,6.1963363,1.448515,13.426285 14 | 10000,8.4389,1.5085,11.4617,6.9566,6.9497,7.3155,1.64,14.306 15 | 17782,10.189462,2.159712,13.280733,8.604038,8.824204,9.230177,2.2748847,16.382465 16 | 31622,11.259218,2.19439,15.286858,9.492474,9.626589,10.392543,2.45481,19.911707 17 | 56234,16.073248,2.3088167,29.581766,8.739464,13.578423,20.866753,2.4903796,25.36108 18 | 100000,17.46262,2.32778,22.28634,10.32512,14.43229,22.792,2.53203,28.13811 19 | 177827,17.224106,1.5161252,24.876661,13.565139,10.459368,21.944075,1.5987449,32.799923 20 | 316227,24.521938,1.4419926,49.83222,15.486619,13.8668585,32.42438,5.912291,70.97381 21 | 562341,29.629885,6.4323745,40.65725,24.448853,20.996775,81.977974,1.5803969,87.66715 22 | 1000000,33.788033,1.873249,65.53855,35.926792,18.855904,45.184235,1.975906,63.871895 23 | -------------------------------------------------------------------------------- /doc/benchmarks/data/list_iterate.csv: -------------------------------------------------------------------------------- 1 | size,bifurcan.List,java.ArrayList,clojure.PersistentVector,vavr.Vector,scala.Vector,paguro.RrbTree,bifurcan.LinearList,clojure.core.rrb-vector 2 | 10,1.4,1.6,1.7,1.8,3.6,1.8,1.7,10.8 3 | 17,1.117647,1.2352941,1.2941177,1.4117647,3.1176472,1.2941177,1.3529412,10.176471 4 | 31,0.9032258,1.0,1.1290323,1.1612903,2.2903225,1.0,1.1290323,9.774194 5 | 56,1.5178572,0.875,1.7678572,1.8392857,2.0535715,5.25,1.0357143,11.125 6 | 100,2.18,0.79,1.81,1.92,2.13,3.67,0.97,11.61 7 | 177,2.220339,0.7288136,1.8474576,1.9491526,2.0056498,3.0960453,0.90960455,11.536723 8 | 316,2.205696,0.65189874,1.7974683,1.9398735,1.8765823,2.414557,0.81329113,11.462026 9 | 562,2.208185,0.6049822,1.7829181,1.8790035,1.8291816,2.4163702,0.75266904,11.725979 10 | 1000,2.204,0.582,1.779,1.909,1.794,2.241,0.729,11.649 11 | 1778,1.4386952,0.56299216,1.8335208,1.78009,1.831271,2.1017997,0.70528686,15.119798 12 | 3162,1.4111322,0.5553447,1.8358634,1.771031,1.7931689,2.0714738,0.6960784,15.343453 13 | 5623,1.4131247,0.5568202,1.8289169,1.772897,1.790859,2.0835853,0.6960697,15.150987 14 | 10000,1.4238,0.5572,1.8315,1.7793,1.7861,2.1406,0.693,15.3119 15 | 17782,1.4203689,0.55775505,1.820324,1.7672366,1.7702171,2.0864358,0.6955348,15.254021 16 | 31622,1.4137626,0.5536336,1.8266397,1.769496,1.7614319,2.1101131,0.6935045,15.445481 17 | 56234,1.6594943,0.55566025,1.8979799,1.8362023,2.0761282,2.1287477,0.700128,18.380625 18 | 100000,1.67543,0.56148,1.91083,1.83461,2.07209,2.14485,0.69761,18.35797 19 | 177827,1.6713041,0.55777246,1.9065102,1.8242562,2.0584614,2.178387,0.69495636,18.372795 20 | 316227,1.6643709,0.55767536,1.9050903,1.8317759,2.070364,2.2198262,0.6955415,18.44597 21 | 562341,1.6746601,0.55650574,1.9053137,1.847212,2.061694,2.2391484,0.7381873,18.396255 22 | 1000000,1.745738,0.562946,1.935245,1.891016,2.107386,2.320294,0.709329,18.548725 23 | -------------------------------------------------------------------------------- /doc/crrbv-27/proposed-fix-needs-thought-and-testing.patch: -------------------------------------------------------------------------------- 1 | diff --git a/src/main/clojure/clojure/core/rrb_vector/rrbt.clj b/src/main/clojure/clojure/core/rrb_vector/rrbt.clj 2 | index 1d231a6..45bfb6b 100644 3 | --- a/src/main/clojure/clojure/core/rrb_vector/rrbt.clj 4 | +++ b/src/main/clojure/clojure/core/rrb_vector/rrbt.clj 5 | @@ -1426,7 +1426,16 @@ 6 | (aset new-rngs 32 (inc i)) 7 | (recur (inc i) (next bs))))) 8 | (aset new-arr 32 new-rngs) 9 | - (set! (.-val transferred-leaves) cnt2) 10 | +;; (set! (.-val transferred-leaves) cnt2) 11 | + (when-not (zero? (.-val transferred-leaves)) 12 | + (println "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX") 13 | + (println "dbg CHANGED rebalance #3" 14 | + "shift=" shift 15 | + "cnt1=" cnt1 16 | + "cnt2=" cnt2 17 | + "cnt2 added to transferred-leaves=" (.-val transferred-leaves))) 18 | + (set! (.-val transferred-leaves) 19 | + (+ (.-val transferred-leaves) cnt2)) 20 | (pair new-n1 nil)) 21 | 22 | :else 23 | diff --git a/src/test/clojure/clojure/core/rrb_vector/long_test.clj b/src/test/clojure/clojure/core/rrb_vector/long_test.clj 24 | index b68cf5a..ea2540f 100644 25 | --- a/src/test/clojure/clojure/core/rrb_vector/long_test.clj 26 | +++ b/src/test/clojure/clojure/core/rrb_vector/long_test.clj 27 | @@ -84,7 +84,7 @@ 28 | (defn vector-push-f [v my-catvec extra-checks-catvec] 29 | (loop [v v 30 | i 0] 31 | - (let [check? (or (zero? (mod i 10000)) 32 | + (let [check? (or (zero? (mod i 100)) 33 | (and (> i 99000) (zero? (mod i 100))) 34 | (and (> i 99900) (zero? (mod i 10))))] 35 | (when check? 36 | -------------------------------------------------------------------------------- /doc/benchmarks/data/list_construct.csv: -------------------------------------------------------------------------------- 1 | size,bifurcan.List,java.ArrayList,clojure.PersistentVector,vavr.Vector,scala.Vector,paguro.RrbTree,bifurcan.LinearList,clojure.core.rrb-vector 2 | 10,9.4,6.0,24.9,122.8,9.8,11.8,8.6,23.6 3 | 17,9.058824,7.7647057,22.588236,125.82353,6.9411764,7.7647057,8.823529,19.411764 4 | 31,6.064516,6.645161,18.096775,135.64516,4.967742,7.0,6.096774,15.967742 5 | 56,8.035714,7.071429,16.071428,151.92857,5.803571,9.964286,6.214286,13.428572 6 | 100,7.91,6.62,15.3,170.31,5.65,9.22,6.03,11.9 7 | 177,7.9265537,8.853107,14.734464,160.52542,5.2768364,9.474576,5.8248587,11.672317 8 | 316,7.455696,7.990506,14.35443,167.99051,5.313291,10.537974,6.132911,11.341772 9 | 562,7.524911,9.339858,14.231317,173.57474,5.1921706,9.854093,6.6654806,11.188612 10 | 1000,7.284,7.784,14.172,180.283,5.072,11.107,4.923,11.064 11 | 1778,7.2581553,6.5410576,14.166479,182.65804,5.0646796,10.391451,5.104612,11.093364 12 | 3162,7.2046175,6.6514864,15.516445,191.10689,5.0246677,10.761543,4.9487667,10.951612 13 | 5623,7.386804,6.4266405,15.535657,196.15543,4.969945,11.1223545,5.796372,10.959986 14 | 10000,7.2806,8.4652,15.4981,179.5069,4.9662,11.2118,5.9852,10.9576 15 | 17782,7.3116074,6.484254,15.474975,209.4459,4.9549546,11.320211,6.3204927,10.910415 16 | 31622,7.2696857,8.90766,15.500917,219.30981,4.7470746,11.485738,4.6408515,10.997312 17 | 56234,7.403955,6.7953553,15.512821,208.06172,4.4433618,11.892645,4.966017,11.01467 18 | 100000,6.83919,6.66432,15.61274,190.86024,4.46145,12.22418,4.63192,11.00608 19 | 177827,7.2658763,7.4858656,15.6791935,197.01678,4.4837565,12.332981,9.392382,11.047546 20 | 316227,7.426023,12.031664,15.7804365,203.03307,4.428657,12.973231,14.0233,11.079547 21 | 562341,7.748018,17.517395,15.697704,195.32433,4.542214,13.011118,16.349981,10.937115 22 | 1000000,7.310803,14.379088,15.673957,203.99283,4.582003,12.813206,9.882679,11.052688 23 | -------------------------------------------------------------------------------- /doc/benchmarks/data/concat.csv: -------------------------------------------------------------------------------- 1 | size,bifurcan.List,java.ArrayList,clojure.PersistentVector,vavr.Vector,scala.Vector,paguro.RrbTree,bifurcan.LinearList,clojure.core.rrb-vector 2 | 10,13.6,11.4,62.5,15.0,16.6,26.5,16.1,37.8 3 | 17,8.176471,8.058824,44.235294,10.0,10.0,16.941177,9.588235,28.294117 4 | 31,4.483871,3.483871,31.548388,6.774194,5.419355,11.387096,6.032258,21.258064 5 | 56,2.5714285,2.8035715,27.428572,5.464286,5.017857,14.071428,3.375,14.964286 6 | 100,2.41,2.31,23.35,5.72,5.18,3.98,1.89,29.58 7 | 177,1.7740113,2.1694915,21.17514,4.8248587,3.3107345,1.3615819,1.0621469,21.62712 8 | 316,1.25,2.1677215,19.256329,3.8544304,2.335443,0.8575949,0.5949367,18.503164 9 | 562,0.97153026,1.8505338,18.604982,3.955516,1.9306049,0.60854095,0.33629894,16.382563 10 | 1000,0.639,1.767,17.906,3.542,1.546,0.727,0.187,15.333 11 | 1778,0.82452196,1.6310462,17.655231,3.6091113,1.4150732,0.23622048,0.10686164,5.8593926 12 | 3162,0.10974067,1.7125237,17.795067,4.4756484,1.5623024,0.2852625,0.059772298,1.4746996 13 | 5623,0.073092654,1.7412413,17.545439,4.21252,1.4636315,0.11613018,0.03361195,1.0497955 14 | 10000,0.0447,1.8509,17.5732,4.5135,1.2487,0.0655,0.0189,0.772 15 | 17782,0.0351479,1.9495558,17.450119,4.1053877,1.4498931,0.03649758,0.010516252,0.6182094 16 | 31622,0.019796344,2.2976408,17.373095,4.3760357,1.4443742,0.02580482,0.006008475,0.5335526 17 | 56234,0.026229683,2.1811893,17.619324,4.3532915,1.2567486,0.012074546,0.0026318596,0.37804532 18 | 100000,0.00297,2.24121,17.79133,4.42338,1.28174,0.00898,0.00132,0.299 19 | 177827,0.0018726066,2.31764,17.851732,4.433815,1.2783042,0.0051848143,7.2542415E-4,0.18800294 20 | 316227,0.0011510719,6.0872726,17.925129,4.502373,1.284359,0.002539315,4.110971E-4,0.11331417 21 | 562341,8.535746E-4,7.5673695,18.17226,4.511332,1.2696211,0.0018725293,2.3117645E-4,0.06735593 22 | 1000000,4.81E-4,5.719794,18.577366,4.775235,0.936365,5.07E-4,1.35E-4,0.042625 23 | -------------------------------------------------------------------------------- /src/test/cljs/clojure/core/rrb_vector/test_cljs.cljs: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.test-cljs 2 | (:require [cljs.test :as test] 3 | [clojure.core.rrb-vector :as fv] 4 | [clojure.core.rrb-vector.test-utils :as u] 5 | clojure.core.rrb-vector.test-common 6 | clojure.core.rrb-vector.test-cljs-only 7 | clojure.core.rrb-vector.long-test)) 8 | 9 | ;; This file was copied from namespace clojure.data.xml.test-cljs 10 | ;; in the data.xml library tests, then modified for use by 11 | ;; core.rrb-vector, so that core.rrb-vector's ClojureScript tests 12 | ;; could also be run on build.clojure.org via a mvn command. 13 | 14 | (def ^:dynamic *results*) 15 | 16 | (defmethod test/report [::test/default :end-run-tests] 17 | [m] 18 | (assert (nil? *results*)) 19 | (set! *results* m)) 20 | 21 | ;; Function -main-nashorn is called when running tests using a maven 22 | ;; command such as this one: 23 | ;; mvn -DCLOJURE_VERSION=1.10.1 -Dclojure.version=1.10.1 clean test 24 | 25 | ;; Function run is called when running tests using a 'lein cljsbuild' 26 | ;; command such as this one: 27 | ;; lein with-profile +cljs,+1.10 cljsbuild test 28 | 29 | ;; To run the tests in a namespace from here, it must be :require'd 30 | ;; above, and its name must match the regex given to a run-all-tests 31 | ;; call. You may also call run-all-tests with no args to run tests in 32 | ;; all namespaces, but even then it must be :require'd above. 33 | 34 | (defn ^:export -main-nashorn [] 35 | (set! *print-newline* false) 36 | (set! *print-fn* js/print) 37 | (set! *print-err-fn* js/print) 38 | (binding [*results* nil] 39 | (println "Running Basic Tests") 40 | (test/run-all-tests #"clojure\.core\.rrb-vector\.test-.*") 41 | ;;(test/run-all-tests #"clojure\.core\.rrb-vector\..*test.*") 42 | (pr-str *results*))) 43 | 44 | (defn ^:export run [] 45 | (println "Running Basic Tests") 46 | (test/run-all-tests #"clojure\.core\.rrb-vector\.test-.*") 47 | ;;(test/run-all-tests #"clojure\.core\.rrb-vector\..*test.*") 48 | ) 49 | -------------------------------------------------------------------------------- /src/test/clojure/clojure/core/rrb_vector/test_cljs.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns ^{:doc "Clojurescript tests for core.rrb-vector"} 10 | clojure.core.rrb-vector.test-cljs 11 | (:require 12 | [clojure.test :refer :all])) 13 | 14 | ;; This file was copied from namespace clojure.data.xml.test-cljs 15 | ;; in the data.xml library tests, then modified for use by 16 | ;; core.rrb-vector, so that core.rrb-vector's ClojureScript tests 17 | ;; could also be run on build.clojure.org via a mvn command. 18 | 19 | (deftest ^:cljs-nashorn clojurescript-test-suite 20 | (try 21 | (require 'clojure.core.rrb-vector.cljs-testsuite) 22 | (eval '(clojure.core.rrb-vector.cljs-testsuite/run-testsuite! "target/cljs-test-nashorn")) 23 | (catch Exception e 24 | (if (or (neg? (compare ((juxt :major :minor) *clojure-version*) 25 | [1 8])) 26 | (neg? (compare (System/getProperty "java.runtime.version") 27 | "1.8"))) 28 | (println "WARN: ignoring cljs testsuite error on clojure < 1.8 or jdk < 1.8" 29 | *clojure-version* (System/getProperty "java.runtime.name") 30 | (System/getProperty "java.vm.version") (System/getProperty "java.runtime.version") 31 | \newline (str e)) 32 | (do (println "ERROR: cljs nashorn test suite should be able to run on clojure >= 1.8 and jdk >= 1.8" 33 | *clojure-version* (System/getProperty "java.runtime.name") 34 | (System/getProperty "java.vm.version") (System/getProperty "java.runtime.version")) 35 | (throw e)))))) 36 | -------------------------------------------------------------------------------- /src/test/resources/clojure/core/rrb_vector/cljs_testsuite.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.cljs-testsuite 2 | (:require 3 | [clojure.test :refer [is]] 4 | [cljs.repl.nashorn :as repl-nh] 5 | [cljs.build.api :as bapi] 6 | [clojure.java.io :as io] 7 | [clojure.core.rrb-vector.test-utils :as u]) 8 | (:import 9 | java.nio.file.Files 10 | java.nio.file.attribute.FileAttribute)) 11 | 12 | ;; This file was copied from namespace clojure.data.xml.cljs-testsuite 13 | ;; in the data.xml library tests, then modified for use by 14 | ;; core.rrb-vector, so that core.rrb-vector's ClojureScript tests 15 | ;; could also be run on build.clojure.org via a mvn command. 16 | 17 | (defn tempdir [] 18 | (str (Files/createTempDirectory 19 | "cljs-nashorn-" (into-array FileAttribute [])))) 20 | 21 | (defn compile-testsuite! [dir] 22 | (let [out (io/file dir "tests.js") 23 | inputs ["src/main/clojure" "src/test/clojure" "src/test/cljs"]] 24 | (println "INFO" "Compiling cljs testsuite from" inputs "into" (str out)) 25 | (bapi/build (apply bapi/inputs inputs) 26 | {:output-to (str out) 27 | :output-dir dir 28 | :main 'clojure.core.rrb-vector.test-cljs 29 | :optimizations :advanced 30 | :pseudo-names true 31 | :pretty-print true}))) 32 | 33 | (defn run-testsuite! [dir] 34 | (System/setProperty "nashorn.persistent.code.cache" "target/nashorn_code_cache") 35 | (let [t1 (u/now-msec) 36 | engine (repl-nh/create-engine)] 37 | (compile-testsuite! dir) 38 | (println "INFO" "Elapsed time (sec)" (/ (- (u/now-msec) t1) 1000.0) 39 | "to compile ClojureScript code") 40 | (println "INFO" "Running cljs tests in nashorn with persistent code cache in" (System/getProperty "nashorn.persistent.code.cache")) 41 | (.eval engine (io/reader (io/file dir "tests.js"))) 42 | (let [{:as res :keys [fail error]} (read-string (.eval engine "clojure.core.rrb_vector.test_cljs._main_nashorn()"))] 43 | (is (and (zero? fail) (zero? error)) 44 | (pr-str res))))) 45 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject org.clojure/core.rrb-vector "0.1.1-SNAPSHOT" 2 | :description "RRB-Trees for Clojure(Script) -- see Bagwell & Rompf" 3 | :url "https://github.com/clojure/core.rrb-vector" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :min-lein-version "2.2.0" 7 | :parent [org.clojure/pom.contrib "1.2.0"] 8 | :dependencies [[org.clojure/clojure "1.9.0"]] 9 | :source-paths ["src/main/clojure" "src/main/cljs"] 10 | ;;:source-paths ["src/parameterized/clojure" "src/main/cljs"] 11 | :test-paths ["src/test/clojure"] 12 | :test-selectors {:default (complement :cljs-nashorn)} 13 | :jvm-opts ^:replace ["-XX:+UseG1GC" 14 | "-XX:-OmitStackTraceInFastThrow"] 15 | :profiles {:dev {:dependencies [[org.clojure/test.check "1.1.1"]] 16 | :plugins [[lein-cljsbuild "1.1.7"]]} 17 | :coll {:test-paths ["src/test_local/clojure"] 18 | :dependencies [[org.clojure/test.check "1.1.1"] 19 | [collection-check "0.1.7"]]} 20 | :cljs {:dependencies [[org.clojure/clojure "1.10.1"] 21 | [org.clojure/clojurescript "1.10.238"]]} 22 | :1.9 {:dependencies [[org.clojure/clojure "1.9.0"]]} 23 | :1.10 {:dependencies [[org.clojure/clojure "1.10.1"]]} 24 | :master {:dependencies [[org.clojure/clojure "1.11.0-master-SNAPSHOT"]]}} 25 | :cljsbuild {:builds {:test {:source-paths ["src/main/cljs" 26 | "src/test/cljs"] 27 | :compiler {;;:optimizations :none 28 | ;;:optimizations :whitespace 29 | ;;:optimizations :simple 30 | :optimizations :advanced 31 | :output-to "out/test.js"}}} 32 | :test-commands 33 | {"node" ["node" "-e" 34 | "require(\"./out/test\"); clojure.core.rrb_vector.test_cljs.run()"] 35 | "spidermonkey" ["js52" "-f" "out/test.js" 36 | "--execute=clojure.core.rrb_vector.test_cljs.run()"]}}) 37 | -------------------------------------------------------------------------------- /src/test/cljs/clojure/core/rrb_vector/test_utils.cljs: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.test-utils 2 | (:require [clojure.test :as test] 3 | [clojure.core.rrb-vector.rrbt :as rrbt])) 4 | 5 | ;; Parts of this file are nearly identical to 6 | ;; src/test/clojure/clojure/core/rrb_vector/test_utils.clj, but also 7 | ;; significant parts are specific to each of the clj/cljs versions, so 8 | ;; while they could later be combined into a .cljc file, it may not 9 | ;; give much benefit to do so. 10 | 11 | (def extra-checks? false) 12 | 13 | (defn reset-optimizer-counts! [] 14 | (println "reset all optimizer counts to 0") 15 | (reset! rrbt/peephole-optimization-count 0) 16 | (reset! rrbt/fallback-to-slow-splice-count1 0) 17 | (reset! rrbt/fallback-to-slow-splice-count2 0)) 18 | 19 | (defn print-optimizer-counts [] 20 | (println "optimizer counts: peephole=" @rrbt/peephole-optimization-count 21 | "fallback1=" @rrbt/fallback-to-slow-splice-count1 22 | "fallback2=" @rrbt/fallback-to-slow-splice-count2)) 23 | 24 | (defn now-msec [] 25 | (js/Date.now)) 26 | 27 | (def num-deftests-started (atom 0)) 28 | (def last-deftest-start-time (atom nil)) 29 | 30 | (defn print-test-env-info [] 31 | (println "extra-checks?=" extra-checks?) 32 | (println "*clojurescript-version*" *clojurescript-version*)) 33 | 34 | (defmethod test/report [:cljs.test/default :begin-test-var] 35 | [m] 36 | (let [n (swap! num-deftests-started inc)] 37 | (when (== n 1) 38 | (print-test-env-info))) 39 | (println) 40 | (println "starting cljs test" (:var m)) 41 | (reset! last-deftest-start-time (now-msec))) 42 | 43 | (defmethod test/report [:cljs.test/default :end-test-var] 44 | [m] 45 | (println "elapsed time (sec)" (/ (- (now-msec) @last-deftest-start-time) 46 | 1000.0))) 47 | 48 | ;; Enable tests to be run on versions of Clojure before 1.10, when 49 | ;; ex-message was added. 50 | 51 | (defn ex-message-copy 52 | "Returns the message attached to the given Error / ExceptionInfo object. 53 | For non-Errors returns nil." 54 | [ex] 55 | (when (instance? js/Error ex) 56 | (.-message ex))) 57 | 58 | (defn ex-cause-copy 59 | "Returns exception cause (an Error / ExceptionInfo) if ex is an 60 | ExceptionInfo. 61 | Otherwise returns nil." 62 | [ex] 63 | (when (instance? ExceptionInfo ex) 64 | (.-cause ex))) 65 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | core.rrb-vector 5 | 0.2.1-SNAPSHOT 6 | core.rrb-vector 7 | RRB-Trees for Clojure(Script) -- see Bagwell & Rompf 8 | 9 | 10 | 11 | Eclipse Public License 1.0 12 | https://opensource.org/license/epl-1-0/ 13 | repo 14 | 15 | 16 | 17 | 18 | org.clojure 19 | pom.contrib 20 | 1.3.0 21 | 22 | 23 | 24 | 25 | Michał Marczyk 26 | https://github.com/michalmarczyk 27 | 28 | 29 | 30 | 31 | scm:git:git://github.com/clojure/core.rrb-vector.git 32 | scm:git:git://github.com/clojure/core.rrb-vector.git 33 | https://github.com/clojure/core.rrb-vector 34 | HEAD 35 | 36 | 37 | 38 | 1.9.0 39 | true 40 | 41 | 42 | 43 | 44 | 45 | org.codehaus.mojo 46 | build-helper-maven-plugin 47 | 48 | 49 | add-clojurescript-source-dirs 50 | generate-sources 51 | 52 | add-resource 53 | 54 | 55 | 56 | 57 | src/main/cljs 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | ${project.basedir}/src/test/resources 68 | 69 | 70 | ${project.basedir}/src/test/cljs 71 | 72 | 73 | 74 | 75 | 76 | 77 | org.clojure 78 | test.check 79 | 1.1.1 80 | test 81 | 82 | 83 | org.clojure 84 | clojurescript 85 | 1.10.439 86 | test 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/test/clojure/clojure/core/rrb_vector/test_utils.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.test-utils 2 | (:require [clojure.test :as test] 3 | [clojure.string :as str] 4 | [clojure.core.rrb-vector.rrbt :as rrbt])) 5 | 6 | ;; Parts of this file are nearly identical to 7 | ;; src/test/cljs/clojure/core/rrb_vector/test_utils.cljs, but also 8 | ;; significant parts are specific to each of the clj/cljs versions, so 9 | ;; while they could later be combined into a .cljc file, it may not 10 | ;; give much benefit to do so. 11 | 12 | (def extra-checks? false) 13 | 14 | (defn reset-optimizer-counts! [] 15 | (println "reset all optimizer counts to 0") 16 | (reset! rrbt/peephole-optimization-count 0) 17 | (reset! rrbt/fallback-to-slow-splice-count1 0) 18 | (reset! rrbt/fallback-to-slow-splice-count2 0)) 19 | 20 | (defn print-optimizer-counts [] 21 | (println "optimizer counts: peephole=" @rrbt/peephole-optimization-count 22 | "fallback1=" @rrbt/fallback-to-slow-splice-count1 23 | "fallback2=" @rrbt/fallback-to-slow-splice-count2)) 24 | 25 | (defn now-msec [] 26 | (System/currentTimeMillis)) 27 | 28 | (def num-deftests-started (atom 0)) 29 | (def last-deftest-start-time (atom nil)) 30 | 31 | (defn print-jvm-classpath [] 32 | (let [cp-str (System/getProperty "java.class.path") 33 | cp-strs (str/split cp-str #":")] 34 | (println "java.class.path:") 35 | (doseq [cp-str cp-strs] 36 | (println " " cp-str)))) 37 | 38 | (defn print-test-env-info [] 39 | (try 40 | (let [shift-var (resolve 41 | 'clojure.core.rrb-vector.parameters/shift-increment)] 42 | (println "shift-increment=" @shift-var " (from parameters namespace)")) 43 | (catch Exception e 44 | (println "shift-increment=5 (assumed because no parameters namespace)"))) 45 | (println "extra-checks?=" extra-checks?) 46 | (let [p (System/getProperties)] 47 | (println "java.vm.name" (get p "java.vm.name")) 48 | (println "java.vm.version" (get p "java.vm.version")) 49 | (print-jvm-classpath) 50 | (println "(clojure-version)" (clojure-version)))) 51 | 52 | (defmethod test/report :begin-test-var 53 | [m] 54 | (let [n (swap! num-deftests-started inc)] 55 | (when (== n 1) 56 | (print-test-env-info))) 57 | (println) 58 | (println "starting clj test" (:var m)) 59 | (reset! last-deftest-start-time (now-msec))) 60 | 61 | (defmethod test/report :end-test-var 62 | [m] 63 | (println "elapsed time (sec)" (/ (- (now-msec) @last-deftest-start-time) 64 | 1000.0))) 65 | 66 | ;; Enable tests to be run on versions of Clojure before 1.10, when 67 | ;; ex-message was added. 68 | 69 | (defn ex-message-copy 70 | "Returns the message attached to ex if ex is a Throwable. 71 | Otherwise returns nil." 72 | {:added "1.10"} 73 | [ex] 74 | (when (instance? Throwable ex) 75 | (.getMessage ^Throwable ex))) 76 | 77 | (defn ex-cause-copy 78 | "Returns the cause of ex if ex is a Throwable. 79 | Otherwise returns nil." 80 | {:added "1.10"} 81 | [ex] 82 | (when (instance? Throwable ex) 83 | (.getCause ^Throwable ex))) 84 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector 10 | 11 | "An implementation of the confluently persistent vector data 12 | structure introduced in Bagwell, Rompf, \"RRB-Trees: Efficient 13 | Immutable Vectors\", EPFL-REPORT-169879, September, 2011. 14 | 15 | RRB-Trees build upon Clojure's PersistentVectors, adding logarithmic 16 | time concatenation and slicing. 17 | 18 | The main API entry points are clojure.core.rrb-vector/catvec, 19 | performing vector concatenation, and clojure.core.rrb-vector/subvec, 20 | which produces a new vector containing the appropriate subrange of 21 | the input vector (in contrast to cljs.core/subvec, which returns a 22 | view on the input vector). 23 | 24 | The implementation allows for seamless interoperability with 25 | cljs.core/PersistentVector and cljs.core.Subvec instances: 26 | clojure.core.rrb-vector/catvec and clojure.core.rrb-vector/subvec 27 | convert their inputs to clojure.core.rrb-vector.rrbt/Vector 28 | instances whenever necessary (this is a very fast constant time 29 | operation for PersistentVector; for Subvec it is O(log n), where n 30 | is the size of the underlying vector). 31 | 32 | clojure.core.rrb-vector also exports its own versions of vector and 33 | vec which always produce clojure.core.rrb-vector.rrbt.Vector 34 | instances." 35 | 36 | {:author "Michał Marczyk"} 37 | 38 | (:refer-clojure :exclude [vector vec subvec]) 39 | (:require [clojure.core.rrb-vector.protocols :refer [-slicev -splicev]] 40 | [clojure.core.rrb-vector.rrbt :refer [-as-rrbt]] 41 | clojure.core.rrb-vector.interop) 42 | (:require-macros [clojure.core.rrb-vector.macros :refer [gen-vector-method]])) 43 | 44 | (defn catvec 45 | "Concatenates the given vectors in logarithmic time." 46 | ([] 47 | []) 48 | ([v1] 49 | v1) 50 | ([v1 v2] 51 | (-splicev v1 v2)) 52 | ([v1 v2 v3] 53 | (-splicev (-splicev v1 v2) v3)) 54 | ([v1 v2 v3 v4] 55 | (-splicev (-splicev v1 v2) (-splicev v3 v4))) 56 | ([v1 v2 v3 v4 & vn] 57 | (-splicev (-splicev (-splicev v1 v2) (-splicev v3 v4)) 58 | (apply catvec vn)))) 59 | 60 | (defn subvec 61 | "Returns a new vector containing the elements of the given vector v 62 | lying between the start (inclusive) and end (exclusive) indices in 63 | logarithmic time. end defaults to end of vector. The resulting 64 | vector shares structure with the original, but does not hold on to 65 | any elements of the original vector lying outside the given index 66 | range." 67 | ([v start] 68 | (-slicev v start (count v))) 69 | ([v start end] 70 | (-slicev v start end))) 71 | 72 | (defn vector 73 | "Creates a new vector containing the args." 74 | ([] 75 | (gen-vector-method)) 76 | ([x1] 77 | (gen-vector-method x1)) 78 | ([x1 x2] 79 | (gen-vector-method x1 x2)) 80 | ([x1 x2 x3] 81 | (gen-vector-method x1 x2 x3)) 82 | ([x1 x2 x3 x4] 83 | (gen-vector-method x1 x2 x3 x4)) 84 | ([x1 x2 x3 x4 & xn] 85 | (into (vector x1 x2 x3 x4) xn) 86 | #_ 87 | (loop [v (vector x1 x2 x3 x4) 88 | xn xn] 89 | (if xn 90 | (recur (-conj ^not-native v (first xn)) 91 | (next xn)) 92 | v)))) 93 | 94 | (defn vec 95 | "Returns a vector containing the contents of coll. 96 | 97 | If coll is a vector, returns an RRB vector using the internal tree 98 | of coll." 99 | [coll] 100 | (if (vector? coll) 101 | (-as-rrbt coll) 102 | (apply vector coll))) 103 | -------------------------------------------------------------------------------- /script/replace-params: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #_( 3 | #_DEPS is same format as deps.edn. Multiline is okay. 4 | DEPS='{:deps { 5 | org.clojure/clojure {:mvn/version "1.10.1"} 6 | }}' 7 | 8 | #_You can put other options here 9 | OPTS='-J-XX:-OmitStackTraceInFastThrow' 10 | 11 | exec clojure $OPTS -Sdeps "$DEPS" "$0" "$@" 12 | ) 13 | 14 | ;; For every file in this directory and its subdirectories: 15 | ;; src/parameterized/clojure/clojure/core/rrb_vector/ 16 | ;; create a corresponding file: 17 | ;; src/hardcoded/clojure/clojure/core/rrb_vector/ 18 | 19 | ;; that has the same contents, except for the string substitutions 20 | ;; specified in the var named 'substitutions' throughout the file, 21 | ;; wherever they occur. 22 | 23 | (require '[clojure.java.io :as io] 24 | '[clojure.java.shell :as sh] 25 | '[clojure.edn :as edn] 26 | '[clojure.string :as str] 27 | '[clojure.pprint :as pp]) 28 | 29 | (def substitutions [ 30 | ;; some comments have special cases like this, 31 | ;; which I want to replace before the other more 32 | ;; normal cases below. 33 | ["1/(p/max-branches)" "1/32"] 34 | ;;["by max-branches-squared" "by 1024"] 35 | ["1/max-branches-squared" "1/1024"] 36 | ["1/(p/max-branches-squared)" "1/1024"] 37 | ;; special case expression that appears in a few 38 | ;; places in the code. 39 | ["(inc p/max-branches)" "33"] 40 | 41 | ;; Note that these must be replaced before 42 | ;; p/max-branches is, because the substitution 43 | ;; code below does not know anything about 44 | ;; symbols, just raw sequences of characters. All 45 | ;; of these are suffixes of p/max-branches. 46 | ["p/max-branches-squared" "1024"] 47 | ["p/max-branches-minus-1" "31"] 48 | ["p/max-branches-minus-2" "30"] 49 | 50 | ;; Similarly, shift-increment-times-2 must be 51 | ;; replaced before shift-increment. 52 | ["p/shift-increment-times-2" "10"] 53 | 54 | ["p/shift-increment" "5"] 55 | ["p/max-branches" "32"] 56 | ["p/branch-mask" "0x1f"] 57 | ["p/non-regular-array-len" "33"] 58 | ["max-capacity-divided-by-max-branches-squared" 59 | "max-capacity-divided-by-1024"] 60 | ]) 61 | 62 | (def source-dir-prefix "src/parameterized/") 63 | (def target-dir-prefix "src/hardcoded/") 64 | (def common-intermediate-path "clojure/clojure/core") 65 | 66 | (def source-dir (str source-dir-prefix common-intermediate-path)) 67 | (def target-dir (str target-dir-prefix common-intermediate-path)) 68 | 69 | (defn source-to-target-name [source-fname] 70 | (let [expected-path? (str/starts-with? source-fname source-dir)] 71 | (if expected-path? 72 | (str target-dir (subs source-fname (count source-dir))) 73 | (throw (ex-info (format "Unexpected source path '%s' does not begin with '%s'" 74 | source-fname source-dir) 75 | {:source-fname source-fname 76 | :source-dir source-dir}))))) 77 | 78 | (defn make-all-substitutions [content substitution-pairs] 79 | (reduce (fn [content [to-replace-str replace-with-str]] 80 | (str/replace content to-replace-str replace-with-str)) 81 | content 82 | substitution-pairs)) 83 | 84 | (let [source-dir (str source-dir-prefix common-intermediate-path)] 85 | (doseq [source-f (file-seq (io/file source-dir))] 86 | (let [source-fname (str source-f) 87 | target-fname (source-to-target-name source-fname) 88 | target-f (io/file target-fname)] 89 | (println) 90 | (println "source file :" source-fname) 91 | (println "target file1:" target-fname) 92 | (println "target file2:" (str target-f)) 93 | (if (. source-f (isDirectory)) 94 | (println "skipping directory") 95 | (do 96 | (io/make-parents target-f) 97 | (let [contents (slurp source-f) 98 | new-contents (make-all-substitutions contents substitutions)] 99 | (spit target-f new-contents))))))) 100 | -------------------------------------------------------------------------------- /src/test/cljs/clojure/core/rrb_vector/long_test.cljs: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.long-test 2 | (:require [clojure.test :as test :refer [deftest testing is are]] 3 | [clojure.core.rrb-vector.test-utils :as u] 4 | [clojure.core.rrb-vector :as fv] 5 | [clojure.core.rrb-vector.debug :as dv] 6 | [clojure.core.rrb-vector.debug-platform-dependent :as dpd])) 7 | 8 | ;; The intent is to keep this file as close to 9 | ;; src/test/clojure/clojure/core/rrb_vector/long_test.clj as possible, 10 | ;; so that when we start requiring Clojure 1.7.0 and later for this 11 | ;; library, this file and that one can be replaced with a common test 12 | ;; file with the suffix .cljc 13 | 14 | ;; Note that the namespace of this file _intentionally_ does not match 15 | ;; the pattern of namespaces that are run for ClojureScript tests by 16 | ;; default. That is because of how long the tests in this file take 17 | ;; to run. It seems best to include them in the set of tests in such 18 | ;; a way that it is only run when a developer explicitly wants to run 19 | ;; longer tests. It should not be run by default when running on 20 | ;; build.clojure.org. 21 | 22 | ;; Currently the Clojure/JVM versions of these tests _are_ run by 23 | ;; default, and on build.clojure.org, but at least the ones in here 24 | ;; now run significantly faster on Clojure/JVM than they do in any of 25 | ;; the JavaScript runtimes I have tested with. 26 | 27 | (dv/set-debug-opts! dv/full-debug-opts) 28 | 29 | (def generative-test-length :short) 30 | 31 | (def check-subvec-params (case generative-test-length 32 | :short [125 100000 10] 33 | :medium [250 200000 20] 34 | :long [250 200000 20])) 35 | 36 | (deftest test-slicing-generative 37 | (testing "slicing (generative)" 38 | (is (try 39 | (apply dv/generative-check-subvec u/extra-checks? check-subvec-params) 40 | (catch js/Error e 41 | (throw (ex-info (dpd/format "%s: %s %s" 42 | (u/ex-message-copy e) 43 | (:init-cnt (ex-data e)) 44 | (:s&es (ex-data e))) 45 | {} 46 | (u/ex-cause-copy e)))))))) 47 | 48 | ;; short: 2 to 3 sec 49 | ;; medium: 50 to 60 sec 50 | (def check-catvec-params (case generative-test-length 51 | :short [ 10 30 10 60000] 52 | :medium [250 30 10 60000] 53 | :long [250 30 10 60000])) 54 | 55 | (deftest test-splicing-generative 56 | (testing "splicing (generative)" 57 | (is (try 58 | (apply dv/generative-check-catvec u/extra-checks? check-catvec-params) 59 | (catch js/Error e 60 | (throw (ex-info (dpd/format "%s: %s" 61 | (u/ex-message-copy e) 62 | (:cnts (ex-data e))) 63 | {} 64 | (u/ex-cause-copy e)))))))) 65 | 66 | 67 | ;; This problem reproduction code is from CRRBV-17 ticket: 68 | ;; https://clojure.atlassian.net/projects/CRRBV/issues/CRRBV-17 69 | 70 | (def benchmark-size 100000) 71 | 72 | ;; This small variation of the program in the ticket simply does 73 | ;; progress debug printing occasionally, as well as extra debug 74 | ;; checking of the results occasionally. 75 | 76 | ;; If you enable the printing of the message that begins 77 | ;; with "splice-rrbts result had shift" in function 78 | ;; fallback-to-slow-splice-if-needed, then run this test, you will see 79 | ;; it called hundreds or perhaps thousands of times. The fallback 80 | ;; approach is effective at avoiding a crash for this scenario, but at 81 | ;; a dramatic extra run-time cost. 82 | 83 | (defn vector-push-f [v my-catvec extra-checks-catvec] 84 | (loop [v v 85 | i 0] 86 | (let [check? (or (zero? (mod i 10000)) 87 | (and (> i 99000) (zero? (mod i 100))) 88 | (and (> i 99900) (zero? (mod i 10))))] 89 | (when check? 90 | (print "i=" i " ") 91 | (u/print-optimizer-counts)) 92 | (if (< i benchmark-size) 93 | (recur (if check? 94 | (extra-checks-catvec (fv/vector i) v) 95 | (my-catvec (fv/vector i) v)) 96 | (inc i)) 97 | v)))) 98 | 99 | ;; Approximate run times for this test on a 2015 MacBook Pro 100 | ;; 36 sec - clj 1.10.1, OpenJDK 11.0.4 101 | ;; 465 sec - cljs 1.10.439, OpenJDK 11.0.4, Nashorn JS runtime 102 | ;; 138 sec - cljs 1.10.238, OpenJDK 11.0.4, nodejs 8.10.0 103 | ;; 137 sec - cljs 1.10.238, OpenJDK 11.0.4, Spidermonkey JavaScript-C52.9.1 104 | (deftest test-crrbv-17 105 | (u/reset-optimizer-counts!) 106 | (is (= (reverse (range benchmark-size)) 107 | (vector-push-f (fv/vector) fv/catvec dv/checking-catvec)))) 108 | -------------------------------------------------------------------------------- /src/test/clojure/clojure/core/rrb_vector/long_test.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.long-test 2 | (:require [clojure.test :as test :refer [deftest testing is are]] 3 | [clojure.core.rrb-vector.test-utils :as u] 4 | [clojure.core.rrb-vector :as fv] 5 | [clojure.core.rrb-vector.debug :as dv] 6 | [clojure.core.rrb-vector.debug-platform-dependent :as dpd]) 7 | (:import (clojure.lang ExceptionInfo))) 8 | 9 | ;; The intent is to keep this file as close to 10 | ;; src/test/cljs/clojure/core/rrb_vector/long_test.cljs as possible, 11 | ;; so that when we start requiring Clojure 1.7.0 and later for this 12 | ;; library, this file and that one can be replaced with a common test 13 | ;; file with the suffix .cljc 14 | 15 | ;; Note that the namespace of this file _intentionally_ does not match 16 | ;; the pattern of namespaces that are run for ClojureScript tests by 17 | ;; default. That is because of how long the tests in this file take 18 | ;; to run. It seems best to include them in the set of tests in such 19 | ;; a way that it is only run when a developer explicitly wants to run 20 | ;; longer tests. It should not be run by default when running on 21 | ;; build.clojure.org. 22 | 23 | ;; Currently the Clojure/JVM versions of these tests _are_ run by 24 | ;; default, and on build.clojure.org, but at least the ones in here 25 | ;; now run significantly faster on Clojure/JVM than they do in any of 26 | ;; the JavaScript runtimes I have tested with. 27 | 28 | (dv/set-debug-opts! dv/full-debug-opts) 29 | 30 | (def generative-test-length :short) 31 | 32 | (def check-subvec-params (case generative-test-length 33 | :short [125 100000 10] 34 | :medium [250 200000 20] 35 | :long [250 200000 20])) 36 | 37 | (deftest test-slicing-generative 38 | (testing "slicing (generative)" 39 | (is (try 40 | (apply dv/generative-check-subvec u/extra-checks? check-subvec-params) 41 | (catch ExceptionInfo e 42 | (throw (ex-info (dpd/format "%s: %s %s" 43 | (u/ex-message-copy e) 44 | (:init-cnt (ex-data e)) 45 | (:s&es (ex-data e))) 46 | {} 47 | (u/ex-cause-copy e)))))))) 48 | 49 | ;; short: 2 to 3 sec 50 | ;; medium: 50 to 60 sec 51 | (def check-catvec-params (case generative-test-length 52 | :short [ 10 30 10 60000] 53 | :medium [250 30 10 60000] 54 | :long [250 30 10 60000])) 55 | 56 | (deftest test-splicing-generative 57 | (testing "splicing (generative)" 58 | (is (try 59 | (apply dv/generative-check-catvec u/extra-checks? check-catvec-params) 60 | (catch ExceptionInfo e 61 | (throw (ex-info (dpd/format "%s: %s" 62 | (u/ex-message-copy e) 63 | (:cnts (ex-data e))) 64 | {} 65 | (u/ex-cause-copy e)))))))) 66 | 67 | 68 | ;; This problem reproduction code is from CRRBV-17 ticket: 69 | ;; https://clojure.atlassian.net/projects/CRRBV/issues/CRRBV-17 70 | 71 | (def benchmark-size 100000) 72 | 73 | ;; This small variation of the program in the ticket simply does 74 | ;; progress debug printing occasionally, as well as extra debug 75 | ;; checking of the results occasionally. 76 | 77 | ;; If you enable the printing of the message that begins 78 | ;; with "splice-rrbts result had shift" in function 79 | ;; fallback-to-slow-splice-if-needed, then run this test, you will see 80 | ;; it called hundreds or perhaps thousands of times. The fallback 81 | ;; approach is effective at avoiding a crash for this scenario, but at 82 | ;; a dramatic extra run-time cost. 83 | 84 | (defn vector-push-f [v my-catvec extra-checks-catvec] 85 | (loop [v v 86 | i 0] 87 | (let [check? (or (zero? (mod i 10000)) 88 | (and (> i 99000) (zero? (mod i 100))) 89 | (and (> i 99900) (zero? (mod i 10))))] 90 | (when check? 91 | (print "i=" i " ") 92 | (u/print-optimizer-counts)) 93 | (if (< i benchmark-size) 94 | (recur (if check? 95 | (extra-checks-catvec (fv/vector i) v) 96 | (my-catvec (fv/vector i) v)) 97 | (inc i)) 98 | v)))) 99 | 100 | ;; Approximate run times for this test on a 2015 MacBook Pro 101 | ;; 36 sec - clj 1.10.1, OpenJDK 11.0.4 102 | ;; 465 sec - cljs 1.10.439, OpenJDK 11.0.4, Nashorn JS runtime 103 | ;; 138 sec - cljs 1.10.238, OpenJDK 11.0.4, nodejs 8.10.0 104 | ;; 137 sec - cljs 1.10.238, OpenJDK 11.0.4, Spidermonkey JavaScript-C52.9.1 105 | (deftest test-crrbv-17 106 | (u/reset-optimizer-counts!) 107 | (is (= (reverse (range benchmark-size)) 108 | (vector-push-f (fv/vector) fv/catvec dv/checking-catvec)))) 109 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector 2 | 3 | "An implementation of the confluently persistent vector data 4 | structure introduced in Bagwell, Rompf, \"RRB-Trees: Efficient 5 | Immutable Vectors\", EPFL-REPORT-169879, September, 2011. 6 | 7 | RRB-Trees build upon Clojure's PersistentVectors, adding logarithmic 8 | time concatenation and slicing. 9 | 10 | The main API entry points are clojure.core.rrb-vector/catvec, 11 | performing vector concatenation, and clojure.core.rrb-vector/subvec, 12 | which produces a new vector containing the appropriate subrange of 13 | the input vector (in contrast to clojure.core/subvec, which returns 14 | a view on the input vector). 15 | 16 | core.rrb-vector's vectors can store objects or unboxed primitives. 17 | The implementation allows for seamless interoperability with 18 | clojure.lang.PersistentVector, clojure.core.Vec (more commonly known 19 | as gvec) and clojure.lang.APersistentVector$SubVector instances: 20 | clojure.core.rrb-vector/catvec and clojure.core.rrb-vector/subvec 21 | convert their inputs to clojure.core.rrb-vector.rrbt.Vector 22 | instances whenever necessary (this is a very fast constant time 23 | operation for PersistentVector and gvec; for SubVector it is O(log 24 | n), where n is the size of the underlying vector). 25 | 26 | clojure.core.rrb-vector also exports its own versions of vector and 27 | vector-of and vec which always produce 28 | clojure.core.rrb-vector.rrbt.Vector instances. Note that vector-of 29 | accepts :object as one of the possible type arguments, in addition 30 | to keywords naming primitive types." 31 | 32 | {:author "Michał Marczyk"} 33 | 34 | (:refer-clojure :exclude [vector vector-of vec subvec]) 35 | (:require [clojure.core.rrb-vector.parameters :as p] 36 | [clojure.core.rrb-vector.protocols :refer [slicev splicev]] 37 | [clojure.core.rrb-vector.nodes 38 | :refer [ams object-am object-nm primitive-nm 39 | empty-pv-node empty-gvec-node]] 40 | [clojure.core.rrb-vector.rrbt :refer [as-rrbt]] 41 | clojure.core.rrb-vector.interop) 42 | (:import (clojure.core.rrb_vector.rrbt Vector) 43 | (clojure.core.rrb_vector.nodes NodeManager) 44 | (clojure.core ArrayManager))) 45 | 46 | (set! *warn-on-reflection* true) 47 | (set! *unchecked-math* true) ;; :warn-on-boxed 48 | 49 | (defn catvec 50 | "Concatenates the given vectors in logarithmic time." 51 | ([] 52 | []) 53 | ([v1] 54 | v1) 55 | ([v1 v2] 56 | (splicev v1 v2)) 57 | ([v1 v2 v3] 58 | (splicev (splicev v1 v2) v3)) 59 | ([v1 v2 v3 v4] 60 | (splicev (splicev v1 v2) (splicev v3 v4))) 61 | ([v1 v2 v3 v4 & vn] 62 | (splicev (splicev (splicev v1 v2) (splicev v3 v4)) 63 | (apply catvec vn)))) 64 | 65 | (defn subvec 66 | "Returns a new vector containing the elements of the given vector v 67 | lying between the start (inclusive) and end (exclusive) indices in 68 | logarithmic time. end defaults to end of vector. The resulting 69 | vector shares structure with the original, but does not hold on to 70 | any elements of the original vector lying outside the given index 71 | range." 72 | ([v start] 73 | (slicev v start (count v))) 74 | ([v start end] 75 | (slicev v start end))) 76 | 77 | (defmacro ^:private gen-vector-method [& params] 78 | (let [arr (with-meta (gensym "arr__") {:tag 'objects})] 79 | `(let [~arr (object-array ~(count params))] 80 | ~@(map-indexed (fn [i param] 81 | `(aset ~arr ~i ~param)) 82 | params) 83 | (Vector. ^NodeManager object-nm ^ArrayManager object-am 84 | ~(count params) p/shift-increment empty-pv-node ~arr nil 0 0)))) 85 | 86 | (defn vector 87 | "Creates a new vector containing the args." 88 | ([] 89 | (gen-vector-method)) 90 | ([x1] 91 | (gen-vector-method x1)) 92 | ([x1 x2] 93 | (gen-vector-method x1 x2)) 94 | ([x1 x2 x3] 95 | (gen-vector-method x1 x2 x3)) 96 | ([x1 x2 x3 x4] 97 | (gen-vector-method x1 x2 x3 x4)) 98 | ([x1 x2 x3 x4 & xn] 99 | (loop [v (transient (vector x1 x2 x3 x4)) 100 | xn xn] 101 | (if xn 102 | (recur (.conj ^clojure.lang.ITransientCollection v (first xn)) 103 | (next xn)) 104 | (persistent! v))))) 105 | 106 | (defn vec 107 | "Returns a vector containing the contents of coll. 108 | 109 | If coll is a vector, returns an RRB vector using the internal tree 110 | of coll." 111 | [coll] 112 | (if (vector? coll) 113 | (as-rrbt coll) 114 | (apply vector coll))) 115 | 116 | (defmacro ^:private gen-vector-of-method [t & params] 117 | (let [am (gensym "am__") 118 | nm (gensym "nm__") 119 | arr (gensym "arr__")] 120 | `(let [~am ^ArrayManager (ams ~t) 121 | ~nm ^NodeManager (if (identical? ~t :object) object-nm primitive-nm) 122 | ~arr (.array ~am ~(count params))] 123 | ~@(map-indexed (fn [i param] 124 | `(.aset ~am ~arr ~i ~param)) 125 | params) 126 | (Vector. ~nm ~am ~(count params) p/shift-increment 127 | (if (identical? ~t :object) empty-pv-node empty-gvec-node) 128 | ~arr nil 0 0)))) 129 | 130 | (defn vector-of 131 | "Creates a new vector capable of storing homogenous items of type t, 132 | which should be one of :object, :int, :long, :float, :double, :byte, 133 | :short, :char, :boolean. Primitives are stored unboxed. 134 | 135 | Optionally takes one or more elements to populate the vector." 136 | ([t] 137 | (gen-vector-of-method t)) 138 | ([t x1] 139 | (gen-vector-of-method t x1)) 140 | ([t x1 x2] 141 | (gen-vector-of-method t x1 x2)) 142 | ([t x1 x2 x3] 143 | (gen-vector-of-method t x1 x2 x3)) 144 | ([t x1 x2 x3 x4] 145 | (gen-vector-of-method t x1 x2 x3 x4)) 146 | ([t x1 x2 x3 x4 & xn] 147 | (loop [v (transient (vector-of t x1 x2 x3 x4)) 148 | xn xn] 149 | (if xn 150 | (recur (.conj ^clojure.lang.ITransientCollection v (first xn)) 151 | (next xn)) 152 | (persistent! v))))) 153 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/debug_platform_dependent.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.debug-platform-dependent 10 | (:require [clojure.core.rrb-vector.rrbt :as rrbt] 11 | [clojure.core.rrb-vector.nodes :refer [regular? node-ranges]] 12 | [clojure.core.rrb-vector :as fv] 13 | [goog.string :as gstring] 14 | goog.string.format)) 15 | 16 | (defn format [& args] 17 | (apply gstring/format args)) 18 | 19 | (defn printf [& args] 20 | (print (apply gstring/format args))) 21 | 22 | (defn internal-node? [x] 23 | ;; TBD: Is there another type that should be included here? Clojure 24 | ;; only has this one, plus a VecNode that is distinct to the 25 | ;; primitive vectors, which ClojureScript does not have. 26 | (instance? cljs.core/VectorNode x)) 27 | 28 | (defn persistent-vector? [x] 29 | (or (instance? cljs.core/PersistentVector x) 30 | (instance? cljs.core/Subvec x) 31 | (instance? clojure.core.rrb-vector.rrbt/Vector x))) 32 | 33 | (defn transient-vector? [x] 34 | (or (instance? cljs.core/TransientVector x) 35 | (instance? clojure.core.rrb-vector.rrbt/Transient x))) 36 | 37 | (defn is-vector? [x] 38 | (or (persistent-vector? x) 39 | (transient-vector? x))) 40 | 41 | (defn dbg-tailoff [v] 42 | (cond 43 | (or (instance? cljs.core/PersistentVector v) 44 | (instance? cljs.core/TransientVector v)) 45 | (#'cljs.core/tail-off v) 46 | 47 | (or (instance? clojure.core.rrb-vector.rrbt/Vector v) 48 | (instance? clojure.core.rrb-vector.rrbt/Transient v)) 49 | (rrbt/-tail-offset v) 50 | 51 | :else 52 | (throw (ex-info (str "Called debug-tailoff on value with unsupported type " 53 | (pr-str (type v))) 54 | {:value v})))) 55 | 56 | (defn subvector-data [v] 57 | (if (instance? cljs.core/Subvec v) 58 | {:orig-v v 59 | :subvector? true 60 | :v (.-v v) 61 | :subvec-start (.-start v) 62 | :subvec-end (.-end v)} 63 | {:orig-v v 64 | :subvector? false 65 | :v v})) 66 | 67 | (defn unwrap-subvec-accessors-for [v] 68 | (let [{:keys [v] :as m} (subvector-data v)] 69 | (merge m {:get-root #(.-root %) 70 | :get-shift #(.-shift %) 71 | :get-tail #(.-tail %) 72 | :get-cnt #(.-cnt %) 73 | :get-array #(.-arr %) 74 | :get-ranges node-ranges 75 | :regular? regular? 76 | :tail-len alength}))) 77 | 78 | (defn dbg-tidx [tv] 79 | (if (transient-vector? tv) 80 | (if (instance? cljs.core/TransientVector tv) 81 | (let [c (.-cnt tv)] 82 | (if (== c 32) 83 | 32 84 | (bit-and c 0x01f))) 85 | (.-tidx tv)))) 86 | 87 | (defn abbrev-for-type-of [obj] 88 | (let [tn (pr-str (type obj)) 89 | d (.lastIndexOf tn ".")] 90 | (subs tn (inc d)))) 91 | 92 | (defn same-coll? [a b] 93 | (and (= (count a) 94 | (count b)) 95 | (= a b) 96 | (= b a) 97 | (= (hash a) (hash b)) 98 | ;; TBD: Is there anything JavaScript specific that corresponds 99 | ;; to Java's .hashCode method? 100 | ;;(= (.hashCode ^Object a) (.hashCode ^Object b)) 101 | )) 102 | 103 | ;; In ClojureScript, as in JavaScript, there is only 1 thread, so all 104 | ;; updates to transients are automatically thread confined, to the 105 | ;; only thread. There is still an edit field/property on the vector 106 | ;; tree nodes, but it is used only for the purpose of distinguishing 107 | ;; which nodes are uniquely "owned" by this transient vector, 108 | ;; vs. those tree nodes that might be shared with other persistent 109 | ;; vectors. For that purpose, when a vector is made transient, its 110 | ;; root tree node is assigned a new unique JavaScript object, as 111 | ;; returned by (js-obj). Any tree node whose edit field is identical 112 | ;; to that one is owned by this transient, any not identical are not. 113 | ;; Only the root tree node has its edit field changed to nil when the 114 | ;; transient vector is converted to persistent, because any other tree 115 | ;; nodes that are identical to it will never be identical to another 116 | ;; new (js-obj) return value. 117 | 118 | ;; Thus for the JavaScript implementation, it is not the case that in 119 | ;; persistent vectors that all edit fields must be nil, or an 120 | ;; AtomicReference that contains nil. Some of them can be return 121 | ;; values of (js-obj) from times when they were part of a transient 122 | ;; vector, but are no longer. 123 | 124 | ;; About the only thing we can check here that I believe must always 125 | ;; be true, is that persistent vectors have a root tree node with edit 126 | ;; field equal to nil, and transient vectors must have a root tree 127 | ;; node with edit field not equal to nil. 128 | 129 | (defn edit-nodes-errors [v _] 130 | (let [{:keys [v get-root]} (unwrap-subvec-accessors-for v) 131 | root-edit (.-edit (get-root v)) 132 | root-edit-is-nil? (nil? root-edit)] 133 | (cond 134 | (and (transient-vector? v) 135 | root-edit-is-nil?) 136 | {:error true 137 | :description (str "A transient vector with type" (pr-str (type v)) 138 | " has a root edit property with value nil" 139 | " - expecting a non-nil JavaScript object") 140 | :data v} 141 | 142 | (and (persistent-vector? v) 143 | (not root-edit-is-nil?)) 144 | {:error false, :warning true, 145 | :description (str "A persistent vector with type " (pr-str (type v)) 146 | " has a root edit property with value " root-edit 147 | " - often this is nil instead." 148 | " It requires more thought to be certain" 149 | " whether this could lead to problems," 150 | " hence why this is only a warning") 151 | :data v} 152 | 153 | :else 154 | {:error false}))) 155 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changes in 0.2.0 2 | 3 | * Update dependencies and versions 4 | 5 | # Changes in 0.1.2 6 | 7 | Bug fixes: 8 | 9 | * Correct handling of reduce-kv for empty vectors [CRRBV-29](https://clojure.atlassian.net/browse/CRRBV-29) 10 | 11 | # Changes in 0.1.1 12 | 13 | ## Changes visible to users of the library 14 | 15 | Bug fixes: 16 | 17 | * Eliminate warning issued by ClojureScript compiler caused by use of a not-yet-defined type name [CRRBV-24](https://clojure.atlassian.net/browse/CRRBV-24) 18 | 19 | Minor code cleanup: 20 | 21 | * Eliminate redundant require of a namespace 22 | [commit](https://github.com/clojure/core.rrb-vector/commit/8c3bdc03f4d4c73326ac0146310bf472cda4d035) 23 | 24 | Documentation: 25 | 26 | * Created this change log. 27 | * Added new introductory text at beginning of README explaining 28 | briefly why someone might want to use this library. 29 | * Added doc/benchmarks/benchmarks.md document, with link from README, 30 | showing some benchmark results of this library versus several other 31 | implementations of vector data structures, some of which are based 32 | on RRB trees, some of which have only a linear time vector 33 | concatenation operation. 34 | * Added doc/rrb-tree-notes.md with links to other implementations of 35 | RRB trees, and papers and theses that have been written about them. 36 | 37 | 38 | ## Changes relevant to those who test and develop the library itself 39 | 40 | * Added doc/use-transducers/ directory with README and proposed patch 41 | for speeding up some of the code by using transducers, which in the 42 | cases used provide a speedup by avoiding allocating multiple 43 | intermediate sequences. 44 | 45 | 46 | # Changes in 0.1.0 47 | 48 | ## Changes visible to users of the library 49 | 50 | Bug fixes: 51 | 52 | * Test case added that was failing before the fixes for other bugs listed below, but now works with those fixes, so likely its root cause has also been corrected [CRRBV-12](https://clojure.atlassian.net/browse/CRRBV-12) 53 | * Fixed bug that caused assoc and assoc! to fail when used on vectors of primitives [CRRBV-13](https://clojure.atlassian.net/browse/CRRBV-13) 54 | * Fixed bug where internal tree data structure gets too "tall and skinny", eventually exceeding the limits supported by the library's implementation [CRRBV-14](https://clojure.atlassian.net/browse/CRRBV-14) 55 | * Bug with similar root cause, and the same fix as, CRRBV-14 [CRRBV-17](https://clojure.atlassian.net/browse/CRRBV-17) 56 | * Fix incorrect condition check for when a subtree was full or not [CRRBV-20](https://clojure.atlassian.net/browse/CRRBV-20) 57 | * Fix of several bugs found during implementation and testing of other issues [CRRBV-21](https://clojure.atlassian.net/browse/CRRBV-21) 58 | * Fix off by one bug that caused incorrect results for pop and pop! operations on vectors of certain sizes [CRRBV-22](https://clojure.atlassian.net/browse/CRRBV-22) 59 | * Fix incorrect hash calculation for empty RRB vectors in ClojureScript implementation [CRRBV-25](https://clojure.atlassian.net/browse/CRRBV-25) 60 | * Fix potential data race for multi-threaded Clojure programs using this library where the hash value could be returned incorrectly as -1 instead of the correct value [CRRBV-26](https://clojure.atlassian.net/browse/CRRBV-26) 61 | 62 | Enhancements: 63 | 64 | * Support bootstrapped ClojureScript [CRRBV-16](https://clojure.atlassian.net/browse/CRRBV-16) 65 | 66 | ## Changes relevant to those who test and develop the library itself 67 | 68 | In order to continue to support Clojure 1.6.0, core.rrb-vector uses 69 | neither `.cljc` files nor transducers, for which Clojure added support 70 | in Clojure 1.7.0. However, in preparation for a future 71 | core.rrb-vector release that requires Clojure 1.7.0 or later, several 72 | test namespaces have been made nearly identical between their Clojure 73 | and ClojureScript versions, so that they can be replaced with a single 74 | `.cljc` file in the future, with only a few small uses of reader 75 | conditionals. 76 | 77 | The implementation of core.rrb-vector still contains similar, but 78 | independent, implementations in Clojure and ClojureScript, and no 79 | attempt has been made to make their implementations so similar that 80 | merging them into a combined .cljc file would be reasonable. 81 | Replacing some or all of the Clojure implementation with Java source 82 | code may help improve the constant factors of the execution time 83 | enough to warrant such a change in a future version of this library. 84 | 85 | * Updated Maven pom.xml file, Leiningen project.clj file, and Clojure 86 | deps.edn files, so that any of them may be used for the purposes of 87 | developing and testing this library further. Added examples of 88 | commands for all of these tools for running this library's tests, 89 | and commands for Ubuntu Linux and macOS for installing two different 90 | JavaScript run time engines that can be used to test the 91 | ClojureScript implementation. 92 | * The ClojureScript tests are now run, using JDK's Nashorn JavaScript 93 | run time environment, on build.clojure.org, in addition to the 94 | previous behavior of running the Clojure tests. 95 | * Rearranged tests in the test namespaces extensively, including some 96 | of their namespace names. There is now a `test-common` namespace 97 | that contains most of the tests, which can thus be run on both the 98 | Clojure and ClojureScript implementations. There are 99 | `test-clj-only` and `test-cljs-only` namespaces for tests unique to 100 | one of the implementations. 101 | * The `clojure.core.rrb-vector.debug/dbg-vec` function has been 102 | enhanced to support showing internal details of both the built in 103 | Clojure vectors, persistent and transient, as well as 104 | core.rrb-vector's data structures. The `debug` namespace also has 105 | several `checking-*` functions, e.g. `checking-catvec`, 106 | `checking-subvec`, etc. that behave the same as their non-checking 107 | counterparts, but perform significant sanity checking of their 108 | return values before they return, and while they have some 109 | configuration options, by default they throw exceptions if they find 110 | errors in the returned values. Any such exceptions are likely to be 111 | due to bugs in core.rrb-vector. These checking functions are 112 | significantly slower than the non-checking variants, and only 113 | intended for testing the core.rrb-vector library. The details of 114 | configuring their options are not intended to be stable, and thus 115 | likely to change in future releases of this library. 116 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector 10 | 11 | "An implementation of the confluently persistent vector data 12 | structure introduced in Bagwell, Rompf, \"RRB-Trees: Efficient 13 | Immutable Vectors\", EPFL-REPORT-169879, September, 2011. 14 | 15 | RRB-Trees build upon Clojure's PersistentVectors, adding logarithmic 16 | time concatenation and slicing. 17 | 18 | The main API entry points are clojure.core.rrb-vector/catvec, 19 | performing vector concatenation, and clojure.core.rrb-vector/subvec, 20 | which produces a new vector containing the appropriate subrange of 21 | the input vector (in contrast to clojure.core/subvec, which returns 22 | a view on the input vector). 23 | 24 | core.rrb-vector's vectors can store objects or unboxed primitives. 25 | The implementation allows for seamless interoperability with 26 | clojure.lang.PersistentVector, clojure.core.Vec (more commonly known 27 | as gvec) and clojure.lang.APersistentVector$SubVector instances: 28 | clojure.core.rrb-vector/catvec and clojure.core.rrb-vector/subvec 29 | convert their inputs to clojure.core.rrb-vector.rrbt.Vector 30 | instances whenever necessary (this is a very fast constant time 31 | operation for PersistentVector and gvec; for SubVector it is O(log 32 | n), where n is the size of the underlying vector). 33 | 34 | clojure.core.rrb-vector also exports its own versions of vector and 35 | vector-of and vec which always produce 36 | clojure.core.rrb-vector.rrbt.Vector instances. Note that vector-of 37 | accepts :object as one of the possible type arguments, in addition 38 | to keywords naming primitive types." 39 | 40 | {:author "Michał Marczyk"} 41 | 42 | (:refer-clojure :exclude [vector vector-of vec subvec]) 43 | (:require [clojure.core.rrb-vector.parameters :as p] 44 | [clojure.core.rrb-vector.protocols :refer [slicev splicev]] 45 | [clojure.core.rrb-vector.nodes 46 | :refer [ams object-am object-nm primitive-nm 47 | empty-pv-node empty-gvec-node]] 48 | [clojure.core.rrb-vector.rrbt :refer [as-rrbt]] 49 | clojure.core.rrb-vector.interop) 50 | (:import (clojure.core.rrb_vector.rrbt Vector) 51 | (clojure.core.rrb_vector.nodes NodeManager) 52 | (clojure.core ArrayManager))) 53 | 54 | (set! *warn-on-reflection* true) 55 | (set! *unchecked-math* true) ;; :warn-on-boxed 56 | 57 | (defn catvec 58 | "Concatenates the given vectors in logarithmic time." 59 | ([] 60 | []) 61 | ([v1] 62 | v1) 63 | ([v1 v2] 64 | (splicev v1 v2)) 65 | ([v1 v2 v3] 66 | (splicev (splicev v1 v2) v3)) 67 | ([v1 v2 v3 v4] 68 | (splicev (splicev v1 v2) (splicev v3 v4))) 69 | ([v1 v2 v3 v4 & vn] 70 | (splicev (splicev (splicev v1 v2) (splicev v3 v4)) 71 | (apply catvec vn)))) 72 | 73 | (defn subvec 74 | "Returns a new vector containing the elements of the given vector v 75 | lying between the start (inclusive) and end (exclusive) indices in 76 | logarithmic time. end defaults to end of vector. The resulting 77 | vector shares structure with the original, but does not hold on to 78 | any elements of the original vector lying outside the given index 79 | range." 80 | ([v start] 81 | (slicev v start (count v))) 82 | ([v start end] 83 | (slicev v start end))) 84 | 85 | (defmacro ^:private gen-vector-method [& params] 86 | (let [arr (with-meta (gensym "arr__") {:tag 'objects})] 87 | `(let [~arr (object-array ~(count params))] 88 | ~@(map-indexed (fn [i param] 89 | `(aset ~arr ~i ~param)) 90 | params) 91 | (Vector. ^NodeManager object-nm ^ArrayManager object-am 92 | ~(count params) 5 empty-pv-node ~arr nil 0 0)))) 93 | 94 | (defn vector 95 | "Creates a new vector containing the args." 96 | ([] 97 | (gen-vector-method)) 98 | ([x1] 99 | (gen-vector-method x1)) 100 | ([x1 x2] 101 | (gen-vector-method x1 x2)) 102 | ([x1 x2 x3] 103 | (gen-vector-method x1 x2 x3)) 104 | ([x1 x2 x3 x4] 105 | (gen-vector-method x1 x2 x3 x4)) 106 | ([x1 x2 x3 x4 & xn] 107 | (loop [v (transient (vector x1 x2 x3 x4)) 108 | xn xn] 109 | (if xn 110 | (recur (.conj ^clojure.lang.ITransientCollection v (first xn)) 111 | (next xn)) 112 | (persistent! v))))) 113 | 114 | (defn vec 115 | "Returns a vector containing the contents of coll. 116 | 117 | If coll is a vector, returns an RRB vector using the internal tree 118 | of coll." 119 | [coll] 120 | (if (vector? coll) 121 | (as-rrbt coll) 122 | (apply vector coll))) 123 | 124 | (defmacro ^:private gen-vector-of-method [t & params] 125 | (let [am (gensym "am__") 126 | nm (gensym "nm__") 127 | arr (gensym "arr__")] 128 | `(let [~am ^ArrayManager (ams ~t) 129 | ~nm ^NodeManager (if (identical? ~t :object) object-nm primitive-nm) 130 | ~arr (.array ~am ~(count params))] 131 | ~@(map-indexed (fn [i param] 132 | `(.aset ~am ~arr ~i ~param)) 133 | params) 134 | (Vector. ~nm ~am ~(count params) 5 135 | (if (identical? ~t :object) empty-pv-node empty-gvec-node) 136 | ~arr nil 0 0)))) 137 | 138 | (defn vector-of 139 | "Creates a new vector capable of storing homogenous items of type t, 140 | which should be one of :object, :int, :long, :float, :double, :byte, 141 | :short, :char, :boolean. Primitives are stored unboxed. 142 | 143 | Optionally takes one or more elements to populate the vector." 144 | ([t] 145 | (gen-vector-of-method t)) 146 | ([t x1] 147 | (gen-vector-of-method t x1)) 148 | ([t x1 x2] 149 | (gen-vector-of-method t x1 x2)) 150 | ([t x1 x2 x3] 151 | (gen-vector-of-method t x1 x2 x3)) 152 | ([t x1 x2 x3 x4] 153 | (gen-vector-of-method t x1 x2 x3 x4)) 154 | ([t x1 x2 x3 x4 & xn] 155 | (loop [v (transient (vector-of t x1 x2 x3 x4)) 156 | xn xn] 157 | (if xn 158 | (recur (.conj ^clojure.lang.ITransientCollection v (first xn)) 159 | (next xn)) 160 | (persistent! v))))) 161 | -------------------------------------------------------------------------------- /doc/crrbv-27/description.md: -------------------------------------------------------------------------------- 1 | The production version of the Clojure/Java core.rrb-vector library is 2 | in the directory `src/main/clojure`. It uses a maximum tree branching 3 | factor of 32, i.e. all tree nodes have at most 32 children, the same 4 | as Clojure's built in persistent vectors. This is good for look-up 5 | efficiency, but when testing the code, it requires a large number of 6 | vector elements and/or operations on the vectors in order to reach 7 | "interesting" tree structures that exercise all parts of the code, and 8 | find bugs that may be there. 9 | 10 | The `src/parameterized/clojure` directory contains source code for a 11 | modified version of the core.rrb-vector library, which uses parameters 12 | defined in file 13 | `src/parameterized/clojure/clojure/core/rrb_vector/parameters.clj` to 14 | control the maximum tree branching factor. The existing code allows 15 | you to change the value of `shift-increment` to any value that is 2 or 16 | larger, and the maximum tree branching factor will then be 2 to the 17 | power of `shift-increment`. 18 | 19 | I have found a bug in the core.rrb-vector library that I do not yet 20 | fully understand. I have a patch to the code that causes the problem 21 | not to occur, but I do not understand the original or modified code 22 | well enough to be confident that it is a correct fix. 23 | 24 | I do have an easily reproducible test case that causes the problem to 25 | occur with the parameterized version of the code when 26 | `shift-increment` is 2, so a maximum tree branch factor of 4. I 27 | believe it is likely that a different test sequence could be found 28 | that exhibits the same bug with the production code's branch factor of 29 | 32, but it would likely be a far longer sequence of operations, 30 | e.g. perhaps as long as millions of operations or more. 31 | 32 | Here is a way to reproduce the problem with the parameterized version 33 | of the code. 34 | 35 | ```bash 36 | git clone https://github.com/clojure/core.rrb-vector 37 | cd core.rrb-vector 38 | git checkout f69df0f95e450bb4ff8e3294f3265d3d25f4e5db 39 | patch -p1 < use-shift-increment-2.patch 40 | ``` 41 | 42 | You can see these tests fail: 43 | 44 | ```bash 45 | ./script/jdo check 46 | ``` 47 | 48 | To reproduce the same problem in the REPL, it takes a few more steps, 49 | shown below. 50 | 51 | ```bash 52 | # Start a JVM with a socket REPL listening on TCP port 50505 53 | ./script/jdo 54 | ``` 55 | 56 | ```clojure 57 | ;; Connect to the socket REPL from your favorite dev environment, via 58 | ;; either a socket REPL, nREPL, whatever you prefer and know how to 59 | ;; set up. 60 | 61 | (require '[clojure.core.rrb-vector :as fv] 62 | '[clojure.core.rrb-vector.debug :as dv] 63 | '[clojure.core.rrb-vector.rrbt :as rrbt]) 64 | 65 | ;; Enable full debug options, except trace printing is off, for all 66 | ;; checking-* functions in the debug namespace. 67 | (dv/set-debug-opts! dv/full-debug-opts) 68 | 69 | ;; The shortest sequence of operations I currently know that reaches 70 | ;; the point where the bug occurs only after about 2000 calls to 71 | ;; insert-by-sub-vcatvec, with a particular sequence of arguments. 72 | 73 | ;; It takes a couple of minutes to get there when using 74 | ;; checking-catvec and checking-subvec, but we can capture the vectors 75 | ;; that cause the problem once we get there, and as long as we keep 76 | ;; the same JVM running we can examine their contents all we want. 77 | 78 | (def my-catvec dv/checking-catvec) 79 | (def my-subvec dv/checking-subvec) 80 | 81 | ;; This code is slightly modified from that in the deftest named 82 | ;; test-reduce-subvec-catvec in the test-common namespace. 83 | 84 | (defn insert-at-index-n [v n] 85 | (my-catvec (my-subvec v 0 n) 86 | (dv/cvec ['x]) 87 | (my-subvec v n))) 88 | 89 | (defn insert-by-sub-catvec [v [n sz]] 90 | (let [ret (insert-at-index-n v n)] 91 | (when (or (< n 20) 92 | (zero? (mod n 10))) 93 | (println "n=" n)) 94 | (let [ret-nums (filter number? ret)] 95 | (if (not= (filter number? ret) (range sz)) 96 | (throw (ex-info (str "Failure for sz=" sz " n=" n) 97 | {:v v :sz sz :n n})))) 98 | ret)) 99 | 100 | (defn repeated-subvec-catvec [sz] 101 | (reduce insert-by-sub-catvec 102 | (dv/cvec (range sz)) 103 | (map (fn [x] [x sz]) 104 | (range sz 0 -1)))) 105 | 106 | (count @dv/failure-data) 107 | ;; should be 0 initially 108 | 109 | (def sz 2061) 110 | (def x (repeated-subvec-catvec sz)) 111 | ;; That caused an exception to be thrown just after "n= 15" was printed 112 | 113 | ;; Record the exception in e1 114 | (def e1 *e) 115 | (count @dv/failure-data) 116 | ;; should be 1 now, since data about 1 error has been appended to the vector 117 | ;; @dv/failure-data. 118 | 119 | ;; extract the data from the error 120 | (def ed (nth @dv/failure-data 0)) 121 | (keys ed) 122 | (:err-desc-str ed) 123 | ;; "splice-rrbts-main" 124 | (def vret (:ret ed)) 125 | (count (:args ed)) 126 | ;; should be 4. We only care about the last two args, which are 127 | ;; the two vectors that we were trying to concatenate. 128 | (def v1 (nth (:args ed) 2)) 129 | (def v2 (nth (:args ed) 3)) 130 | (def errinf (:error-info ed)) 131 | errinf 132 | ;; {:error true, :description "One or more errors found", :data ({:error true, :kind :internal, :description "Found internal non-regular node with 1 non-nil, 3 nil children, and # children prefix sums: (39) - expected that to match stored ranges: (33 0 0 0 1)"})} 133 | 134 | (dv/dbg-vec v1) 135 | (dv/dbg-vec v2) 136 | (dv/dbg-vec vret) 137 | ``` 138 | 139 | In the long output of the dbg-vec on vret, there is only one node that 140 | has a ranges array printed as (33 0 0 0 1), so that must be the one 141 | that the error message above is referring to. 142 | 143 | Here are the few lines before and after that node, which are near the 144 | beginning of the output: 145 | 146 | ``` 147 | Vector (4109 elements): 148 | 16:00 PersistentVector$Node: (33 4108 0 0 2) 149 | 14:00 PersistentVector$Node: (33 0 0 0 1) 150 | 12:00 PersistentVector$Node: (37 39 0 0 2) 151 | 10:00 PersistentVector$Node: (21 29 35 37 4) 152 | ``` 153 | 154 | The line starting with 14:00 is the node that has the reported 155 | problem. 156 | 157 | Its first child is the node printed on the line starting with 12:00. 158 | Note that its ranges array contains (37 39 0 0 2), so that node has 2 159 | children, the first with 37 vector elements as leaves beneath it, the 160 | second with 39-37=2 vector elements with leaves beneath it (if those 161 | values are correct -- I believe they are, else ranges-errors would 162 | have complained about those before its parent node). 163 | 164 | The node 14:00 should have a ranges array (39 0 0 0 1), but it is (33 165 | 0 0 0 1), so it is under-counting the number of vector elements 166 | beneath it by 6. Its parent node, the one printed on line 16:00, is 167 | consistent with node 14:00, but also too small by 6. 168 | 169 | -------------------------------------------------------------------------------- /src/test/clojure/clojure/core/rrb_vector/test_clj_only.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.test-clj-only 2 | (:require [clojure.test :as test :refer [deftest testing is are]] 3 | [clojure.template :refer [do-template]] 4 | [clojure.reflect :as ref] 5 | [clojure.core.rrb-vector.test-utils :as u] 6 | [clojure.core.rrb-vector :as fv] 7 | [clojure.core.rrb-vector.debug :as dv] 8 | [clojure.test.check :as tc] 9 | [clojure.test.check.properties :as prop] 10 | [clojure.test.check.generators :as gen]) 11 | (:import (java.util NoSuchElementException))) 12 | 13 | (dv/set-debug-opts! dv/full-debug-opts) 14 | 15 | (defn clj-version-at-least [major-minor-vector] 16 | (let [clj-version ((juxt :major :minor) *clojure-version*) 17 | cmp (compare clj-version major-minor-vector)] 18 | (>= cmp 0))) 19 | 20 | (deftest test-iterators 21 | (let [v (fv/catvec (dv/cvec (range 1000)) (dv/cvec (range 1000 2048)))] 22 | (is (= (iterator-seq (.iterator ^Iterable v)) 23 | (iterator-seq (.iterator ^Iterable (seq v))) 24 | (iterator-seq (.listIterator ^java.util.List v)) 25 | (iterator-seq (.listIterator ^java.util.List (seq v))) 26 | (range 2048))) 27 | (is (= (iterator-seq (.listIterator ^java.util.List v 100)) 28 | (iterator-seq (.listIterator ^java.util.List (seq v) 100)) 29 | (range 100 2048))) 30 | (letfn [(iterator [xs] 31 | (.iterator ^Iterable xs)) 32 | (list-iterator 33 | ([xs] 34 | (.listIterator ^java.util.List xs)) 35 | ([xs start] 36 | (.listIterator ^java.util.List xs start)))] 37 | (do-template [iexpr cnt] 38 | (is (thrown? NoSuchElementException 39 | (let [iter iexpr] 40 | (dotimes [_ (inc cnt)] 41 | (.next ^java.util.Iterator iter))))) 42 | (iterator v) 2048 43 | (iterator (seq v)) 2048 44 | (list-iterator v) 2048 45 | (list-iterator (seq v)) 2048 46 | (list-iterator v 100) 1948 47 | (list-iterator (seq v) 100) 1948)))) 48 | 49 | ;; This test can run in cljs, too, but at least in my testing only if 50 | ;; we use test.check version 0.10.0 or later. However, that seems to 51 | ;; be incompatible with running cljs tests with Clojure 1.6.0, so for 52 | ;; now at least this test is clj-only. 53 | ;; 54 | ;; Note: according to several deftest forms within the test.check 55 | ;; library's own internal set of tests, it 56 | ;; uses (is (:result (tc/quick-check ...))) to check whether a result 57 | ;; passes or fails. 58 | ;; 59 | ;; The doc string for the latest version of test.check as of 60 | ;; 2019-Sep-25 says :result is a legacy key, and that :pass? should 61 | ;; have the same value. I like the descriptiveness of :pass? better, 62 | ;; and would prefer to use that here, but core.rrb-vector is not using 63 | ;; that latest version of test.check yet. Consider updating to use 64 | ;; key :pass? instead of :result if core.rrb-vector updates to a 65 | ;; version of test.check that returns that key. 66 | ;; 67 | ;; When quick-check finds a failing test case, it still returns a map 68 | ;; that Clojure considers to be a logical true value, so the test will 69 | ;; still pass if you do `(is (tc/quick-check ...))`. 70 | 71 | (deftest test-reduce-subvec-catvec-generative 72 | (letfn [(insert-by-sub-catvec [v n] 73 | (fv/catvec (fv/subvec v 0 n) (dv/cvec ['x]) (fv/subvec v n))) 74 | (repeated-subvec-catvec [i] 75 | (reduce insert-by-sub-catvec (dv/cvec (range i)) (range i 0 -1)))] 76 | (is (:result (tc/quick-check 1000 77 | (prop/for-all [cnt (gen/fmap 78 | (comp inc #(mod % 60000)) 79 | gen/pos-int)] 80 | (= (repeated-subvec-catvec cnt) 81 | (interleave (range cnt) (repeat 'x))))))))) 82 | 83 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 84 | 85 | ;; This code was copied from the issue: 86 | ;; https://clojure.atlassian.net/projects/CRRBV/issues/CRRBV-13 87 | 88 | (defn assoc-in-bytevec [my-vector-of use-transient? n indices] 89 | (let [coll (into (my-vector-of :byte) (range n)) 90 | coll2 (reduce (fn [coll i] 91 | (if use-transient? 92 | (assoc! coll i -1) 93 | (assoc coll i -1))) 94 | (if use-transient? 95 | (transient coll) 96 | coll) 97 | indices)] 98 | (if use-transient? 99 | (persistent! coll2) 100 | coll2))) 101 | 102 | (defn assoc-in-bytevec-core [& args] 103 | (apply assoc-in-bytevec clojure.core/vector-of args)) 104 | 105 | (defn assoc-in-bytevec-rrbv [& args] 106 | (apply assoc-in-bytevec fv/vector-of args)) 107 | 108 | (deftest test-crrbv-13 109 | (doseq [use-transient? [false true]] 110 | (doseq [args [[10 [5]] 111 | [32 [0]] 112 | [32 [32]] 113 | [64 [32]] 114 | [64 [64]]]] 115 | (is (= (apply assoc-in-bytevec-core false args) 116 | (apply assoc-in-bytevec-rrbv use-transient? args)) 117 | (str "args=" (cons use-transient? args)))) 118 | (doseq [args [[64 [0]] 119 | [64 [1]] 120 | [64 [31]]]] 121 | (is (= (apply assoc-in-bytevec-core false args) 122 | (apply assoc-in-bytevec-rrbv use-transient? args)) 123 | (str "args=" (cons use-transient? args)))))) 124 | 125 | ;; Double check that the type of the mutable fields used to store the 126 | ;; cached hash values of collections are 32-bit int, not 64-bit long, 127 | ;; because 64-bit long do not have the Java Memory Model thread-safety 128 | ;; guarantees that 32-bit int values do. 129 | 130 | (defn member-data-by-name [klass field-name-as-symbol] 131 | (let [klass-dat (ref/type-reflect klass) 132 | members (:members klass-dat)] 133 | (first (filter (fn [x] (= field-name-as-symbol (:name x))) 134 | members)))) 135 | 136 | (deftest test-crrbv-26 137 | ;; For reasons I do not understand, the code below throws an 138 | ;; exception with Clojure 1.6.0 and earlier because it cannot find 139 | ;; the Vector and VecSeq classes. It seems to work fine on Clojure 140 | ;; 1.7.0 and later, and checking on those versions is enough for the 141 | ;; purposes of this test. 142 | (when (clj-version-at-least [1 7]) 143 | (let [vector-class (Class/forName "clojure.core.rrb_vector.rrbt.Vector") 144 | vecseq-class (Class/forName "clojure.core.rrb_vector.rrbt.VecSeq")] 145 | (is (= 'int (:type (member-data-by-name vector-class '_hash)))) 146 | (is (= 'int (:type (member-data-by-name vector-class '_hasheq)))) 147 | (is (= 'int (:type (member-data-by-name vecseq-class '_hash)))) 148 | (is (= 'int (:type (member-data-by-name vecseq-class '_hasheq))))))) 149 | -------------------------------------------------------------------------------- /doc/benchmarks/benchmarks.md: -------------------------------------------------------------------------------- 1 | # core.rrb-vector benchmarks 2 | 3 | See the section "How the benchmarks were run" below for how the 4 | results were created. 5 | 6 | 7 | ## Benchmark results 8 | 9 | These benchmark results are based upon benchmark code written for the 10 | [`bifurcan`](https://github.com/lacuna/bifurcan) library. The 11 | [benchmarks published originally 12 | here](https://github.com/lacuna/bifurcan/blob/master/doc/comparison.md) 13 | include comparisons of data structures other than vectors, e.g. hash 14 | maps and sorted sets. 15 | 16 | Note that for the data structures we care most about here, those 17 | results call them "lists" rather than "vectors". I will use "lists" 18 | here to be consistent with the `bifurcan` benchmarks. The portion of 19 | the `bifurcan` results relevant to lists can be found 20 | [here](https://github.com/lacuna/bifurcan/blob/master/doc/comparison.md#lists). 21 | 22 | The time measurements here were made on a different machine than the 23 | `bifurcan` results, so you should not infer anything from the absolute 24 | time measurements here vs. there. Only the relative time measurements 25 | between the libraries in one set of benchmark results. 26 | 27 | --- 28 | 29 | ![](images/list_construct.png) 30 | 31 | As in the original benchmark results, construction times are quite 32 | consistent (at the scale shown in this graph) across different 33 | libraries, except for vavr. The graph below shows the results for all 34 | libraries except vavr, so that any differences between them become 35 | more apparent. Even at that finer scale the maximum differences above 36 | 100 element vectors appears to be at most 3x. 37 | 38 | ![](images/list_construct_all_but_vavr.png) 39 | 40 | --- 41 | 42 | ![](images/list_iterate.png) 43 | 44 | Without core.rrb-vector included, Zach Tellman's original comment on 45 | these results was "The mutable collections, which are stored 46 | contiguously, are only moderately faster than their immutable 47 | counterparts". 48 | 49 | Note that the benchmark here creates a Java 50 | [`Iterator`](https://docs.oracle.com/javase/8/docs/api/java/util/Iterator.html) 51 | object from the collection, and then in a loop uses that interface's 52 | `hasNext` and `next` methods to traverse the entire collection until 53 | `hasNext` returns false. The more typical way to traverse collections 54 | in Clojure is to call `seq` on them (or use one of many functions that 55 | call `seq` for you under the covers), and then use `rest` or `next` to 56 | advance one step. 57 | 58 | core.rrb-vector version 0.1.0 is significantly slower, primarily 59 | because it has less optimized code for its Java `Iterator` 60 | implementation than it has for its `seq`/`rest`/`next` implementation. 61 | The reason for the increases near 1K elements and 32K elements are 62 | most likely because those are the sizes where the 32-way tree data 63 | structure gets one level deeper. 64 | 65 | The graph below shows the results for all except the core.rrb-vector 66 | library, so any differences between those libraries can be seen. 67 | Several of them show at least some change in run time when crossing 68 | the 1K and 32K element points. 69 | 70 | ![](images/list_iterate_all_but_core_rrb_vector.png) 71 | 72 | --- 73 | 74 | ![](images/list_lookup.png) 75 | 76 | Zach's comment on the original results, which are nearly identical to 77 | those shown above: "The mutable collections are O(1) while their 78 | immutable counterparts are unmistakably O(log N)." 79 | 80 | --- 81 | 82 | ![](images/concat.png) 83 | 84 | Zach's comment on the original results: "Concatenation is O(N) for 85 | every library except Paguro and Bifurcan, which are O(log N) due to 86 | their use of RRB trees." 87 | 88 | I know for a fact that clojure.PersistentVector does not use RRB 89 | trees, and concatenation takes at least linear time (multiplied by 90 | some small log factor) in the length of the second vector. This 91 | appears to be the case for vavr.Vector and java.ArrayList as well. 92 | 93 | I would have guessed that scala.Vector used RRB trees as well, but 94 | have not checked the implementation yet to verify. If it does use RRB 95 | trees, it is by far the slowest of the ones that do, at least for 96 | vectors 100K in size and larger -- perhaps the scala.Vector authors 97 | chose to implement some more extensive tree rebalancing than other RRB 98 | implementers did, in order to preserve faster run times for other 99 | operations? 100 | 101 | The next graph shows the results with only the 4 libraries that are 102 | the fastest for concatenation. Unlike the previous graphs, the 103 | vertical axis is the elapsed time, not "elapsed time per vector 104 | element". RRB trees should enable O(log N) run time for 105 | concatenation. 106 | 107 | core.rrb-vector is the slowest of these by a large factor, probably 108 | because of a concatenation implementation that has not been 109 | scrutinized for optimization opportunities yet. 110 | 111 | ![](images/concat_time_all_rrb.png) 112 | 113 | The next graph shows only the 3 libraries that are the fastest for 114 | concatenation, to see any detail that might be of interest there. 115 | Like the previous one, the vertical axis is elapsed time, not elapsed 116 | time per vector element. 117 | 118 | TBD: How can bifurcan.LinearList have pretty much a constant run time 119 | for all vector sizes? 120 | 121 | ![](images/concat_time_all_rrb_but_core_rrb_vector.png) 122 | 123 | --- 124 | 125 | 126 | ## How the benchmarks were run 127 | 128 | To run benchmarks from the Bifurcan project, with small modifications 129 | that add core.rrb-vector to the list of libraries that are measured, 130 | follow these steps. Note that the version of the `bifurcan.List` data 131 | structure code used in these results has a few proposed bug fixes from 132 | the version of the code in the original `bifurcan` repository, but I 133 | do not believe they affect the performance in any noticeable way. 134 | 135 | ```bash 136 | $ git clone https://github.com/jafingerhut/bifurcan 137 | $ cd bifurcan 138 | $ git checkout 457fd0346b78392f39e4c0e79f1e43b7847ea93b 139 | $ ./benchmarks/run-vectors-only.sh 140 | ``` 141 | 142 | To copy the data and images produced as a result of the above, to 143 | where I copied them in this repository: 144 | 145 | ```bash 146 | $ DST=/path/to/my/clone/of/core.rrb-vector/doc/benchmarks 147 | $ mkdir $DST/images $DST/data 148 | $ cp -p benchmarks/images/list*.png benchmarks/images/concat*.png $DST/images 149 | $ cp -p benchmarks/data/benchmarks.edn benchmarks/data/concat.csv benchmarks/data/list*.csv $DST/data 150 | ``` 151 | 152 | The benchmark results here were measured on a system with these 153 | properties: 154 | 155 | * MacBook Pro model 11,2, 2.5 GHz Intel Core i7 with peak clock speed 156 | 3.6 GHz, 16 GB RAM 157 | * macOS 10.14.6 158 | * AdoptOpenJDK 11.0.4, 64-bit server build 11.0.4+11 159 | * Leiningen 2.9.1 160 | * To see the versions of the list libraries that were measured, look 161 | in the project.clj file of the bifurcan project at the commit 162 | mentioned above. For core.rrb-vector, the only measured library 163 | that is written in Clojure, that project.clj file currently 164 | specifies Clojure version 1.8.0. The other libraries are written in 165 | Java. 166 | -------------------------------------------------------------------------------- /doc/rrb-tree-notes.md: -------------------------------------------------------------------------------- 1 | # Other implementations and descriptions of RRB trees 2 | 3 | Note that most implementations have an associated paper. If they have 4 | an author in common, then typically the paper or talk describes the 5 | implementation they published. 6 | 7 | Implementations: 8 | 9 | * TBD: Tiark Rompf and Phil Bagwell's original implementation code? 10 | * If it is available somewhere, likely it is instrumented for 11 | experimentation on multiple variations of the algorithms, and 12 | counting significant events that they were trying to optimize. 13 | Thus more of a research implementation than one intended for use 14 | in production. 15 | * [`scala-rrb-vector`](https://github.com/nicolasstucki/scala-rrb-vector) 16 | Scala library by Nicolas Stucki 17 | * As of Oct 2019, this library has a few unfixed bugs that were 18 | reported as Github issues in 2017. I have not examined them to 19 | see how easy they might be to fix. 20 | * [`bifurcan`](https://github.com/lacuna/bifurcan) Java library by 21 | Zach Tellman, Java class `io.lacuna.bifurcan.List` 22 | * [`Paguro`](https://github.com/GlenKPeterson/Paguro) Java library by 23 | Glen Peterson, Java class `org.organicdesign.fp.collections.RrbTree`. 24 | * From some comments in the code, it appears that perhaps this 25 | implementation is more precisely a data structure based upon 26 | B-trees, because those comments imply that nodes in the tree can 27 | have a number of children varying between some branching factor B, 28 | and be as low as B/2. 29 | * [`c-rrb`](https://github.com/hypirion/c-rrb) C library by Jean 30 | Niklas L'orange 31 | * [`Array`](https://github.com/xash/Array) RRB trees implemented in 32 | JavaScript for use in the Elm programming language. 33 | * [`immer`](https://sinusoid.es/immer) C++ library by Juan Pedro 34 | Bolivar Puente 35 | * [`Vector`](https://docs.rs/im/12.3.3/im/vector/enum.Vector.html) 36 | Rust implementation of RRB trees by Bodil Stokke 37 | * [Source code](https://docs.rs/crate/im/12.3.3/source/) 38 | * [`RRBVector`](https://github.com/rmunn/FSharpx.Collections/blob/rrb-vector/src/FSharpx.Collections.Experimental/RRBVector.fs) 39 | F# library by Robin Munn. 40 | * According to discussion on [this Github 41 | issue](https://github.com/fsprojects/FSharpx.Collections/issues/72) 42 | it appears to have known bugs the author would still like to find 43 | and fix as of 2019-Oct-10. 44 | 45 | 46 | Implementation of immutable vectors that are not RRB trees: 47 | 48 | * [`Clojure`](https://github.com/clojure/clojure) collections library, 49 | Java class `clojure.lang.PersistentVector` implemented in Java 50 | * Also class `clojure.core.Vector` implemented in Clojure, with 51 | memory/time optimizations achieved by restricting vector elements 52 | to all be the same type of Java primitive, e.g. all `long` vector 53 | elements, or all `double`. 54 | * [`Scala`](https://github.com/scala/scala) collection library, Java 55 | class `scala.collection.immutable.Vector` 56 | * In source file `src/library/scala/collection/immutable/Vector.scala` 57 | * As far as I can tell, as of 2019-Oct-10, it appears that this 58 | class does _not_ use RRB trees, and thus implements concatenation 59 | of vectors in linear time in the length of the second vector. 60 | [This Github 61 | issue](https://github.com/nicolasstucki/scala-rrb-vector/issues/9) 62 | from April 2019 implies that Scala has not yet had an RRB tree 63 | implementation incorporated into its standard library. 64 | 65 | 66 | Published papers and theses: 67 | 68 | * Phil Bagwell, Tiark Rompf, "RRB-Trees: Efficient Immutable Vectors", 69 | EPFL-REPORT-169879, September, 2011 70 | [[PDF]](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.592.5377&rep=rep1&type=pdf) 71 | [[SemanticScholar 72 | page]](https://www.semanticscholar.org/paper/RRB-Trees-%3A-Efficient-Immutable-Vectors-Phil-Tiark-Bagwell-Rompf/30c8c562f6421ab6b00d0b7faebd897c407de69c) 73 | * Phil Bagwell talk 74 | [[video]](https://www.youtube.com/watch?v=K2NYwP90bNs), "Striving 75 | to Make Things Simple and Fast", January 2013, given at Clojure 76 | conj conference 77 | * Jean Niklas L'orange, "Improving RRB-Tree Performance through 78 | Transience", Master Thesis, 2014, 79 | [[PDF]](https://hypirion.com/thesis.pdf) 80 | * "RRB Vector: A Practical General Purpose Immutable Sequence", 81 | Nicolas Stucki, Tiark, Rompf, Vlad Ureche, Phil Bagwell, Proc. of 82 | the 20th ACM SIGPLAN International Conference on Functional 83 | Programming, 2015 [[ACM digital library 84 | link]](http://dx.doi.org/10.1145/2784731.2784739) 85 | [[PDF]](https://github.com/nicolasstucki/scala-rrb-vector/blob/master/documents/RRB%20Vector%20-%20A%20Practical%20General%20Purpose%20Immutable%20Sequence.pdf) 86 | * Nicolas Stucki, "Turning Relaxed Radix Balanced Vector from Theory 87 | into Practice for Scala Collections", Master Thesis, 2015 88 | [[PDF]](https://github.com/nicolasstucki/scala-rrb-vector/blob/master/documents/Master%20Thesis%20-%20Nicolas%20Stucki%20-%20Turning%20Relaxed%20Radix%20Balanced%20Vector%20from%20Theory%20into%20Practice%20for%20Scala%20Collections.pdf?raw=true) 89 | * Juan Pedro Bolivar Puente, "Persistence for the Masses: RRB-Vectors 90 | in a Systems Language", Proc. ACM Program. Lang. 1, ICFP, Article 16 91 | (September 2017), https://doi.org/10.1145/3110260 92 | [[PDF]](https://public.sinusoid.es/misc/immer/immer-icfp17.pdf) 93 | * Juan's talk [[video]](https://www.youtube.com/watch?v=sPhpelUfu8Q) 94 | "Postmodern immutable data structures" given at CppCon 2017 95 | * Bodil Stokke talk 96 | [[video]](https://www.youtube.com/watch?v=cUx2b_FO8EQ) "Meetings With 97 | Remarkable Trees" given at ClojuTRE 2018 98 | 99 | 100 | Related things: 101 | 102 | * Jean Niklas L'orange's series of articles on Clojure's persistent 103 | vector data structure and how it works inside. These are good 104 | tutorial style articles. I have not found any similar articles like 105 | these on RRB trees. 106 | * ["Understanding Clojure's Persistent Vectors, Part 107 | 1"](https://hypirion.com/musings/understanding-persistent-vector-pt-1), 108 | September 2013 109 | * ["Understanding Clojure's Persistent Vectors, Part 110 | 2](https://hypirion.com/musings/understanding-persistent-vector-pt-2), 111 | October 2013 112 | * ["Understanding Clojure's Persistent Vectors, Part 113 | 3](https://hypirion.com/musings/understanding-persistent-vector-pt-3) 114 | April 2014 115 | * ["Understanding Clojure's 116 | Transients"](https://hypirion.com/musings/understanding-clojure-transients), 117 | October 2014 118 | * ["Persistent Vector 119 | Performance"](https://hypirion.com/musings/persistent-vector-performance), 120 | January 2015 121 | * ["Persistent Vector Performance 122 | Summarised"](https://hypirion.com/musings/persistent-vector-performance-summarised), 123 | February 2015 124 | * StackOverflow 125 | [question](https://stackoverflow.com/questions/14007153/what-invariant-do-rrb-trees-maintain) 126 | "What invariant do RRB-trees maintain?" 127 | 128 | 129 | Not RRB trees, but somewhat related ideas: 130 | 131 | * ["Theory and practice of chunked 132 | sequences"](http://www.andrew.cmu.edu/user/mrainey//chunkedseq/chunkedseq.html) 133 | web page has links to papers, talks, and Github repository 134 | containing C++ of their ideas. 135 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/trees.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.trees 10 | (:refer-clojure :exclude [array-for push-tail pop-tail new-path do-assoc]) 11 | (:require [clojure.core.rrb-vector.nodes 12 | :refer [regular? clone node-ranges last-range overflow?]])) 13 | 14 | (defn new-path [tail edit shift current-node] 15 | (if (== (alength tail) 32) 16 | (loop [s 0 n current-node] 17 | (if (== s shift) 18 | n 19 | (let [arr (make-array 32) 20 | ret (->VectorNode edit arr)] 21 | (aset arr 0 n) 22 | (recur (+ s 5) ret)))) 23 | (loop [s 0 n current-node] 24 | (if (== s shift) 25 | n 26 | (let [arr (make-array 33) 27 | rngs (make-array 33) 28 | ret (->VectorNode edit arr)] 29 | (aset arr 0 n) 30 | (aset arr 32 rngs) 31 | (aset rngs 32 1) 32 | (aset rngs 0 (alength tail)) 33 | (recur (+ s 5) ret)))))) 34 | 35 | (defn push-tail [shift cnt root-edit current-node tail-node] 36 | (if (regular? current-node) 37 | (let [arr (aclone (.-arr current-node)) 38 | ret (->VectorNode (.-edit current-node) arr)] 39 | (loop [n ret shift shift] 40 | (let [arr (.-arr n) 41 | subidx (bit-and (bit-shift-right (dec cnt) shift) 0x1f)] 42 | (if (== shift 5) 43 | (aset arr subidx tail-node) 44 | (if-let [child (aget arr subidx)] 45 | (let [new-carr (aclone (.-arr child)) 46 | new-child (->VectorNode root-edit new-carr)] 47 | (aset arr subidx new-child) 48 | (recur new-child (- shift 5))) 49 | (aset arr subidx 50 | (new-path (.-arr tail-node) 51 | root-edit 52 | (- shift 5) 53 | tail-node)))))) 54 | ret) 55 | (let [arr (aclone (.-arr current-node)) 56 | rngs (node-ranges current-node) 57 | li (dec (aget rngs 32)) 58 | ret (->VectorNode (.-edit current-node) arr) 59 | cret (if (== shift 5) 60 | nil 61 | (let [child (aget arr li) 62 | ccnt (+ (if (pos? li) 63 | (- (aget rngs li) (aget rngs (dec li))) 64 | (aget rngs 0)) 65 | ;; add 32 elems to account for the new 66 | ;; 32-elem tail we plan to add to the 67 | ;; subtree. 68 | 32)] 69 | ;; See Note 2 in file transients.cljs 70 | (if-not (overflow? child (- shift 5) ccnt) 71 | (push-tail (- shift 5) ccnt root-edit 72 | child 73 | tail-node))))] 74 | (if cret 75 | (do (aset arr li cret) 76 | (aset rngs li (+ (aget rngs li) 32)) 77 | ret) 78 | (do (when (>= li 31) 79 | ;; See Note 1 in file transients.cljs 80 | (let [msg (str "Assigning index " (inc li) " of vector" 81 | " object array to become a node, when that" 82 | " index should only be used for storing" 83 | " range arrays.") 84 | data {:shift shift, :cnt cnt, :current-node current-node, 85 | :tail-node tail-node, :rngs rngs, :li li, 86 | :cret cret}] 87 | (throw (ex-info msg data)))) 88 | (aset arr (inc li) 89 | (new-path (.-arr tail-node) 90 | root-edit 91 | (- shift 5) 92 | tail-node)) 93 | (aset rngs (inc li) (+ (aget rngs li) 32)) 94 | (aset rngs 32 (inc (aget rngs 32))) 95 | ret))))) 96 | 97 | (defn pop-tail [shift cnt root-edit current-node] 98 | (if (regular? current-node) 99 | (let [subidx (bit-and (bit-shift-right (- cnt 2) shift) 0x1f)] 100 | (cond 101 | (> shift 5) 102 | (let [new-child (pop-tail (- shift 5) cnt root-edit 103 | (aget (.-arr current-node) subidx))] 104 | (if (and (nil? new-child) (zero? subidx)) 105 | nil 106 | (let [arr (aclone (.-arr current-node))] 107 | (aset arr subidx new-child) 108 | (->VectorNode root-edit arr)))) 109 | 110 | (zero? subidx) 111 | nil 112 | 113 | :else 114 | (let [arr (aclone (.-arr current-node))] 115 | (aset arr subidx nil) 116 | (->VectorNode root-edit arr)))) 117 | (let [rngs (node-ranges current-node) 118 | subidx (dec (aget rngs 32)) 119 | new-rngs (aclone rngs)] 120 | (cond 121 | (> shift 5) 122 | (let [child (aget (.-arr current-node) subidx) 123 | child-cnt (if (zero? subidx) 124 | (aget rngs 0) 125 | (- (aget rngs subidx) (aget rngs (dec subidx)))) 126 | new-child (pop-tail (- shift 5) child-cnt root-edit child)] 127 | (cond 128 | (and (nil? new-child) (zero? subidx)) 129 | nil 130 | 131 | (regular? child) 132 | (let [arr (aclone (.-arr current-node))] 133 | (aset new-rngs subidx (- (aget new-rngs subidx) 32)) 134 | (aset arr subidx new-child) 135 | (aset arr 32 new-rngs) 136 | (if (nil? new-child) 137 | (aset new-rngs 32 (dec (aget new-rngs 32)))) 138 | (->VectorNode root-edit arr)) 139 | 140 | :else 141 | (let [rng (last-range child) 142 | diff (- rng (if new-child (last-range new-child) 0)) 143 | arr (aclone (.-arr current-node))] 144 | (aset new-rngs subidx (- (aget new-rngs subidx) diff)) 145 | (aset arr subidx new-child) 146 | (aset arr 32 new-rngs) 147 | (if (nil? new-child) 148 | (aset new-rngs 32 (dec (aget new-rngs 32)))) 149 | (->VectorNode root-edit arr)))) 150 | 151 | (zero? subidx) 152 | nil 153 | 154 | :else 155 | (let [arr (aclone (.-arr current-node)) 156 | child (aget arr subidx) 157 | new-rngs (aclone rngs)] 158 | (aset arr subidx nil) 159 | (aset arr 32 new-rngs) 160 | (aset new-rngs subidx 0) 161 | (aset new-rngs 32 (dec (aget new-rngs 32))) 162 | (->VectorNode root-edit arr)))))) 163 | 164 | (defn do-assoc [shift current-node i val] 165 | (if (regular? current-node) 166 | (let [node (clone shift current-node)] 167 | (loop [shift shift 168 | node node] 169 | (if (zero? shift) 170 | (let [arr (.-arr node)] 171 | (aset arr (bit-and i 0x1f) val)) 172 | (let [arr (.-arr node) 173 | subidx (bit-and (bit-shift-right i shift) 0x1f) 174 | child (clone shift (aget arr subidx))] 175 | (aset arr subidx child) 176 | (recur (- shift 5) child)))) 177 | node) 178 | (let [arr (aclone (.-arr current-node)) 179 | rngs (node-ranges current-node) 180 | subidx (bit-and (bit-shift-right i shift) 0x1f) 181 | subidx (loop [subidx subidx] 182 | (if (< i (int (aget rngs subidx))) 183 | subidx 184 | (recur (inc subidx)))) 185 | i (if (zero? subidx) i (- i (aget rngs (dec subidx))))] 186 | (aset arr subidx 187 | (do-assoc (- shift 5) (aget arr subidx) i val)) 188 | (->VectorNode (.-edit current-node) arr)))) 189 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | ;; See shell scripts './script/sdo' and './script/jdo' for sample 2 | ;; useful combinations of aliases to use to acommplish common tasks. 3 | 4 | ;; One way to specify a local version of ClojureScript, in case you 5 | ;; want to test with modifications to it: 6 | ;; org.clojure/clojurescript {:local/root "/Users/jafinger/clj/clojurescript"} 7 | 8 | {:paths ["src/main/clojure" "src/main/cljs" "src/main/cljc"] 9 | ;;:paths ["src/parameterized/clojure" "src/main/cljs" "src/main/cljc"] 10 | :aliases 11 | {;; Common alias to use for all Clojure/Java commands 12 | :clj {:jvm-opts ["-XX:-OmitStackTraceInFastThrow"]} 13 | ;; Common alias to use for all ClojureScript commands 14 | :cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.520"}} 15 | :jvm-opts ["-XX:-OmitStackTraceInFastThrow"]} 16 | 17 | ;; - start a Clojure/Java Socket REPL on port 50505 18 | :clj-socket {:jvm-opts ["-Dclojure.server.repl={:port,50505,:accept,clojure.core.server/repl}"]} 19 | ;; start a Node-based ClojureScript socket REPL on port 50505 20 | :cljs-socket {:jvm-opts ["-Dclojure.server.repl={:port,50505,:accept,cljs.server.node/repl}"]} 21 | 22 | ;; Common alias to use for all Clojure/Java commands that run tests 23 | :clj-test {:extra-paths ["src/test/clojure" "src/test/cljc"] 24 | :extra-deps {org.clojure/test.check {:mvn/version "0.7.0"}}} 25 | ;; Common alias to use for all ClojureScript commands that run tests 26 | :cljs-test {:extra-paths ["src/test/cljs" "src/test/cljc"] 27 | :extra-deps {org.clojure/test.check {:mvn/version "0.7.0"}}} 28 | 29 | ;; Run 'short' tests 30 | :clj-runt {:main-opts ["-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.test-clj-only,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-clj-only)"]} 31 | :cljs-runt {:main-opts ["-m" "cljs.main" 32 | "-re" "node" 33 | "-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.test-cljs-only,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-cljs-only)"]} 34 | 35 | ;; Run 'short' tests with extra-checks? enabled 36 | :clj-extrachecks-runt {:main-opts ["-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.test-clj-only,'clojure.core.rrb-vector.test-common),(alter-var-root,#'clojure.core.rrb-vector.test-utils/extra-checks?,(constantly,true)),(t/run-tests,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-clj-only)"]} 37 | :cljs-extrachecks-runt {:main-opts ["-m" "cljs.main" 38 | "-re" "node" 39 | "-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.test-cljs-only,'clojure.core.rrb-vector.test-common),(set!,clojure.core.rrb-vector.test-utils/extra-checks?,true),(t/run-tests,'clojure.core.rrb-vector.test-common),(t/run-tests,'clojure.core.rrb-vector.test-cljs-only)"]} 40 | 41 | ;; Run generative and/or 'long' tests 42 | :clj-runlongtests {:main-opts ["-e" 43 | "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.long-test),(t/run-tests,'clojure.core.rrb-vector.long-test)"]} 44 | :cljs-runlongtests {:main-opts ["-m" "cljs.main" 45 | "-re" "node" 46 | "-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector.long-test),(t/run-tests,'clojure.core.rrb-vector.long-test)"]} 47 | 48 | ;; Using collections-check requires this minimum version of 49 | ;; test.check, and at least Clojure 1.7.0 50 | :clj-check {:extra-paths ["src/test_local/clojure"] 51 | :extra-deps {collection-check/collection-check {:mvn/version "0.1.7"} 52 | com.gfredericks/test.chuck {:mvn/version "0.2.10"} 53 | org.clojure/test.check {:mvn/version "0.9.0"}}} 54 | :clj-runcheck {:main-opts ["-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector-check),(t/run-tests,'clojure.core.rrb-vector-check)"]} 55 | :cljs-check {:extra-paths ["src/test_local/clojure"] 56 | :extra-deps {collection-check/collection-check {:mvn/version "0.1.7"} 57 | com.gfredericks/test.chuck {:mvn/version "0.2.10"} 58 | org.clojure/test.check {:mvn/version "0.9.0"}}} 59 | :cljs-runcheck {:main-opts ["-m" "cljs.main" 60 | "-re" "node" 61 | "-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector-check),(t/run-tests,'clojure.core.rrb-vector-check)"]} 62 | 63 | ;; Run performance tests 64 | :clj-runperf {:main-opts ["-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector-performance-test),(t/run-tests,'clojure.core.rrb-vector-performance-test)"]} 65 | :cljs-runperf {:main-opts ["-m" "cljs.main" 66 | "-re" "node" 67 | "-e" "(require,'[clojure.test,:as,t],'clojure.core.rrb-vector-performance-test),(t/run-tests,'clojure.core.rrb-vector-performance-test)"]} 68 | 69 | ;; Run whatever the current 'focus' tests are 70 | :clj-runfocus {:main-opts ["-e" "(require,'[clojure.test,:as,t],'[clojure.core.rrb-vector.test-common,:as,ct]),(ct/test-reduce-subvec-catvec2)"]} 71 | :cljs-runfocus {:main-opts ["-m" "cljs.main" 72 | "-re" "node" 73 | "-e" "(require,'[clojure.test,:as,t],'[clojure.core.rrb-vector.test-common,:as,ct]),(ct/test-reduce-subvec-catvec2)"]} 74 | 75 | ;; I have tried using cljs-test-runner for running clojure.test 76 | ;; tests in a modified version of core.rrb-vector, but my guess is 77 | ;; that since an older version of core.rrb-vector (version 0.0.11) 78 | ;; is in the transitive dependencies of the cljs-test-runner project 79 | ;; itself, that version conflicts with the version I am attempting 80 | ;; to test. See 81 | ;; https://github.com/Olical/cljs-test-runner/issues/34 82 | ;; :cljs-runner {:extra-deps {olical/cljs-test-runner {:mvn/version "3.7.0"}} 83 | ;; :main-opts ["-m" "cljs-test-runner.main" 84 | ;; "-d" "src/test/cljs"]} 85 | ;; :cljol {:extra-deps {cljol {:local/root "/Users/andy/clj/cljol"} 86 | ;; org.clojure/clojure {:mvn/version "1.7.0"}}} 87 | :cljol {:extra-deps {cljol/cljol {:git/url "https://github.com/jafingerhut/cljol" 88 | :sha "bb5549e9832e73e4a9fc5dfdf695c48e797729fa"}}} 89 | :cap {;; recommended options from README of 90 | ;; https://github.com/clojure-goes-fast/clj-async-profiler 91 | :jvm-opts ["-Djdk.attach.allowAttachSelf" 92 | ;; I have trouble entering password for this from 93 | ;; clj REPL. Maybe clojure command instead of clj 94 | ;; is better for this? 95 | "-Djol.tryWithSudo=true" 96 | "-XX:+UnlockDiagnosticVMOptions" 97 | "-XX:+DebugNonSafepoints"] 98 | :extra-deps {com.clojure-goes-fast/clj-async-profiler 99 | {:mvn/version "0.4.0"}}} 100 | :nodis {:extra-deps {com.clojure-goes-fast/clj-java-decompiler 101 | {:mvn/version "0.2.1"}}} 102 | :eastwood {:extra-deps {jonase/eastwood {:mvn/version "0.3.5"}} 103 | :main-opts ["-m" "eastwood.lint" 104 | "{:source-paths,[\"src/main/clojure\"],:test-paths,[\"src/test/clojure\",\"src/test/cljs\",\"src/test_local/clojure\"],:add-linters,[:unused-fn-args,:unused-locals,:unused-namespaces,:unused-private-vars],:exclude-linters,[:implicit-dependencies],:exclude-namespaces,[]}"]} 105 | :clj-kondo {:extra-deps {clj-kondo/clj-kondo {:mvn/version "RELEASE"}} 106 | :main-opts ["-m" "clj-kondo.main"]} 107 | 108 | ;; pull in specific versions of Clojure: 109 | :1.5 {:override-deps {org.clojure/clojure {:mvn/version "1.5.1"}}} 110 | :1.6 {:override-deps {org.clojure/clojure {:mvn/version "1.6.0"}}} 111 | :1.7 {:override-deps {org.clojure/clojure {:mvn/version "1.7.0"}}} 112 | :1.8 {:override-deps {org.clojure/clojure {:mvn/version "1.8.0"}}} 113 | :1.9 {:override-deps {org.clojure/clojure {:mvn/version "1.9.0"}}} 114 | :1.10.0 {:override-deps {org.clojure/clojure {:mvn/version "1.10.0"}}} 115 | :1.10 {:override-deps {org.clojure/clojure {:mvn/version "1.10.1"}}} 116 | :master {:override-deps {org.clojure/clojure {:mvn/version "1.11.0-master-SNAPSHOT"}}}}} 117 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/transients.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.transients 10 | (:refer-clojure :exclude [new-path]) 11 | (:require [clojure.core.rrb-vector.nodes 12 | :refer [regular? clone node-ranges last-range overflow?]] 13 | [clojure.core.rrb-vector.trees :refer [new-path]])) 14 | 15 | (defn ensure-editable [edit node] 16 | (if (identical? (.-edit node) edit) 17 | node 18 | (let [new-arr (aclone (.-arr node))] 19 | (if (== 33 (alength new-arr)) 20 | (aset new-arr 32 (aclone (aget new-arr 32)))) 21 | (VectorNode. edit new-arr)))) 22 | 23 | (defn editable-root [root] 24 | (let [new-arr (aclone (.-arr root))] 25 | (if (== 33 (alength new-arr)) 26 | (aset new-arr 32 (aclone (aget new-arr 32)))) 27 | (VectorNode. (js-obj) new-arr))) 28 | 29 | (defn editable-tail [tail] 30 | (let [ret (make-array 32)] 31 | (array-copy tail 0 ret 0 (alength tail)) 32 | ret)) 33 | 34 | ;; Note 1: This condition check and exception are a little bit closer 35 | ;; to the source of the cause for what was issue CRRBV-20, added in 36 | ;; case there is still some remaining way to cause this condition to 37 | ;; occur. 38 | 39 | ;; Note 2: In the worst case, when the tree is nearly full, calling 40 | ;; overflow? here takes run time O(tree_depth^2) here. That could be 41 | ;; made O(tree_depth). One way would be to call push-tail! in hopes 42 | ;; that it succeeds, but return some distinctive value indicating a 43 | ;; failure on the full condition, and create the node via a new-path 44 | ;; call at most recent recursive push-tail! call that has an empty 45 | ;; slot available. 46 | (defn push-tail! [shift cnt root-edit current-node tail-node] 47 | (let [ret (ensure-editable root-edit current-node)] 48 | (if (regular? ret) 49 | (do (loop [n ret shift shift] 50 | (let [arr (.-arr n) 51 | subidx (bit-and (bit-shift-right (dec cnt) shift) 0x1f)] 52 | (if (== shift 5) 53 | (aset arr subidx tail-node) 54 | (let [child (aget arr subidx)] 55 | (if (nil? child) 56 | (aset arr subidx 57 | (new-path (.-arr tail-node) 58 | root-edit 59 | (- shift 5) 60 | tail-node)) 61 | (let [editable-child (ensure-editable root-edit child)] 62 | (aset arr subidx editable-child) 63 | (recur editable-child (- shift 5)))))))) 64 | ret) 65 | (let [arr (.-arr ret) 66 | rngs (node-ranges ret) 67 | li (dec (aget rngs 32)) 68 | cret (if (== shift 5) 69 | nil 70 | (let [child (ensure-editable root-edit (aget arr li)) 71 | ccnt (+ (if (pos? li) 72 | (- (aget rngs li) (aget rngs (dec li))) 73 | (aget rngs 0)) 74 | ;; add 32 elems to account for the 75 | ;; new full tail we plan to add to 76 | ;; the subtree. 77 | 32)] 78 | ;; See Note 2 79 | (if-not (overflow? child (- shift 5) ccnt) 80 | (push-tail! (- shift 5) ccnt root-edit 81 | child 82 | tail-node))))] 83 | (if cret 84 | (do (aset arr li cret) 85 | (aset rngs li (+ (aget rngs li) 32)) 86 | ret) 87 | (do (when (>= li 31) 88 | ;; See Note 1 89 | (let [msg (str "Assigning index " (inc li) " of vector" 90 | " object array to become a node, when that" 91 | " index should only be used for storing" 92 | " range arrays.") 93 | data {:shift shift, :cnd cnt, 94 | :current-node current-node, 95 | :tail-node tail-node, :rngs rngs, :li li, 96 | :cret cret}] 97 | (throw (ex-info msg data)))) 98 | (aset arr (inc li) 99 | (new-path (.-arr tail-node) 100 | root-edit 101 | (- shift 5) 102 | tail-node)) 103 | (aset rngs (inc li) (+ (aget rngs li) 32)) 104 | (aset rngs 32 (inc (aget rngs 32))) 105 | ret)))))) 106 | 107 | (defn pop-tail! [shift cnt root-edit current-node] 108 | (let [ret (ensure-editable root-edit current-node)] 109 | (if (regular? ret) 110 | (let [subidx (bit-and (bit-shift-right (- cnt 2) shift) 0x1f)] 111 | (cond 112 | (> shift 5) 113 | (let [child (pop-tail! (- shift 5) cnt root-edit 114 | (aget (.-arr ret) subidx))] 115 | (if (and (nil? child) (zero? subidx)) 116 | nil 117 | (let [arr (.-arr ret)] 118 | (aset arr subidx child) 119 | ret))) 120 | 121 | (zero? subidx) 122 | nil 123 | 124 | :else 125 | (let [arr (.-arr ret)] 126 | (aset arr subidx nil) 127 | ret))) 128 | (let [rngs (node-ranges ret) 129 | subidx (dec (aget rngs 32))] 130 | (cond 131 | (> shift 5) 132 | (let [child (aget (.-arr ret) subidx) 133 | child-cnt (if (zero? subidx) 134 | (aget rngs 0) 135 | (- (aget rngs subidx) (aget rngs (dec subidx)))) 136 | new-child (pop-tail! (- shift 5) child-cnt root-edit child)] 137 | (cond 138 | (and (nil? new-child) (zero? subidx)) 139 | nil 140 | 141 | (regular? child) 142 | (let [arr (.-arr ret)] 143 | (aset rngs subidx (- (aget rngs subidx) 32)) 144 | (aset arr subidx new-child) 145 | (if (nil? new-child) 146 | (aset rngs 32 (dec (aget rngs 32)))) 147 | ret) 148 | 149 | :else 150 | (let [rng (last-range child) 151 | diff (- rng (if new-child (last-range new-child) 0)) 152 | arr (.-arr ret)] 153 | (aset rngs subidx (- (aget rngs subidx) diff)) 154 | (aset arr subidx new-child) 155 | (if (nil? new-child) 156 | (aset rngs 32 (dec (aget rngs 32)))) 157 | ret))) 158 | 159 | (zero? subidx) 160 | nil 161 | 162 | :else 163 | (let [arr (.-arr ret) 164 | child (aget arr subidx)] 165 | (aset arr subidx nil) 166 | (aset rngs subidx 0) 167 | (aset rngs 32 (dec (aget rngs 32))) 168 | ret)))))) 169 | 170 | (defn do-assoc! [shift root-edit current-node i val] 171 | (let [ret (ensure-editable root-edit current-node)] 172 | (if (regular? ret) 173 | (loop [shift shift 174 | node ret] 175 | (if (zero? shift) 176 | (let [arr (.-arr node)] 177 | (aset arr (bit-and i 0x1f) val)) 178 | (let [arr (.-arr node) 179 | subidx (bit-and (bit-shift-right i shift) 0x1f) 180 | child (ensure-editable root-edit (aget arr subidx))] 181 | (aset arr subidx child) 182 | (recur (- shift 5) child)))) 183 | (let [arr (.-arr ret) 184 | rngs (node-ranges ret) 185 | subidx (bit-and (bit-shift-right i shift) 0x1f) 186 | subidx (loop [subidx subidx] 187 | (if (< i (int (aget rngs subidx))) 188 | subidx 189 | (recur (inc subidx)))) 190 | i (if (zero? subidx) i (- i (aget rngs (dec subidx))))] 191 | (aset arr subidx 192 | (do-assoc! (- shift 5) root-edit (aget arr subidx) i val)))) 193 | ret)) 194 | -------------------------------------------------------------------------------- /doc/hash-details.md: -------------------------------------------------------------------------------- 1 | # Background on Clojure collection hash calculation 2 | 3 | The persistent collections included with Clojure are immutable when 4 | accessed via Clojure's published methods, e.g. `conj`, `assoc`, 5 | `peek`, `seq`, etc. 6 | 7 | Their implementation actually uses mutable fields to store cached 8 | versions of their Java `.hashCode` and `clojure.core/hash` values 9 | (those values are different from each other for most collection 10 | values starting with Clojure 1.6.0). 11 | 12 | On the JVM, all fields of a newly constructed object are first 13 | initialized to their default JVM initial values, e.g. 0 for a 14 | primitive `int` field, `null` for all references, etc. Then any 15 | values assigned in the constructor are assigned. If the field is 16 | declared `final`, then as long as the reference to the object is not 17 | made visible to any other object before the constructor finishes 18 | executing, any thread that later sees the object should see only the 19 | value assigned while the constructor executed, not the default JVM 20 | initial value. This is promised by the Java Memory Model and the 21 | specialness of the `final` field modifier. 22 | 23 | However, the cached hash fields in Clojure, and some other Java 24 | objects, is intentionally stored in non-final fields, so that if no 25 | other code ever needs to know the hash of the value, no time is ever 26 | spent calculating it. This is a performance optimization. 27 | 28 | If some other code does later want the hash value, then it is 29 | calculated on demand at that time, and the calculated value is written 30 | into the field, so that later calls to get the hash value by the same 31 | thread are guaranteed to avoid calculating it again. If another 32 | thread calls the function/method to get the hash value, because the 33 | hash field has no special Java modifiers like `final` or `volatile`, 34 | the Java Memory Model says that it might get the updated value, or it 35 | might get the value from an older write to that field, e.g. the 36 | initial value of 0 from the JVM default initialization of all fields 37 | -- _even if_ the constructor assigned a non-0 value to the field. 38 | 39 | Thus for thread safety of getting hash values of immutable values 40 | using this "initialize a default value, and calculate on demand 41 | later", if the field where the cached value is stored has no special 42 | modifiers like `final` or `volatile`, and the function to get the hash 43 | value is not declared `synchronized` (none of which are true for the 44 | Clojure implementation), the only safe value to assign in the 45 | constructor is none at all (leaving the field as the default JVM 46 | initial value of 0), or to assign a value of 0 explicitly. If any 47 | other value is assigned during the constructor, e.g. -1, other threads 48 | calling the hash function might read a 0 from that field, or -1, 49 | depending upon all kinds of factors that are impossible to control or 50 | observe from a Java program. 51 | 52 | Before Clojure 1.9.0, every collection did assign a value of -1 to 53 | these fields during the constructor call, which was unsafe. This was 54 | fixed in the Clojure 1.9.0 release. See this JIRA ticket for more 55 | details: https://clojure.atlassian.net/browse/CLJ-2091 56 | 57 | It links to this article on this pattern of writing Java code that has 58 | intentional data races, but is still correct according to the Java 59 | Memory Model: 60 | http://jeremymanson.blogspot.com/2008/12/benign-data-races-in-java.html 61 | 62 | Note that JavaScript run time environments are single threaded (not 63 | counting WebWorkers, but as far as I know, no ClojureScript objects 64 | are shared between the main thread and any WebWorker threads via 65 | shared memory), so these issues do not arise, and any initial value 66 | can be stored in the hash field without a problem. 67 | 68 | 69 | # Java `.hashCode` vs. `clojure.core/hash` 70 | 71 | Some background and history on why `.hashCode` and `clojure.core/hash` 72 | return different values from each other can be found in the Clojure 73 | equality guide, especially the ["Equality and hash" 74 | section](https://clojure.org/guides/equality#equality_and_hash). 75 | 76 | 77 | # Details in core.rrb-vector Clojure implementation 78 | 79 | `core.rrb-vector`'s Clojure implementation should use an initial value 80 | of the mutable hash fields of 0, for the same reasons described above 81 | that any such fields should be initialized to 0 on the JVM. 82 | 83 | In `core.rrb-vector` release 0.0.14 and earlier, these fields were 84 | incorrectly initialized to -1. Again, this does not cause any 85 | problems in a single-threaded program, and would only cause problems 86 | in some timing-dependent cases (perhaps rarely, but there is no 87 | guarantee of this) in a multi-threaded program. 88 | 89 | This is a list of classes in `core.rrb-vector`'s Clojure 90 | implementation that were changed to correct this problem, after the 91 | release of version 0.0.14: 92 | 93 | * `VecSeq` - last field is Clojure _hasheq, second to last is Java _hash 94 | * `Vector` - same as `VecSeq` 95 | * `Transient` - no hash fields. 96 | 97 | I believe that calling `clojure.core/hash` on a transient collection 98 | falls through to some default hash implementation that is based on the 99 | identity of the mutable object, and is not the same for all transients 100 | with "the same" contents, the way it is for immutable collections. 101 | 102 | 103 | ## `VecSeq` methods and constructor calls 104 | 105 | The constructor to `VecSeq` in method `withMeta` actually appears safe 106 | to me to initialize the returned object with the same values as the 107 | original collection. The hash of the returned collection is 108 | guaranteed to be the same as that of the collection on which 109 | `withMeta` was called, so the initial values will either be 0, or the 110 | correct final hash value. 111 | 112 | All occurrences of the string VecSeq in the implementation are in the 113 | one source file rrbt.clj, so I feel safe in saying I have corrected 114 | all constructor calls to VecSeq. 115 | 116 | I also corrected an unsafe data race in the implementation of method 117 | `hasheq` for class `VecSeq`. It was reading the field `_hasheq` twice 118 | per call, instead of only once. My fixed version reads that field at 119 | most once. See this article for the details of the changes I made and 120 | why: 121 | http://jeremymanson.blogspot.com/2008/12/benign-data-races-in-java.html 122 | 123 | 124 | ## `Vector` methods and constructor calls 125 | 126 | The constructor to `Vector` in method `withMeta` actually appears safe 127 | to me, for the same reason as it is for the same method in class 128 | `VecSeq` described in the previous section. 129 | 130 | Files containing constructor calls for class `Vector`: 131 | 132 | * rrbt.clj - many, fixed 133 | * rrb_vector.clj - fixed 134 | 135 | Files checked for occurrences of `-1`: 136 | 137 | * rrb_vector.clj - none remaining after constructor calls to `Vector` 138 | were fixed. 139 | * debug.clj - only one, used to return a "not found" value from 140 | `first-diff` 141 | * fork_join.clj - none, and no constructor calls 142 | * interop.clj.clj - none, and no constructor calls 143 | * nodes.clj - none, and no constructor calls to `Vector` or `VecSeq`. 144 | Many to `VecNode`, but it has no hash fields. 145 | * protocols.clj - none, and no constructor calls 146 | * rrbt.clj 147 | * Many occurrences of `Vector.` constructor calls were updated to 148 | use initial hash values of 0 instead of -1. 149 | * Fixed racy hash function/method bugs in these methods, for the 150 | same reason as described in previous section. 151 | * Vector/hashCode 152 | * Vector/hasheq 153 | * VecSeq/hasheq 154 | * transients.clj - none, and no mentions of `Vector` or `Vecseq` or 155 | hash 156 | 157 | 158 | # Details in core.rrb-vector ClojureScript implementation 159 | 160 | Since there are no thread-safety issues here, the only thing to double 161 | check is that the same value (e.g. 0, -1, or whatever constant value) 162 | is used consistently in all places where a new object is created that 163 | contains a cached hash field, and wherever the hash function is 164 | calculated. 165 | 166 | Classes in the ClojureScript `core.rrb-vector` implementation with a 167 | hash field: 168 | 169 | * `RRBChunkedSeq` - the last field of the constructor call, named `__hash` 170 | * `Vector` - the last field of the constructor call, named `__hash` 171 | 172 | All constructor calls for `RRBChunkedSeq` are in file rrbt.cljs, and 173 | all use `nil` as the initial value for the field `__hash`. Its 174 | `-hash` method calls `caching-hash` provided by the ClojureScript core 175 | code, which uses `nil` as the "hash not calculated yet" value. 176 | 177 | All constructor calls for `Vector` are in a few files, and all use 178 | `nil` as the initial value for the field `__hash`. Its `-hash` method 179 | also uses `caching-hash`, as described in previous paragraph. 180 | -------------------------------------------------------------------------------- /src/main/cljs/clojure/core/rrb_vector/nodes.cljs: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.nodes 10 | (:refer-clojure :exclude [clone])) 11 | 12 | ;;; node ops 13 | 14 | (def empty-node cljs.core.PersistentVector.EMPTY_NODE) 15 | 16 | (defn clone [shift node] 17 | (VectorNode. (.-edit node) (aclone (.-arr node)))) 18 | 19 | (defn regular? [node] 20 | (not (== (alength (.-arr node)) 33))) 21 | 22 | ;;; ranges 23 | 24 | (defn node-ranges [node] 25 | (aget (.-arr node) 32)) 26 | 27 | (defn last-range [node] 28 | (let [rngs (node-ranges node) 29 | i (dec (aget rngs 32))] 30 | (aget rngs i))) 31 | 32 | (defn regular-ranges [shift cnt] 33 | (let [step (bit-shift-left 1 shift) 34 | rngs (make-array 33)] 35 | (loop [i 0 r step] 36 | (if (< r cnt) 37 | (do (aset rngs i r) 38 | (recur (inc i) (+ r step))) 39 | (do (aset rngs i cnt) 40 | (aset rngs 32 (inc i)) 41 | rngs))))) 42 | 43 | ;;; root overflow 44 | 45 | (defn overflow? [root shift cnt] 46 | (if (regular? root) 47 | (> (bit-shift-right cnt 5) 48 | (bit-shift-left 1 shift)) 49 | (let [rngs (node-ranges root) 50 | slc (aget rngs 32)] 51 | (and (== slc 32) 52 | (or (== shift 5) 53 | (recur (aget (.-arr root) (dec slc)) 54 | (- shift 5) 55 | (+ (- (aget rngs 31) (aget rngs 30)) 32))))))) 56 | 57 | ;;; find nil / 0 58 | 59 | (defn index-of-0 [arr] 60 | (loop [l 0 h 31] 61 | (if (>= l (dec h)) 62 | (if (zero? (int (aget arr l))) 63 | l 64 | (if (zero? (int (aget arr h))) 65 | h 66 | 32)) 67 | (let [mid (+ l (bit-shift-right (- h l) 1))] 68 | (if (zero? (int (aget arr mid))) 69 | (recur l mid) 70 | (recur (inc mid) h)))))) 71 | 72 | (defn index-of-nil ^long [arr] 73 | (loop [l 0 h 31] 74 | (if (>= l (dec h)) 75 | (if (nil? (aget arr l)) 76 | l 77 | (if (nil? (aget arr h)) 78 | h 79 | 32)) 80 | (let [mid (+ l (bit-shift-right (- h l) 1))] 81 | (if (nil? (aget arr mid)) 82 | (recur l mid) 83 | (recur (inc mid) h)))))) 84 | 85 | ;;; children 86 | 87 | (defn first-child [node] 88 | (aget (.-arr node) 0)) 89 | 90 | (defn last-child [node] 91 | (let [arr (.-arr node)] 92 | (if (regular? node) 93 | (aget arr (dec (index-of-nil arr))) 94 | (aget arr (dec (aget (node-ranges node) 32)))))) 95 | 96 | (defn remove-leftmost-child [shift parent] 97 | (let [arr (.-arr parent)] 98 | (if (nil? (aget arr 1)) 99 | nil 100 | (let [r? (regular? parent) 101 | new-arr (make-array (if r? 32 33))] 102 | (array-copy arr 1 new-arr 0 31) 103 | (if-not r? 104 | (let [rngs (node-ranges parent) 105 | rng0 (aget rngs 0) 106 | new-rngs (make-array 33) 107 | lim (aget rngs 32)] 108 | (array-copy rngs 1 new-rngs 0 (dec lim)) 109 | (loop [i 0] 110 | (when (< i lim) 111 | (aset new-rngs i (- (aget new-rngs i) rng0)) 112 | (recur (inc i)))) 113 | (aset new-rngs 32 (dec (aget rngs 32))) 114 | (aset new-rngs (dec (aget rngs 32)) 0) 115 | (aset new-arr 32 new-rngs))) 116 | (->VectorNode (.-edit parent) new-arr))))) 117 | 118 | (defn replace-leftmost-child [shift parent pcnt child d] 119 | (if (regular? parent) 120 | (let [step (bit-shift-left 1 shift) 121 | rng0 (- step d) 122 | ncnt (- pcnt d) 123 | li (bit-and (bit-shift-right shift (dec pcnt)) 0x1f) 124 | arr (.-arr parent) 125 | new-arr (make-array 33) 126 | new-rngs (make-array 33)] 127 | (aset new-arr 0 child) 128 | (array-copy arr 1 new-arr 1 li) 129 | (aset new-arr 32 new-rngs) 130 | (aset new-rngs 0 rng0) 131 | (aset new-rngs li ncnt) 132 | (aset new-rngs 32 (inc li)) 133 | (loop [i 1] 134 | (when (<= i li) 135 | (aset new-rngs i (+ (aget new-rngs (dec i)) step)) 136 | (recur (inc i)))) 137 | (->VectorNode nil new-arr)) 138 | (let [new-arr (aclone (.-arr parent)) 139 | rngs (node-ranges parent) 140 | new-rngs (make-array 33) 141 | li (dec (aget rngs 32))] 142 | (aset new-rngs 32 (aget rngs 32)) 143 | (aset new-arr 32 new-rngs) 144 | (aset new-arr 0 child) 145 | (loop [i 0] 146 | (when (<= i li) 147 | (aset new-rngs i (- (aget rngs i) d)) 148 | (recur (inc i)))) 149 | (->VectorNode nil new-arr)))) 150 | 151 | (defn replace-rightmost-child [shift parent child d] 152 | (if (regular? parent) 153 | (let [arr (.-arr parent) 154 | i (dec (index-of-nil arr))] 155 | (if (regular? child) 156 | (let [new-arr (aclone arr)] 157 | (aset new-arr i child) 158 | (->VectorNode nil new-arr)) 159 | (let [arr (.-arr parent) 160 | new-arr (make-array 33) 161 | step (bit-shift-left 1 shift) 162 | rngs (make-array 33)] 163 | (aset rngs 32 (inc i)) 164 | (aset new-arr 32 rngs) 165 | (array-copy arr 0 new-arr 0 i) 166 | (aset new-arr i child) 167 | (loop [j 0 r step] 168 | (when (<= j i) 169 | (aset rngs j r) 170 | (recur (inc j) (+ r step)))) 171 | (aset rngs i (last-range child)) 172 | (->VectorNode nil new-arr)))) 173 | (let [rngs (node-ranges parent) 174 | new-rngs (aclone rngs) 175 | i (dec (aget rngs 32)) 176 | new-arr (aclone (.-arr parent))] 177 | (aset new-arr i child) 178 | (aset new-arr 32 new-rngs) 179 | (aset new-rngs i (+ (aget rngs i) d)) 180 | (->VectorNode nil new-arr)))) 181 | 182 | ;;; fold-tail 183 | 184 | (defn new-path* [shift node] 185 | (let [reg? (== 32 (alength (.-arr node))) 186 | len (if reg? 32 33) 187 | arr (make-array len) 188 | rngs (if-not reg? 189 | (doto (make-array 33) 190 | (aset 0 (alength (.-arr node))) 191 | (aset 32 1))) 192 | ret (->VectorNode nil arr)] 193 | (loop [arr arr shift shift] 194 | (if (== shift 5) 195 | (do (if-not reg? 196 | (aset arr 32 rngs)) 197 | (aset arr 0 node)) 198 | (let [a (make-array len) 199 | e (->VectorNode nil a)] 200 | (aset arr 0 e) 201 | (if-not reg? 202 | (aset arr 32 rngs)) 203 | (recur a (- shift 5))))) 204 | ret)) 205 | 206 | (defn fold-tail [node shift cnt tail] 207 | (let [tlen (alength tail) 208 | reg? (and (regular? node) (== tlen 32)) 209 | arr (.-arr node) 210 | li (index-of-nil arr) 211 | new-arr (make-array (if reg? 32 33)) 212 | rngs (if-not (regular? node) (node-ranges node)) 213 | cret (if (== shift 5) 214 | (->VectorNode nil tail) 215 | (fold-tail (aget arr (dec li)) 216 | (- shift 5) 217 | (if (regular? node) 218 | (mod cnt (bit-shift-left 1 shift)) 219 | (let [li (dec (aget rngs 32))] 220 | (if (pos? li) 221 | (- (aget rngs li) (aget rngs (dec li))) 222 | (aget rngs 0)))) 223 | tail)) 224 | new-rngs (if-not reg? 225 | (if rngs 226 | (aclone rngs) 227 | (regular-ranges shift cnt)))] 228 | (when-not (and (or (nil? cret) (== shift 5)) (== li 32)) 229 | (array-copy arr 0 new-arr 0 li) 230 | (when-not reg? 231 | (if (or (nil? cret) (== shift 5)) 232 | (do (aset new-rngs li 233 | (+ (if (pos? li) 234 | (aget new-rngs (dec li)) 235 | (int 0)) 236 | tlen)) 237 | (aset new-rngs 32 (inc li))) 238 | (do (when (pos? li) 239 | (aset new-rngs (dec li) 240 | (+ (aget new-rngs (dec li)) tlen))) 241 | (aset new-rngs 32 li)))) 242 | (if-not reg? 243 | (aset new-arr 32 new-rngs)) 244 | (if (nil? cret) 245 | (aset new-arr li (new-path* (- shift 5) (->VectorNode nil tail))) 246 | (aset new-arr (if (== shift 5) li (dec li)) cret)) 247 | (->VectorNode nil new-arr)))) 248 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 1.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 4 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 5 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial code and documentation 12 | distributed under this Agreement, and 13 | b) in the case of each subsequent Contributor: 14 | i) changes to the Program, and 15 | ii) additions to the Program; 16 | 17 | where such changes and/or additions to the Program originate from and are 18 | distributed by that particular Contributor. A Contribution 'originates' 19 | from a Contributor if it was added to the Program by such Contributor 20 | itself or anyone acting on such Contributor's behalf. Contributions do not 21 | include additions to the Program which: (i) are separate modules of 22 | software distributed in conjunction with the Program under their own 23 | license agreement, and (ii) are not derivative works of the Program. 24 | 25 | "Contributor" means any person or entity that distributes the Program. 26 | 27 | "Licensed Patents" mean patent claims licensable by a Contributor which are 28 | necessarily infringed by the use or sale of its Contribution alone or when 29 | combined with the Program. 30 | 31 | "Program" means the Contributions distributed in accordance with this 32 | Agreement. 33 | 34 | "Recipient" means anyone who receives the Program under this Agreement, 35 | including all Contributors. 36 | 37 | 2. GRANT OF RIGHTS 38 | a) Subject to the terms of this Agreement, each Contributor hereby grants 39 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 40 | reproduce, prepare derivative works of, publicly display, publicly 41 | perform, distribute and sublicense the Contribution of such Contributor, 42 | if any, and such derivative works, in source code and object code form. 43 | b) Subject to the terms of this Agreement, each Contributor hereby grants 44 | Recipient a non-exclusive, worldwide, royalty-free patent license under 45 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 46 | transfer the Contribution of such Contributor, if any, in source code and 47 | object code form. This patent license shall apply to the combination of 48 | the Contribution and the Program if, at the time the Contribution is 49 | added by the Contributor, such addition of the Contribution causes such 50 | combination to be covered by the Licensed Patents. The patent license 51 | shall not apply to any other combinations which include the Contribution. 52 | No hardware per se is licensed hereunder. 53 | c) Recipient understands that although each Contributor grants the licenses 54 | to its Contributions set forth herein, no assurances are provided by any 55 | Contributor that the Program does not infringe the patent or other 56 | intellectual property rights of any other entity. Each Contributor 57 | disclaims any liability to Recipient for claims brought by any other 58 | entity based on infringement of intellectual property rights or 59 | otherwise. As a condition to exercising the rights and licenses granted 60 | hereunder, each Recipient hereby assumes sole responsibility to secure 61 | any other intellectual property rights needed, if any. For example, if a 62 | third party patent license is required to allow Recipient to distribute 63 | the Program, it is Recipient's responsibility to acquire that license 64 | before distributing the Program. 65 | d) Each Contributor represents that to its knowledge it has sufficient 66 | copyright rights in its Contribution, if any, to grant the copyright 67 | license set forth in this Agreement. 68 | 69 | 3. REQUIREMENTS 70 | 71 | A Contributor may choose to distribute the Program in object code form under 72 | its own license agreement, provided that: 73 | 74 | a) it complies with the terms and conditions of this Agreement; and 75 | b) its license agreement: 76 | i) effectively disclaims on behalf of all Contributors all warranties 77 | and conditions, express and implied, including warranties or 78 | conditions of title and non-infringement, and implied warranties or 79 | conditions of merchantability and fitness for a particular purpose; 80 | ii) effectively excludes on behalf of all Contributors all liability for 81 | damages, including direct, indirect, special, incidental and 82 | consequential damages, such as lost profits; 83 | iii) states that any provisions which differ from this Agreement are 84 | offered by that Contributor alone and not by any other party; and 85 | iv) states that source code for the Program is available from such 86 | Contributor, and informs licensees how to obtain it in a reasonable 87 | manner on or through a medium customarily used for software exchange. 88 | 89 | When the Program is made available in source code form: 90 | 91 | a) it must be made available under this Agreement; and 92 | b) a copy of this Agreement must be included with each copy of the Program. 93 | Contributors may not remove or alter any copyright notices contained 94 | within the Program. 95 | 96 | Each Contributor must identify itself as the originator of its Contribution, 97 | if 98 | any, in a manner that reasonably allows subsequent Recipients to identify the 99 | originator of the Contribution. 100 | 101 | 4. COMMERCIAL DISTRIBUTION 102 | 103 | Commercial distributors of software may accept certain responsibilities with 104 | respect to end users, business partners and the like. While this license is 105 | intended to facilitate the commercial use of the Program, the Contributor who 106 | includes the Program in a commercial product offering should do so in a manner 107 | which does not create potential liability for other Contributors. Therefore, 108 | if a Contributor includes the Program in a commercial product offering, such 109 | Contributor ("Commercial Contributor") hereby agrees to defend and indemnify 110 | every other Contributor ("Indemnified Contributor") against any losses, 111 | damages and costs (collectively "Losses") arising from claims, lawsuits and 112 | other legal actions brought by a third party against the Indemnified 113 | Contributor to the extent caused by the acts or omissions of such Commercial 114 | Contributor in connection with its distribution of the Program in a commercial 115 | product offering. The obligations in this section do not apply to any claims 116 | or Losses relating to any actual or alleged intellectual property 117 | infringement. In order to qualify, an Indemnified Contributor must: 118 | a) promptly notify the Commercial Contributor in writing of such claim, and 119 | b) allow the Commercial Contributor to control, and cooperate with the 120 | Commercial Contributor in, the defense and any related settlement 121 | negotiations. The Indemnified Contributor may participate in any such claim at 122 | its own expense. 123 | 124 | For example, a Contributor might include the Program in a commercial product 125 | offering, Product X. That Contributor is then a Commercial Contributor. If 126 | that Commercial Contributor then makes performance claims, or offers 127 | warranties related to Product X, those performance claims and warranties are 128 | such Commercial Contributor's responsibility alone. Under this section, the 129 | Commercial Contributor would have to defend claims against the other 130 | Contributors related to those performance claims and warranties, and if a 131 | court requires any other Contributor to pay any damages as a result, the 132 | Commercial Contributor must pay those damages. 133 | 134 | 5. NO WARRANTY 135 | 136 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN 137 | "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 138 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each 140 | Recipient is solely responsible for determining the appropriateness of using 141 | and distributing the Program and assumes all risks associated with its 142 | exercise of rights under this Agreement , including but not limited to the 143 | risks and costs of program errors, compliance with applicable laws, damage to 144 | or loss of data, programs or equipment, and unavailability or interruption of 145 | operations. 146 | 147 | 6. DISCLAIMER OF LIABILITY 148 | 149 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 150 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 151 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 152 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 153 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 154 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 155 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 156 | OF SUCH DAMAGES. 157 | 158 | 7. GENERAL 159 | 160 | If any provision of this Agreement is invalid or unenforceable under 161 | applicable law, it shall not affect the validity or enforceability of the 162 | remainder of the terms of this Agreement, and without further action by the 163 | parties hereto, such provision shall be reformed to the minimum extent 164 | necessary to make such provision valid and enforceable. 165 | 166 | If Recipient institutes patent litigation against any entity (including a 167 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 168 | (excluding combinations of the Program with other software or hardware) 169 | infringes such Recipient's patent(s), then such Recipient's rights granted 170 | under Section 2(b) shall terminate as of the date such litigation is filed. 171 | 172 | All Recipient's rights under this Agreement shall terminate if it fails to 173 | comply with any of the material terms or conditions of this Agreement and does 174 | not cure such failure in a reasonable period of time after becoming aware of 175 | such noncompliance. If all Recipient's rights under this Agreement terminate, 176 | Recipient agrees to cease use and distribution of the Program as soon as 177 | reasonably practicable. However, Recipient's obligations under this Agreement 178 | and any licenses granted by Recipient relating to the Program shall continue 179 | and survive. 180 | 181 | Everyone is permitted to copy and distribute copies of this Agreement, but in 182 | order to avoid inconsistency the Agreement is copyrighted and may only be 183 | modified in the following manner. The Agreement Steward reserves the right to 184 | publish new versions (including revisions) of this Agreement from time to 185 | time. No one other than the Agreement Steward has the right to modify this 186 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 187 | Eclipse Foundation may assign the responsibility to serve as the Agreement 188 | Steward to a suitable separate entity. Each new version of the Agreement will 189 | be given a distinguishing version number. The Program (including 190 | Contributions) may always be distributed subject to the version of the 191 | Agreement under which it was received. In addition, after a new version of the 192 | Agreement is published, Contributor may elect to distribute the Program 193 | (including its Contributions) under the new version. Except as expressly 194 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 195 | licenses to the intellectual property of any Contributor under this Agreement, 196 | whether expressly, by implication, estoppel or otherwise. All rights in the 197 | Program not expressly granted under this Agreement are reserved. 198 | 199 | This Agreement is governed by the laws of the State of New York and the 200 | intellectual property laws of the United States of America. No party to this 201 | Agreement will bring a legal action under this Agreement more than one year 202 | after the cause of action arose. Each party waives its rights to a jury trial in 203 | any resulting litigation. 204 | 205 | 206 | -------------------------------------------------------------------------------- /src/main/clojure/clojure/core/rrb_vector/nodes.clj: -------------------------------------------------------------------------------- 1 | ; Copyright (c) Rich Hickey and contributors. All rights reserved. 2 | ; The use and distribution terms for this software are covered by the 3 | ; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php) 4 | ; which can be found in the file epl-v10.html at the root of this distribution. 5 | ; By using this software in any fashion, you are agreeing to be bound by 6 | ; the terms of this license. 7 | ; You must not remove this notice, or any other, from this software. 8 | 9 | (ns clojure.core.rrb-vector.nodes 10 | (:require [clojure.core.rrb-vector.parameters :as p]) 11 | (:import (clojure.core VecNode ArrayManager) 12 | (clojure.lang PersistentVector PersistentVector$Node) 13 | (java.util.concurrent.atomic AtomicReference))) 14 | 15 | (set! *warn-on-reflection* true) 16 | (set! *unchecked-math* true) ;; :warn-on-boxed 17 | 18 | ;;; array managers 19 | 20 | (defmacro mk-am [t] 21 | (#'clojure.core/mk-am &env &form t)) 22 | 23 | (definline object [x] x) 24 | 25 | (def ams 26 | (assoc @#'clojure.core/ams :object (mk-am object))) 27 | 28 | (def object-am 29 | (ams :object)) 30 | 31 | ;;; empty nodes 32 | 33 | (def empty-pv-node PersistentVector/EMPTY_NODE) 34 | 35 | (def empty-gvec-node clojure.core/EMPTY-NODE) 36 | 37 | ;;; node managers 38 | 39 | (definterface NodeManager 40 | (node [^java.util.concurrent.atomic.AtomicReference edit arr]) 41 | (empty []) 42 | (array [node]) 43 | (^java.util.concurrent.atomic.AtomicReference edit [node]) 44 | (^boolean regular [node]) 45 | (clone [^clojure.core.ArrayManager am ^int shift node])) 46 | 47 | (def object-nm 48 | (reify NodeManager 49 | (node [_ edit arr] 50 | (PersistentVector$Node. edit arr)) 51 | (empty [_] 52 | empty-pv-node) 53 | (array [_ node] 54 | (.-array ^PersistentVector$Node node)) 55 | (edit [_ node] 56 | (.-edit ^PersistentVector$Node node)) 57 | (regular [_ node] 58 | (not (== (alength ^objects (.-array ^PersistentVector$Node node)) (int 33)))) 59 | (clone [_ am shift node] 60 | (PersistentVector$Node. 61 | (.-edit ^PersistentVector$Node node) 62 | (aclone ^objects (.-array ^PersistentVector$Node node)))))) 63 | 64 | (def primitive-nm 65 | (reify NodeManager 66 | (node [_ edit arr] 67 | (VecNode. edit arr)) 68 | (empty [_] 69 | empty-gvec-node) 70 | (array [_ node] 71 | (.-arr ^VecNode node)) 72 | (edit [_ node] 73 | (.-edit ^VecNode node)) 74 | (regular [_ node] 75 | (not (== (alength ^objects (.-arr ^VecNode node)) (int 33)))) 76 | (clone [_ am shift node] 77 | (if (zero? shift) 78 | (VecNode. (.-edit ^VecNode node) 79 | (.aclone am (.-arr ^VecNode node))) 80 | (VecNode. (.-edit ^VecNode node) 81 | (aclone ^objects (.-arr ^VecNode node))))))) 82 | 83 | ;;; ranges 84 | 85 | (defmacro ranges [nm node] 86 | `(ints (aget ~(with-meta `(.array ~nm ~node) {:tag 'objects}) 32))) 87 | 88 | (defn last-range [^NodeManager nm node] 89 | (let [rngs (ranges nm node) 90 | i (unchecked-dec-int (aget rngs 32))] 91 | (aget rngs i))) 92 | 93 | (defn regular-ranges [shift cnt] 94 | (let [step (bit-shift-left (int 1) (int shift)) 95 | rngs (int-array 33)] 96 | (loop [i (int 0) r step] 97 | (if (< r cnt) 98 | (do (aset rngs i r) 99 | (recur (unchecked-inc-int i) (unchecked-add-int r step))) 100 | (do (aset rngs i (int cnt)) 101 | (aset rngs 32 (unchecked-inc-int i)) 102 | rngs))))) 103 | 104 | ;;; root overflow 105 | 106 | (defn overflow? [^NodeManager nm root shift cnt] 107 | (if (.regular nm root) 108 | (> (bit-shift-right (unchecked-inc-int (int cnt)) (int 5)) 109 | (bit-shift-left (int 1) (int shift))) 110 | (let [rngs (ranges nm root) 111 | slc (aget rngs 32)] 112 | (and (== slc (int 32)) 113 | (or (== (int shift) (int 5)) 114 | (recur nm 115 | (aget ^objects (.array nm root) (unchecked-dec-int slc)) 116 | (unchecked-subtract-int (int shift) (int 5)) 117 | (unchecked-add-int 118 | (unchecked-subtract-int (aget rngs 31) (aget rngs 30)) 119 | (int 32)))))))) 120 | 121 | ;;; find nil / 0 122 | 123 | (defn index-of-0 ^long [arr] 124 | (let [arr (ints arr)] 125 | (loop [l 0 h 31] 126 | (if (>= l (unchecked-dec h)) 127 | (if (zero? (aget arr l)) 128 | l 129 | (if (zero? (aget arr h)) 130 | h 131 | 32)) 132 | (let [mid (unchecked-add l (bit-shift-right (unchecked-subtract h l) 1))] 133 | (if (zero? (aget arr mid)) 134 | (recur l mid) 135 | (recur (unchecked-inc-int mid) h))))))) 136 | 137 | (defn index-of-nil ^long [arr] 138 | (loop [l 0 h 31] 139 | (if (>= l (unchecked-dec h)) 140 | (if (nil? (aget ^objects arr l)) 141 | l 142 | (if (nil? (aget ^objects arr h)) 143 | h 144 | 32)) 145 | (let [mid (unchecked-add l (bit-shift-right (unchecked-subtract h l) 1))] 146 | (if (nil? (aget ^objects arr mid)) 147 | (recur l mid) 148 | (recur (unchecked-inc-int mid) h)))))) 149 | 150 | ;;; children 151 | 152 | (defn first-child [^NodeManager nm node] 153 | (aget ^objects (.array nm node) 0)) 154 | 155 | (defn last-child [^NodeManager nm node] 156 | (let [arr (.array nm node)] 157 | (if (.regular nm node) 158 | (aget ^objects arr (dec (index-of-nil arr))) 159 | (aget ^objects arr (unchecked-dec-int (aget (ranges nm node) 32)))))) 160 | 161 | (defn remove-leftmost-child [^NodeManager nm shift parent] 162 | (let [arr (.array nm parent)] 163 | (if (nil? (aget ^objects arr 1)) 164 | nil 165 | (let [regular? (.regular nm parent) 166 | new-arr (object-array (if regular? 32 33))] 167 | (System/arraycopy arr 1 new-arr 0 31) 168 | (if-not regular? 169 | (let [rngs (ranges nm parent) 170 | rng0 (aget rngs 0) 171 | new-rngs (int-array 33) 172 | lim (aget rngs 32)] 173 | (System/arraycopy rngs 1 new-rngs 0 (dec lim)) 174 | (loop [i 0] 175 | (when (< i lim) 176 | (aset new-rngs i (- (aget new-rngs i) rng0)) 177 | (recur (inc i)))) 178 | (aset new-rngs 32 (dec (aget rngs 32))) 179 | (aset new-rngs (dec (aget rngs 32)) (int 0)) 180 | (aset ^objects new-arr 32 new-rngs))) 181 | (.node nm (.edit nm parent) new-arr))))) 182 | 183 | (defn replace-leftmost-child [^NodeManager nm shift parent pcnt child d] 184 | (if (.regular nm parent) 185 | (let [step (bit-shift-left 1 shift) 186 | rng0 (- step d) 187 | ncnt (- pcnt d) 188 | li (bit-and (bit-shift-right shift (dec pcnt)) 0x1f) 189 | arr (.array nm parent) 190 | new-arr (object-array 33) 191 | new-rngs (int-array 33)] 192 | (aset ^objects new-arr 0 child) 193 | (System/arraycopy arr 1 new-arr 1 li) 194 | (aset ^objects new-arr 32 new-rngs) 195 | (aset new-rngs 0 (int rng0)) 196 | (aset new-rngs li (int ncnt)) 197 | (aset new-rngs 32 (int (inc li))) 198 | (loop [i 1] 199 | (when (<= i li) 200 | (aset new-rngs i (+ (aget new-rngs (dec i)) step)) 201 | (recur (inc i)))) 202 | (.node nm nil new-arr)) 203 | (let [new-arr (aclone ^objects (.array nm parent)) 204 | rngs (ranges nm parent) 205 | new-rngs (int-array 33) 206 | li (dec (aget rngs 32))] 207 | (aset new-rngs 32 (aget rngs 32)) 208 | (aset ^objects new-arr 32 new-rngs) 209 | (aset ^objects new-arr 0 child) 210 | (loop [i 0] 211 | (when (<= i li) 212 | (aset new-rngs i (- (aget rngs i) (int d))) 213 | (recur (inc i)))) 214 | (.node nm nil new-arr)))) 215 | 216 | (defn replace-rightmost-child [^NodeManager nm shift parent child d] 217 | (if (.regular nm parent) 218 | (let [arr (.array nm parent) 219 | i (unchecked-dec (index-of-nil arr))] 220 | (if (.regular nm child) 221 | (let [new-arr (aclone ^objects arr)] 222 | (aset ^objects new-arr i child) 223 | (.node nm nil new-arr)) 224 | (let [arr (.array nm parent) 225 | new-arr (object-array 33) 226 | step (bit-shift-left 1 shift) 227 | rngs (int-array 33)] 228 | (aset rngs 32 (inc i)) 229 | (aset ^objects new-arr 32 rngs) 230 | (System/arraycopy arr 0 new-arr 0 i) 231 | (aset ^objects new-arr i child) 232 | (loop [j 0 r step] 233 | (when (<= j i) 234 | (aset rngs j r) 235 | (recur (inc j) (+ r step)))) 236 | (aset rngs i (int (last-range nm child))) 237 | (.node nm nil new-arr)))) 238 | (let [rngs (ranges nm parent) 239 | new-rngs (aclone rngs) 240 | i (dec (aget rngs 32)) 241 | new-arr (aclone ^objects (.array nm parent))] 242 | (aset ^objects new-arr i child) 243 | (aset ^objects new-arr 32 new-rngs) 244 | (aset new-rngs i (int (+ (aget rngs i) d))) 245 | (.node nm nil new-arr)))) 246 | 247 | ;;; fold-tail 248 | 249 | (defn new-path [^NodeManager nm ^ArrayManager am shift node] 250 | (let [reg? (== 32 (.alength am (.array nm node))) 251 | len (if reg? 32 33) 252 | arr (object-array len) 253 | rngs (if-not reg? 254 | (doto (int-array 33) 255 | (aset 0 (.alength am (.array nm node))) 256 | (aset 32 1))) 257 | ret (.node nm nil arr)] 258 | (loop [arr arr shift shift] 259 | (if (== shift 5) 260 | (do (if-not reg? 261 | (aset arr 32 rngs)) 262 | (aset arr 0 node)) 263 | (let [a (object-array len) 264 | e (.node nm nil a)] 265 | (aset arr 0 e) 266 | (if-not reg? 267 | (aset arr 32 rngs)) 268 | (recur a (- shift 5))))) 269 | ret)) 270 | 271 | (defn fold-tail [^NodeManager nm ^ArrayManager am node shift cnt tail] 272 | (let [tlen (.alength am tail) 273 | reg? (and (.regular nm node) (== tlen 32)) 274 | arr (.array nm node) 275 | li (index-of-nil arr) 276 | new-arr (object-array (if reg? 32 33)) 277 | rngs (if-not (.regular nm node) (ranges nm node)) 278 | cret (if (== shift 5) 279 | (.node nm nil tail) 280 | (fold-tail nm am 281 | (aget ^objects arr (dec li)) 282 | (- shift 5) 283 | (if (.regular nm node) 284 | (mod cnt (bit-shift-left 1 shift)) 285 | (let [li (unchecked-dec-int (aget rngs 32))] 286 | (if (pos? li) 287 | (unchecked-subtract-int 288 | (aget rngs li) 289 | (aget rngs (unchecked-dec-int li))) 290 | (aget rngs 0)))) 291 | tail)) 292 | new-rngs (ints (if-not reg? 293 | (if rngs 294 | (aclone rngs) 295 | (regular-ranges shift cnt))))] 296 | (when-not (and (or (nil? cret) (== shift 5)) (== li 32)) 297 | (System/arraycopy arr 0 new-arr 0 li) 298 | (when-not reg? 299 | (if (or (nil? cret) (== shift 5)) 300 | (do (aset new-rngs li 301 | (+ (if (pos? li) 302 | (aget new-rngs (dec li)) 303 | (int 0)) 304 | tlen)) 305 | (aset new-rngs 32 (inc li))) 306 | (do (when (pos? li) 307 | (aset new-rngs (dec li) 308 | (+ (aget new-rngs (dec li)) tlen))) 309 | (aset new-rngs 32 li)))) 310 | (if-not reg? 311 | (aset new-arr 32 new-rngs)) 312 | (if (nil? cret) 313 | (aset new-arr li 314 | (new-path nm am 315 | (unchecked-subtract-int shift 5) 316 | (.node nm nil tail))) 317 | (aset new-arr (if (== shift 5) li (dec li)) cret)) 318 | (.node nm nil new-arr)))) 319 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/nodes.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.nodes 2 | (:require [clojure.core.rrb-vector.parameters :as p]) 3 | (:import (clojure.core VecNode ArrayManager) 4 | (clojure.lang PersistentVector PersistentVector$Node) 5 | (java.util.concurrent.atomic AtomicReference))) 6 | 7 | (set! *warn-on-reflection* true) 8 | (set! *unchecked-math* true) ;; :warn-on-boxed 9 | 10 | ;;; array managers 11 | 12 | (defmacro mk-am [t] 13 | (#'clojure.core/mk-am &env &form t)) 14 | 15 | (definline object [x] x) 16 | 17 | (def ams 18 | (assoc @#'clojure.core/ams :object (mk-am object))) 19 | 20 | (def object-am 21 | (ams :object)) 22 | 23 | ;;; empty nodes 24 | 25 | ;; The checking-* functions for the parameterized version expect an 26 | ;; empty node with at most p/max-branches children. The Clojure 27 | ;; versions have 32, unless one also parameterizes the Clojure code, 28 | ;; which I have not done. I will instead try changing the definition 29 | ;; of these empty nodes to have the branch factor of the parameterized 30 | ;; version, so the checking-* functions will not give errors for them. 31 | (def NOEDIT (java.util.concurrent.atomic.AtomicReference. nil)) 32 | (def empty-pv-node (PersistentVector$Node. NOEDIT (object-array p/max-branches))) 33 | 34 | (def empty-gvec-node (VecNode. nil (object-array p/max-branches))) 35 | 36 | ;;; node managers 37 | 38 | (definterface NodeManager 39 | (node [^java.util.concurrent.atomic.AtomicReference edit arr]) 40 | (empty []) 41 | (array [node]) 42 | (^java.util.concurrent.atomic.AtomicReference edit [node]) 43 | (^boolean regular [node]) 44 | (clone [^clojure.core.ArrayManager am ^int shift node])) 45 | 46 | (def object-nm 47 | (reify NodeManager 48 | (node [_ edit arr] 49 | (PersistentVector$Node. edit arr)) 50 | (empty [_] 51 | empty-pv-node) 52 | (array [_ node] 53 | (.-array ^PersistentVector$Node node)) 54 | (edit [_ node] 55 | (.-edit ^PersistentVector$Node node)) 56 | (regular [_ node] 57 | (not (== (alength ^objects (.-array ^PersistentVector$Node node)) (int p/non-regular-array-len)))) 58 | (clone [_ am shift node] 59 | (PersistentVector$Node. 60 | (.-edit ^PersistentVector$Node node) 61 | (aclone ^objects (.-array ^PersistentVector$Node node)))))) 62 | 63 | (def primitive-nm 64 | (reify NodeManager 65 | (node [_ edit arr] 66 | (VecNode. edit arr)) 67 | (empty [_] 68 | empty-gvec-node) 69 | (array [_ node] 70 | (.-arr ^VecNode node)) 71 | (edit [_ node] 72 | (.-edit ^VecNode node)) 73 | (regular [_ node] 74 | (not (== (alength ^objects (.-arr ^VecNode node)) (int p/non-regular-array-len)))) 75 | (clone [_ am shift node] 76 | (if (zero? shift) 77 | (VecNode. (.-edit ^VecNode node) 78 | (.aclone am (.-arr ^VecNode node))) 79 | (VecNode. (.-edit ^VecNode node) 80 | (aclone ^objects (.-arr ^VecNode node))))))) 81 | 82 | ;;; ranges 83 | 84 | (defmacro ranges [nm node] 85 | `(ints (aget ~(with-meta `(.array ~nm ~node) {:tag 'objects}) p/max-branches))) 86 | 87 | (defn last-range [^NodeManager nm node] 88 | (let [rngs (ranges nm node) 89 | i (unchecked-dec-int (aget rngs p/max-branches))] 90 | (aget rngs i))) 91 | 92 | (defn regular-ranges [shift cnt] 93 | (let [step (bit-shift-left (int 1) (int shift)) 94 | rngs (int-array p/non-regular-array-len)] 95 | (loop [i (int 0) r step] 96 | (if (< r cnt) 97 | (do (aset rngs i r) 98 | (recur (unchecked-inc-int i) (unchecked-add-int r step))) 99 | (do (aset rngs i (int cnt)) 100 | (aset rngs p/max-branches (unchecked-inc-int i)) 101 | rngs))))) 102 | 103 | ;;; root overflow 104 | 105 | (defn overflow? [^NodeManager nm root shift cnt] 106 | (if (.regular nm root) 107 | (> (bit-shift-right (unchecked-inc-int (int cnt)) (int p/shift-increment)) 108 | (bit-shift-left (int 1) (int shift))) 109 | (let [rngs (ranges nm root) 110 | slc (aget rngs p/max-branches)] 111 | (and (== slc (int p/max-branches)) 112 | (or (== (int shift) (int p/shift-increment)) 113 | (recur nm 114 | (aget ^objects (.array nm root) (unchecked-dec-int slc)) 115 | (unchecked-subtract-int (int shift) (int p/shift-increment)) 116 | (unchecked-add-int 117 | (unchecked-subtract-int (aget rngs p/max-branches-minus-1) (aget rngs p/max-branches-minus-2)) 118 | (int p/max-branches)))))))) 119 | 120 | ;;; find nil / 0 121 | 122 | (defn index-of-0 ^long [arr] 123 | (let [arr (ints arr)] 124 | (loop [l 0 h p/max-branches-minus-1] 125 | (if (>= l (unchecked-dec h)) 126 | (if (zero? (aget arr l)) 127 | l 128 | (if (zero? (aget arr h)) 129 | h 130 | p/max-branches)) 131 | (let [mid (unchecked-add l (bit-shift-right (unchecked-subtract h l) 1))] 132 | (if (zero? (aget arr mid)) 133 | (recur l mid) 134 | (recur (unchecked-inc-int mid) h))))))) 135 | 136 | (defn index-of-nil ^long [arr] 137 | (loop [l 0 h p/max-branches-minus-1] 138 | (if (>= l (unchecked-dec h)) 139 | (if (nil? (aget ^objects arr l)) 140 | l 141 | (if (nil? (aget ^objects arr h)) 142 | h 143 | p/max-branches)) 144 | (let [mid (unchecked-add l (bit-shift-right (unchecked-subtract h l) 1))] 145 | (if (nil? (aget ^objects arr mid)) 146 | (recur l mid) 147 | (recur (unchecked-inc-int mid) h)))))) 148 | 149 | ;;; children 150 | 151 | (defn first-child [^NodeManager nm node] 152 | (aget ^objects (.array nm node) 0)) 153 | 154 | (defn last-child [^NodeManager nm node] 155 | (let [arr (.array nm node)] 156 | (if (.regular nm node) 157 | (aget ^objects arr (dec (index-of-nil arr))) 158 | (aget ^objects arr (unchecked-dec-int (aget (ranges nm node) p/max-branches)))))) 159 | 160 | (defn remove-leftmost-child [^NodeManager nm shift parent] 161 | (let [arr (.array nm parent)] 162 | (if (nil? (aget ^objects arr 1)) 163 | nil 164 | (let [regular? (.regular nm parent) 165 | new-arr (object-array (if regular? p/max-branches p/non-regular-array-len))] 166 | (System/arraycopy arr 1 new-arr 0 p/max-branches-minus-1) 167 | (if-not regular? 168 | (let [rngs (ranges nm parent) 169 | rng0 (aget rngs 0) 170 | new-rngs (int-array p/non-regular-array-len) 171 | lim (aget rngs p/max-branches)] 172 | (System/arraycopy rngs 1 new-rngs 0 (dec lim)) 173 | (loop [i 0] 174 | (when (< i lim) 175 | (aset new-rngs i (- (aget new-rngs i) rng0)) 176 | (recur (inc i)))) 177 | (aset new-rngs p/max-branches (dec (aget rngs p/max-branches))) 178 | (aset new-rngs (dec (aget rngs p/max-branches)) (int 0)) 179 | (aset ^objects new-arr p/max-branches new-rngs))) 180 | (.node nm (.edit nm parent) new-arr))))) 181 | 182 | (defn replace-leftmost-child [^NodeManager nm shift parent pcnt child d] 183 | (if (.regular nm parent) 184 | (let [step (bit-shift-left 1 shift) 185 | rng0 (- step d) 186 | ncnt (- pcnt d) 187 | li (bit-and (bit-shift-right shift (dec pcnt)) p/branch-mask) 188 | arr (.array nm parent) 189 | new-arr (object-array p/non-regular-array-len) 190 | new-rngs (int-array p/non-regular-array-len)] 191 | (aset ^objects new-arr 0 child) 192 | (System/arraycopy arr 1 new-arr 1 li) 193 | (aset ^objects new-arr p/max-branches new-rngs) 194 | (aset new-rngs 0 (int rng0)) 195 | (aset new-rngs li (int ncnt)) 196 | (aset new-rngs p/max-branches (int (inc li))) 197 | (loop [i 1] 198 | (when (<= i li) 199 | (aset new-rngs i (+ (aget new-rngs (dec i)) step)) 200 | (recur (inc i)))) 201 | (.node nm nil new-arr)) 202 | (let [new-arr (aclone ^objects (.array nm parent)) 203 | rngs (ranges nm parent) 204 | new-rngs (int-array p/non-regular-array-len) 205 | li (dec (aget rngs p/max-branches))] 206 | (aset new-rngs p/max-branches (aget rngs p/max-branches)) 207 | (aset ^objects new-arr p/max-branches new-rngs) 208 | (aset ^objects new-arr 0 child) 209 | (loop [i 0] 210 | (when (<= i li) 211 | (aset new-rngs i (- (aget rngs i) (int d))) 212 | (recur (inc i)))) 213 | (.node nm nil new-arr)))) 214 | 215 | (defn replace-rightmost-child [^NodeManager nm shift parent child d] 216 | (if (.regular nm parent) 217 | (let [arr (.array nm parent) 218 | i (unchecked-dec (index-of-nil arr))] 219 | (if (.regular nm child) 220 | (let [new-arr (aclone ^objects arr)] 221 | (aset ^objects new-arr i child) 222 | (.node nm nil new-arr)) 223 | (let [arr (.array nm parent) 224 | new-arr (object-array p/non-regular-array-len) 225 | step (bit-shift-left 1 shift) 226 | rngs (int-array p/non-regular-array-len)] 227 | (aset rngs p/max-branches (inc i)) 228 | (aset ^objects new-arr p/max-branches rngs) 229 | (System/arraycopy arr 0 new-arr 0 i) 230 | (aset ^objects new-arr i child) 231 | (loop [j 0 r step] 232 | (when (<= j i) 233 | (aset rngs j r) 234 | (recur (inc j) (+ r step)))) 235 | (aset rngs i (int (last-range nm child))) 236 | (.node nm nil new-arr)))) 237 | (let [rngs (ranges nm parent) 238 | new-rngs (aclone rngs) 239 | i (dec (aget rngs p/max-branches)) 240 | new-arr (aclone ^objects (.array nm parent))] 241 | (aset ^objects new-arr i child) 242 | (aset ^objects new-arr p/max-branches new-rngs) 243 | (aset new-rngs i (int (+ (aget rngs i) d))) 244 | (.node nm nil new-arr)))) 245 | 246 | ;;; fold-tail 247 | 248 | (defn new-path [^NodeManager nm ^ArrayManager am shift node] 249 | (let [reg? (== p/max-branches (.alength am (.array nm node))) 250 | len (if reg? p/max-branches p/non-regular-array-len) 251 | arr (object-array len) 252 | rngs (if-not reg? 253 | (doto (int-array p/non-regular-array-len) 254 | (aset 0 (.alength am (.array nm node))) 255 | (aset p/max-branches 1))) 256 | ret (.node nm nil arr)] 257 | (loop [arr arr shift shift] 258 | (if (== shift p/shift-increment) 259 | (do (if-not reg? 260 | (aset arr p/max-branches rngs)) 261 | (aset arr 0 node)) 262 | (let [a (object-array len) 263 | e (.node nm nil a)] 264 | (aset arr 0 e) 265 | (if-not reg? 266 | (aset arr p/max-branches rngs)) 267 | (recur a (- shift p/shift-increment))))) 268 | ret)) 269 | 270 | (defn fold-tail [^NodeManager nm ^ArrayManager am node shift cnt tail] 271 | (let [tlen (.alength am tail) 272 | reg? (and (.regular nm node) (== tlen p/max-branches)) 273 | arr (.array nm node) 274 | li (index-of-nil arr) 275 | new-arr (object-array (if reg? p/max-branches p/non-regular-array-len)) 276 | rngs (if-not (.regular nm node) (ranges nm node)) 277 | cret (if (== shift p/shift-increment) 278 | (.node nm nil tail) 279 | (fold-tail nm am 280 | (aget ^objects arr (dec li)) 281 | (- shift p/shift-increment) 282 | (if (.regular nm node) 283 | (mod cnt (bit-shift-left 1 shift)) 284 | (let [li (unchecked-dec-int (aget rngs p/max-branches))] 285 | (if (pos? li) 286 | (unchecked-subtract-int 287 | (aget rngs li) 288 | (aget rngs (unchecked-dec-int li))) 289 | (aget rngs 0)))) 290 | tail)) 291 | new-rngs (ints (if-not reg? 292 | (if rngs 293 | (aclone rngs) 294 | (regular-ranges shift cnt))))] 295 | (when-not (and (or (nil? cret) (== shift p/shift-increment)) (== li p/max-branches)) 296 | (System/arraycopy arr 0 new-arr 0 li) 297 | (when-not reg? 298 | (if (or (nil? cret) (== shift p/shift-increment)) 299 | (do (aset new-rngs li 300 | (+ (if (pos? li) 301 | (aget new-rngs (dec li)) 302 | (int 0)) 303 | tlen)) 304 | (aset new-rngs p/max-branches (inc li))) 305 | (do (when (pos? li) 306 | (aset new-rngs (dec li) 307 | (+ (aget new-rngs (dec li)) tlen))) 308 | (aset new-rngs p/max-branches li)))) 309 | (if-not reg? 310 | (aset new-arr p/max-branches new-rngs)) 311 | (if (nil? cret) 312 | (aset new-arr li 313 | (new-path nm am 314 | (unchecked-subtract-int shift p/shift-increment) 315 | (.node nm nil tail))) 316 | (aset new-arr (if (== shift p/shift-increment) li (dec li)) cret)) 317 | (.node nm nil new-arr)))) 318 | -------------------------------------------------------------------------------- /src/parameterized/clojure/clojure/core/rrb_vector/debug_platform_dependent.clj: -------------------------------------------------------------------------------- 1 | (ns clojure.core.rrb-vector.debug-platform-dependent 2 | (:refer-clojure :exclude [format printf]) 3 | (:require [clojure.core.rrb-vector.parameters :as p] 4 | clojure.core.rrb-vector.rrbt 5 | [clojure.core.rrb-vector.nodes 6 | :refer [ranges object-nm primitive-nm object-am]] 7 | [clojure.core.rrb-vector :as fv]) 8 | (:import (clojure.lang PersistentVector PersistentVector$TransientVector 9 | PersistentVector$Node APersistentVector$SubVector) 10 | (java.util.concurrent.atomic AtomicReference) 11 | (java.lang.reflect Field Method) 12 | (clojure.core Vec VecNode ArrayManager) 13 | (clojure.core.rrb_vector.rrbt Vector Transient) 14 | (clojure.core.rrb_vector.nodes NodeManager))) 15 | 16 | ;; Work around the fact that several fields of type 17 | ;; PersistentVector$TransientVector are private, but note that this is 18 | ;; only intended for debug use. 19 | (def ^Class transient-core-vec-class (class (transient (vector)))) 20 | (def ^Field transient-core-root-field (.getDeclaredField transient-core-vec-class "root")) 21 | (.setAccessible transient-core-root-field true) 22 | (def ^Field transient-core-shift-field (.getDeclaredField transient-core-vec-class "shift")) 23 | (.setAccessible transient-core-shift-field true) 24 | (def ^Field transient-core-tail-field (.getDeclaredField transient-core-vec-class "tail")) 25 | (.setAccessible transient-core-tail-field true) 26 | (def ^Field transient-core-cnt-field (.getDeclaredField transient-core-vec-class "cnt")) 27 | (.setAccessible transient-core-cnt-field true) 28 | 29 | (def transient-core-vec-tailoff-methods 30 | (filter #(= "tailoff" (.getName ^Method %)) 31 | (.getDeclaredMethods transient-core-vec-class))) 32 | (assert (= (count transient-core-vec-tailoff-methods) 1)) 33 | (def ^Method transient-core-vec-tailoff-method 34 | (first transient-core-vec-tailoff-methods)) 35 | (.setAccessible transient-core-vec-tailoff-method true) 36 | 37 | 38 | (def ^Class persistent-core-vec-class (class (vector))) 39 | (def persistent-core-vec-tailoff-methods 40 | (filter #(= "tailoff" (.getName ^Method %)) 41 | (.getDeclaredMethods persistent-core-vec-class))) 42 | (assert (= (count persistent-core-vec-tailoff-methods) 1)) 43 | (def ^Method persistent-core-vec-tailoff-method 44 | (first persistent-core-vec-tailoff-methods)) 45 | (.setAccessible persistent-core-vec-tailoff-method true) 46 | 47 | 48 | (def format clojure.core/format) 49 | 50 | (def printf clojure.core/printf) 51 | 52 | (defn internal-node? [obj] 53 | (contains? #{PersistentVector$Node VecNode} (class obj))) 54 | 55 | (defn persistent-vector? [obj] 56 | (contains? #{PersistentVector Vec Vector} 57 | (class obj))) 58 | 59 | (defn transient-vector? [obj] 60 | (contains? #{PersistentVector$TransientVector Transient} 61 | (class obj))) 62 | 63 | (defn is-vector? [obj] 64 | (contains? #{PersistentVector Vec Vector 65 | PersistentVector$TransientVector Transient} 66 | (class obj))) 67 | 68 | (defn dbg-tailoff [v] 69 | (cond 70 | (instance? PersistentVector v) 71 | (.invoke persistent-core-vec-tailoff-method v (object-array 0)) 72 | 73 | (= PersistentVector$TransientVector (class v)) 74 | (.invoke transient-core-vec-tailoff-method v (object-array 0)) 75 | 76 | :else 77 | (.tailoff v))) 78 | 79 | (defn dbg-tidx [v] 80 | (- (count v) (dbg-tailoff v))) 81 | 82 | (defn subvector-data [v] 83 | (if (instance? APersistentVector$SubVector v) 84 | (let [^APersistentVector$SubVector v v] 85 | {:orig-v v 86 | :subvector? true 87 | :v (.v v) 88 | :subvec-start (.start v) 89 | :subvec-end (.end v)}) 90 | {:orig-v v 91 | :subvector? false 92 | :v v})) 93 | 94 | ;; All of the classes below have a .tailoff method implementation that 95 | ;; works correctly for that class. You can use the debug-tailoff 96 | ;; function to work around the fact that this method is not public for 97 | ;; some of the vector classes. 98 | 99 | (defn accessors-for [v] 100 | (condp identical? (class v) 101 | PersistentVector (let [nm object-nm, am object-am] 102 | {:get-root #(.-root ^PersistentVector %) 103 | :get-shift #(.-shift ^PersistentVector %) 104 | :get-tail #(.-tail ^PersistentVector %) 105 | :get-cnt #(.-cnt ^PersistentVector %) 106 | :get-array #(.array ^NodeManager nm %) 107 | :get-ranges #(ranges ^NodeManager nm %) 108 | :regular? #(.regular ^NodeManager nm %) 109 | :tail-len #(.alength ^ArrayManager am %) 110 | }) 111 | PersistentVector$TransientVector 112 | (let [nm object-nm, am object-am] 113 | {:get-root #(.get transient-core-root-field ^PersistentVector$TransientVector %) 114 | :get-shift #(.get transient-core-shift-field ^PersistentVector$TransientVector %) 115 | :get-tail #(.get transient-core-tail-field ^PersistentVector$TransientVector %) 116 | :get-cnt #(.get transient-core-cnt-field ^PersistentVector$TransientVector %) 117 | :get-array #(.array ^NodeManager nm %) 118 | :get-ranges #(ranges ^NodeManager nm %) 119 | :regular? #(.regular ^NodeManager nm %) 120 | :tail-len #(.alength ^ArrayManager am %) 121 | }) 122 | Vec (let [nm primitive-nm, am #(.-am ^Vec %)] 123 | {:get-root #(.-root ^Vec %) 124 | :get-shift #(.-shift ^Vec %) 125 | :get-tail #(.-tail ^Vec %) 126 | :get-cnt #(.-cnt ^Vec %) 127 | :get-array #(.array ^NodeManager nm %) 128 | :get-ranges #(ranges ^NodeManager nm %) 129 | :regular? #(.regular ^NodeManager nm %) 130 | :tail-len #(.alength ^ArrayManager am %) 131 | }) 132 | Vector (let [nm (.-nm ^Vector v), am #(.-am ^Vector %)] 133 | {:get-root #(.-root ^Vector %) 134 | :get-shift #(.-shift ^Vector %) 135 | :get-tail #(.-tail ^Vector %) 136 | :get-cnt #(.-cnt ^Vector %) 137 | :get-array #(.array ^NodeManager nm %) 138 | :get-ranges #(ranges ^NodeManager nm %) 139 | :regular? #(.regular ^NodeManager nm %) 140 | :tail-len #(.alength ^ArrayManager am %) 141 | }) 142 | Transient (let [nm (.-nm ^Transient v), am (.-am ^Transient v)] 143 | {:get-root #(.debugGetRoot ^Transient %) 144 | :get-shift #(.debugGetShift ^Transient %) 145 | :get-tail #(.debugGetTail ^Transient %) 146 | :get-cnt #(.debugGetCnt ^Transient %) 147 | :get-array #(.array ^NodeManager nm %) 148 | :get-ranges #(ranges ^NodeManager nm %) 149 | :regular? #(.regular ^NodeManager nm %) 150 | :tail-len #(.alength ^ArrayManager am %) 151 | }))) 152 | 153 | (defn unwrap-subvec-accessors-for [v] 154 | (let [{:keys [v] :as m} (subvector-data v) 155 | accessors (accessors-for v)] 156 | (merge m accessors))) 157 | 158 | (defn abbrev-for-type-of [obj] 159 | (let [cn (.getName (class obj)) 160 | d (.lastIndexOf cn ".")] 161 | (subs cn (inc d)))) 162 | 163 | (defn same-coll? [a b] 164 | (and (= (count a) 165 | (count b) 166 | (.size ^java.util.Collection a) 167 | (.size ^java.util.Collection b)) 168 | (= a b) 169 | (= b a) 170 | (= (hash a) (hash b)) 171 | (= (.hashCode ^Object a) (.hashCode ^Object b)))) 172 | 173 | ;; TBD: No cljs specific version yet 174 | (defn count-nodes [& vs] 175 | (let [m (java.util.IdentityHashMap.)] 176 | (doseq [v vs] 177 | (let [{:keys [v get-root get-shift get-array]} 178 | (unwrap-subvec-accessors-for v)] 179 | (letfn [(go [n shift] 180 | (when n 181 | (.put m n n) 182 | (if-not (zero? shift) 183 | (let [arr (get-array n) 184 | ns (take p/max-branches arr)] 185 | (doseq [n ns] 186 | (go n (- shift p/shift-increment)))))))] 187 | (go (get-root v) (get-shift v))))) 188 | (.size m))) 189 | 190 | (defn int-array? [x] 191 | (and (not (nil? x)) 192 | (.isArray (class x)) 193 | (= Integer/TYPE (. (class x) getComponentType)))) 194 | 195 | ;; TBD: No cljs-specific version of this function yet 196 | #_(defn ranges-not-int-array [x] 197 | (seq (remove int-array? (objects-in-slot-32-of-obj-arrays x)))) 198 | 199 | (defn atomicref? [x] 200 | (instance? AtomicReference x)) 201 | 202 | (defn thread? [x] 203 | (instance? java.lang.Thread x)) 204 | 205 | (defn non-identical-edit-nodes [v all-vector-tree-nodes] 206 | (let [{:keys [v]} (unwrap-subvec-accessors-for v) 207 | node-maps (all-vector-tree-nodes v) 208 | ^java.util.IdentityHashMap ihm (java.util.IdentityHashMap.)] 209 | (doseq [i node-maps] 210 | (when (= :internal (:kind i)) 211 | (.put ihm (.edit (:node i)) true))) 212 | ihm)) 213 | 214 | (defn edit-nodes-errors [v all-vector-tree-nodes] 215 | (let [{:keys [v get-root]} (unwrap-subvec-accessors-for v) 216 | klass (class v) 217 | ^java.util.IdentityHashMap ihm (non-identical-edit-nodes 218 | v all-vector-tree-nodes) 219 | objs-maybe-some-nils (.keySet ihm) 220 | ;; I do not believe that Clojure's built-in vector types can 221 | ;; ever have edit fields equal to nil, but there are some 222 | ;; cases where I have seen core.rrb-vector edit fields equal 223 | ;; to nil. As far as I can tell this seems harmless, as long 224 | ;; as it is in a persistent vector, not a transient one. 225 | objs (remove nil? objs-maybe-some-nils) 226 | neither-nil-nor-atomicref (remove atomicref? objs)] 227 | (if (seq neither-nil-nor-atomicref) 228 | {:error true 229 | :description (str "Found edit object with class " 230 | (class (first neither-nil-nor-atomicref)) 231 | " - expecting nil or AtomicReference") 232 | :data ihm 233 | :not-atomic-refs neither-nil-nor-atomicref} 234 | (let [refd-objs (map #(.get ^AtomicReference %) objs) 235 | non-nils (remove nil? refd-objs) 236 | not-threads (remove thread? non-nils) 237 | root-edit (.edit (get-root v))] 238 | (cond 239 | (seq not-threads) 240 | {:error true 241 | :description (str "Found edit AtomicReference ref'ing neither nil" 242 | " nor a Thread object") 243 | :data ihm} 244 | (persistent-vector? v) 245 | (if (= (count non-nils) 0) 246 | {:error false} 247 | {:error true 248 | :description (str "Within a persistent (i.e. not transient)" 249 | " vector, found at least one edit" 250 | " AtomicReference object that ref's a Thread" 251 | " object. Expected all of them to be nil.") 252 | :data ihm 253 | :val1 (count non-nils) 254 | :val2 non-nils}) 255 | 256 | (transient-vector? v) 257 | (cond 258 | (not= (count non-nils) 1) 259 | {:error true 260 | :description (str "Within a transient vector, found " 261 | (count non-nils) " edit AtomicReference" 262 | " object(s) that ref's a Thread object." 263 | " Expected exactly 1.") 264 | :data ihm 265 | :val1 (count non-nils) 266 | :val2 non-nils} 267 | (not (atomicref? root-edit)) 268 | {:error true 269 | :description (str "Within a transient vector, found root edit" 270 | " field that was ref'ing an object with class " 271 | (class root-edit) 272 | " - expected AtomicReference.") 273 | :data root-edit} 274 | (not (thread? (.get ^AtomicReference root-edit))) 275 | (let [obj (.get ^AtomicReference root-edit)] 276 | {:error true 277 | :description (str "Within a transient vector, found root edit" 278 | " field ref'ing an AtomicReference object," 279 | " but that in turn ref'd something with class " 280 | (class obj) 281 | " - expected java.lang.Thread.") 282 | :data obj}) 283 | :else {:error false}) 284 | 285 | :else {:error true 286 | :description (str "Unknown class " klass " for object checked" 287 | " by edit-nodes-wrong-number-of-threads") 288 | :data v}))))) 289 | --------------------------------------------------------------------------------