├── src ├── test │ ├── java │ │ ├── DummySpecTest.kt │ │ ├── cursors │ │ │ ├── calendar │ │ │ │ └── DaySeqKtTest.kt │ │ │ ├── hash │ │ │ │ └── MD4Test.kt │ │ │ ├── ByColNameKtTest.kt │ │ │ ├── ml │ │ │ │ └── FeatureRangeKtTest.kt │ │ │ ├── NegateColumnTest.kt │ │ │ ├── tbd │ │ │ │ ├── GilbertKtTest.kt │ │ │ │ └── xyz.kt │ │ │ ├── CategoriesKtTest.kt │ │ │ ├── SimpleCursorTest.kt │ │ │ ├── CsvCursorTest.kt │ │ │ ├── HeapCursorTest.kt │ │ │ ├── NinetyDegreeTest.kt │ │ │ ├── io │ │ │ │ └── ISAMCursorKtTest.kt │ │ │ ├── CursorKtTest.kt │ │ │ └── DatabinanceKlineIsamTest.kt │ │ ├── TestDate.kt │ │ ├── vec │ │ │ ├── ml │ │ │ │ └── FeatureRangeKtTest.kt │ │ │ ├── Vect0rTakeTest.kt │ │ │ ├── macros │ │ │ │ └── VectorLikeKtTest.kt │ │ │ └── util │ │ │ │ ├── BloomFilterTest.kt │ │ │ │ └── HorizonTest.kt │ │ ├── CirQlarTest.kt │ │ ├── XPathTest.kt │ │ ├── trie │ │ │ └── TrieTest.kt │ │ └── VectorLikeKtTest.kt │ ├── resources │ │ ├── ninetyDegreesTest19334425141920886859.bin │ │ ├── caven4.fwf │ │ ├── caven20.csv │ │ ├── ninetyDegreesTest19334425141920886859.bin.meta │ │ └── caven20.fwf │ └── kotlin │ │ └── vec │ │ └── ThinSliceTest.kt └── main │ └── java │ ├── cursors │ ├── TypeMemento.kt │ ├── HelpMain.kt │ ├── io │ │ ├── FileAccess.kt │ │ ├── CsvReader.kt │ │ ├── ImageRaster.kt │ │ ├── NioCursor.kt │ │ ├── CursorOf.kt │ │ ├── MappedFile.kt │ │ ├── Cursor.kt │ │ ├── WriteCSV.kt │ │ ├── RowVecExtensions.kt │ │ ├── IOMemento.kt │ │ └── ISAMCursor.kt │ ├── calendar │ │ ├── DaySeq.kt │ │ ├── Feature_range.kt │ │ ├── UnixTimeRemapper.kt │ │ └── JvmCal.kt │ ├── MutableListView.kt │ ├── NegateColumn.kt │ ├── context │ │ ├── NormalizedRange.kt │ │ ├── Addressable.kt │ │ ├── Arity.kt │ │ ├── Ordering.kt │ │ └── RecordBoundary.kt │ ├── SimpleCursor.kt │ ├── tbd │ │ ├── Cursor.kt │ │ └── Gilbert.kt │ ├── Resample.kt │ ├── ByColName.kt │ ├── macros │ │ ├── Operators.kt │ │ └── Join.kt │ ├── hash │ │ └── MD4.kt │ ├── effects │ │ └── Show.kt │ ├── Categories.kt │ └── Clusters.kt │ ├── mu │ ├── KLogging.kt │ ├── KotlinLogging.kt │ └── KLogger.kt │ ├── vec │ ├── ml │ │ ├── DummySpec.kt │ │ └── FeatureRange.kt │ ├── macros │ │ ├── Combine.kt │ │ ├── SparseVect0r.kt │ │ └── Twop13.kt │ ├── FloatCirQlar.kt │ ├── DoubleCirQlar.kt │ ├── IntCirQlar.kt │ ├── util │ │ ├── Horizon.kt │ │ ├── FibonacciReporter.kt │ │ ├── BloomFilter.kt │ │ └── BikeShed.kt │ └── CirQlar.kt │ ├── trie │ ├── Node.kt │ ├── ArrayMap.kt │ └── Trie.kt │ ├── org │ ├── bereft │ │ └── MarketData.kt │ └── slf4j │ │ └── Logger.kt │ └── doc │ └── docs.java ├── bin ├── release-sign.bash └── colenv.sh ├── .gitignore ├── .github └── FUNDING.yml ├── release.properties ├── mvnw.cmd └── README.md /src/test/java/DummySpecTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | internal class DummySpecTest -------------------------------------------------------------------------------- /src/test/java/cursors/calendar/DaySeqKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors.calendar 2 | 3 | class DaySeqKtTest -------------------------------------------------------------------------------- /bin/release-sign.bash: -------------------------------------------------------------------------------- 1 | JAVA_HOME=~/opt/jdk-17/ mvn -P release-sign-artifacts package gpg:sign 2 | -------------------------------------------------------------------------------- /src/main/java/cursors/TypeMemento.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | interface TypeMemento { 4 | val networkSize: Int? 5 | } -------------------------------------------------------------------------------- /src/main/java/cursors/HelpMain.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | 4 | fun main(args: Array) { 5 | println(args.toList()) 6 | } -------------------------------------------------------------------------------- /src/main/java/mu/KLogging.kt: -------------------------------------------------------------------------------- 1 | package mu 2 | 3 | import org.slf4j.Logger 4 | 5 | open class KLogging { 6 | val logger: Logger get() = KLogger() 7 | 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/cursors/io/FileAccess.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import java.io.Closeable 4 | 5 | abstract class FileAccess(open val filename: String) : Closeable -------------------------------------------------------------------------------- /src/test/java/TestDate.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import java.time.chrono.IsoChronology 4 | 5 | object TestDate { 6 | 7 | fun main() { 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/test/resources/ninetyDegreesTest19334425141920886859.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jnorthrup/columnar/HEAD/src/test/resources/ninetyDegreesTest19334425141920886859.bin -------------------------------------------------------------------------------- /bin/colenv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -fx 4 | JDIR=$(dirname $0)/../cursor 5 | exec java -classpath "$JDIR/target/*:$JDIR/target/lib/*" ${EXECMAIN:=columnar.HelpMainKt} "$@" 6 | -------------------------------------------------------------------------------- /src/main/java/vec/ml/DummySpec.kt: -------------------------------------------------------------------------------- 1 | package vec.ml 2 | 3 | /** 4 | * if you specify first or last categories to be the Dummy, this is 5 | */ 6 | enum class DummySpec { 7 | /*Dummy -- */KeepAll, First, Last 8 | } -------------------------------------------------------------------------------- /src/main/java/trie/Node.kt: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | 4 | /** 5 | * Created by kenny on 6/6/16. 6 | */ 7 | class Node(val pathSeg: String, var leaf: Boolean, val payload: Int, var children: Map = linkedMapOf()) 8 | 9 | -------------------------------------------------------------------------------- /src/main/java/mu/KotlinLogging.kt: -------------------------------------------------------------------------------- 1 | package mu 2 | 3 | class KotlinLogging { 4 | companion object { 5 | fun logger(function: () -> Unit): KLogger { 6 | 7 | return KLogger() 8 | } 9 | } 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/org/bereft/MarketData.kt: -------------------------------------------------------------------------------- 1 | package org.bereft 2 | 3 | interface MarketData { 4 | val open: Double 5 | val high: Double 6 | val low: Double 7 | val close: Double 8 | val volume: Double 9 | val closeTime: Long 10 | } -------------------------------------------------------------------------------- /src/main/java/doc/docs.java: -------------------------------------------------------------------------------- 1 | package doc; 2 | /*** 3 | * heres a doc class. this serves as a shim for maven to not error out of creating a doc jarfile. sorry that's all we kotliners have atm. 4 | * 5 | */ 6 | public class docs { 7 | public void waitingForKotlinJavadocs() { 8 | } 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/cursors/calendar/DaySeq.kt: -------------------------------------------------------------------------------- 1 | package cursors.calendar 2 | 3 | import java.time.LocalDate 4 | 5 | fun daySeq(min: LocalDate, max: LocalDate): Sequence { 6 | var marker: LocalDate = min 7 | return sequence { 8 | while (max > marker) { 9 | yield(marker) 10 | marker = marker.plusDays(1) 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /src/main/java/cursors/io/CsvReader.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import cursors.context.TokenizedRow 5 | import java.nio.file.Files 6 | import java.nio.file.Paths 7 | 8 | object CsvReader { 9 | fun csvReader(): CsvReader = this 10 | fun open(pathToLabels: String): Cursor = TokenizedRow.CsvArraysCursor(Files.readAllLines(Paths.get(pathToLabels))) 11 | } 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/cursors/MutableListView.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import vec.macros.Vect0r 4 | 5 | /** 6 | * we can use this to break all of the imutable-size contracts of Vect0r and hide bugs for days 7 | */ 8 | class MutableListView( 9 | val list: List, 10 | ) : Vect0r { 11 | override val first: Int by list::size 12 | override val second: (Int) -> T 13 | get() = list::get 14 | } -------------------------------------------------------------------------------- /src/main/java/cursors/calendar/Feature_range.kt: -------------------------------------------------------------------------------- 1 | package cursors.calendar 2 | 3 | import vec.macros.Pai2 4 | import vec.macros.t2 5 | import vec.ml.featureRange 6 | import java.time.LocalDate 7 | 8 | @JvmName("FRLocalDate") 9 | fun featureRange( 10 | seq: Iterable, 11 | maxMinPai2: Pai2 = LocalDate.MAX t2 LocalDate.MIN, 12 | ): Pai2 = featureRange(seq, maxMinPai2) -------------------------------------------------------------------------------- /src/test/java/vec/ml/FeatureRangeKtTest.kt: -------------------------------------------------------------------------------- 1 | package vec.ml 2 | 3 | import kotlin.test.* 4 | import vec.util._l 5 | 6 | 7 | infix fun Any?.shouldBe(that: Any?) { 8 | assertEquals(that, this) 9 | } 10 | 11 | class FeatureRangeKtTest { 12 | @Test 13 | fun test0() { 14 | val seq = _l[0] as Iterable 15 | val featureRange = featureRange(seq) 16 | System.err.println(featureRange) 17 | } 18 | } -------------------------------------------------------------------------------- /src/test/java/cursors/hash/MD4Test.kt: -------------------------------------------------------------------------------- 1 | package cursors.hash 2 | 3 | 4 | import kotlin.test.* 5 | import vec.util._a 6 | 7 | class MD4Test { 8 | @Test 9 | fun testxlate() { 10 | assertEquals(_a[0xa.toByte()].hex, "0a") 11 | assertEquals(_a[0x1.toByte()].hex, "01") 12 | } 13 | 14 | @Test 15 | fun testest() { 16 | 17 | val md4 = "test".md4 18 | assertEquals("db346d691d7acc4dc2625db19f9e3f52", md4) 19 | 20 | 21 | } 22 | 23 | @Test 24 | fun test1() { 25 | 26 | val md4 = "1".md4 27 | assertEquals("8be1ec697b14ad3a53b371436120641d", md4) 28 | 29 | } 30 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | **/target 3 | *.class 4 | *.iml 5 | *.log 6 | */target 7 | .gradle 8 | .idea 9 | /gradle/ 10 | /out/ 11 | /superannuate/** 12 | /target/ 13 | build 14 | superannuate 15 | target*.ipr 16 | *.iws 17 | /cursor/pom.xml.versionsBackup 18 | /lillypads/pom.xml.versionsBackup 19 | /trie/pom.xml.versionsBackup 20 | /vector-like/pom.xml.versionsBackup 21 | *.versionsBackup 22 | /.mvn/wrapper/maven-wrapper.jar 23 | /cursor/${project.build.directory}/ 24 | *.releaseBackup 25 | .gradle 26 | gradle 27 | .idea/ 28 | /superannuate/ 29 | .project 30 | .settings 31 | /superannuate/superannuated1909.fwf 32 | /.aider.chat.history.md 33 | /.aider.input.history 34 | -------------------------------------------------------------------------------- /src/test/resources/caven4.fwf: -------------------------------------------------------------------------------- 1 | 2017-10-130101761/0101010207/13-14/01 88.000000000000 0E-12 2 | 2017-10-220102211/0101010212/13-14/01 80.000000000000 0E-12 3 | 2017-10-240500020/0101010106/13-14/05 4.000000000000 0E-12 4 | 2017-10-220500020/0101010106/13-14/05 820.000000000000 0E-12 5 | -------------------------------------------------------------------------------- /src/main/java/cursors/NegateColumn.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.io.colIdx 4 | 5 | class NegateColumn(val negated: String) 6 | 7 | /** 8 | * be careful in client code with importing Cursor.get and not Vect0r.get 9 | */ 10 | operator fun Cursor.get(vararg skip: NegateColumn): Cursor { 11 | 12 | // logDebug { "solving for direct-reindexing negation ${skip.map { it.negated }}" }//noisey, seemingly stable now 13 | val indexes = this.colIdx.get(*skip) 14 | // logDebug { "direct-reindexing negation indexes are ${indexes.toList()}" } //noisey, seemingly stable now 15 | return this[indexes] 16 | } 17 | 18 | operator fun String.unaryMinus(): NegateColumn = NegateColumn(this) 19 | 20 | -------------------------------------------------------------------------------- /src/main/java/cursors/context/NormalizedRange.kt: -------------------------------------------------------------------------------- 1 | package cursors.context 2 | 3 | import vec.macros.Tw1n 4 | import kotlin.coroutines.CoroutineContext 5 | 6 | /** 7 | * this counterpart to [#cursors.calendar::feature_range] functions , takes a Tw1n, and links this to a column meta 8 | * @see cursors.calendar.featureRange 9 | * @see vec.ml.featureRange 10 | */ 11 | class NormalizedRange(val range: Tw1n) : CoroutineContext.Element { 12 | companion object { 13 | val normalizedRangeKey: CoroutineContext.Key> = object : CoroutineContext.Key> {} 14 | } 15 | 16 | override val key: CoroutineContext.Key<*> 17 | get() = normalizedRangeKey 18 | } -------------------------------------------------------------------------------- /src/main/java/cursors/context/Addressable.kt: -------------------------------------------------------------------------------- 1 | package cursors.context 2 | 3 | import kotlin.coroutines.CoroutineContext 4 | 5 | sealed class Addressable : CoroutineContext.Element { 6 | 7 | 8 | override val key: addressableKey get() = addressableKey 9 | 10 | object addressableKey : CoroutineContext.Key 11 | 12 | } 13 | 14 | abstract class Forward : Iterable, Addressable() 15 | 16 | class Indexable( 17 | /**count of records*/ 18 | val size: () -> Int, 19 | val seek: (Int) -> Unit, 20 | ) : Addressable() 21 | 22 | open class Abstract(val size: () -> Q, val seek: (T) -> Unit) : Addressable() 23 | class Associative(size: () -> Int, seek: (T) -> Unit) : Abstract(size, seek) 24 | -------------------------------------------------------------------------------- /src/main/java/mu/KLogger.kt: -------------------------------------------------------------------------------- 1 | package mu 2 | 3 | import org.slf4j.Logger 4 | import vec.util.logDebug 5 | 6 | class KLogger: Logger { 7 | fun debug(debugTxt: () -> String) { 8 | logDebug(debugTxt) 9 | } 10 | 11 | override fun debug(s: String) { 12 | logDebug { s } 13 | } 14 | override fun trace(s: String) { 15 | logDebug { s } 16 | } 17 | override fun info(s: String) = logDebug { s } 18 | fun info(function: () -> String) { 19 | debug(function) 20 | } 21 | 22 | override fun warn(s: String) = logDebug { s } 23 | fun warn(function: () -> String) = debug(function) 24 | 25 | fun error(s: String) = logDebug { s } 26 | fun error(function: () -> String) { 27 | debug(function) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/test/java/cursors/ByColNameKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.TokenizedRow 4 | import cursors.effects.head 5 | import cursors.io.colIdx 6 | import kotlin.test.* 7 | import vec.util._l 8 | 9 | class ByColNameKtTest { 10 | 11 | @Test 12 | fun testTheNegation() { 13 | val curs = TokenizedRow.CsvArraysCursor( 14 | _l[ 15 | "c1,c2,c3,c4,c5,c6", 16 | "1,2,3,4,5,6" 17 | ] 18 | ) 19 | 20 | curs["c2", "c3"].head() 21 | curs[-"c2", -"c3"].head() 22 | 23 | val ints = curs.colIdx["c2", "c3"] 24 | val ints1 = curs.colIdx[-"c2", -"c3"] 25 | println(ints.toList()) 26 | println(ints1.toList()) 27 | curs[ints + ints1].head() 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: jnorthrup 4 | #patreon: # Replace with a single Patreon username 5 | #open_collective: # Replace with a single Open Collective username 6 | #ko_fi: # Replace with a single Ko-fi username 7 | #tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | #community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | #liberapay: # Replace with a single Liberapay username 10 | #issuehunt: # Replace with a single IssueHunt username 11 | #otechie: # Replace with a single Otechie username 12 | custom: $XMR://8AMh15gKG4NiwtVK5BcLS5K9gFyxsMTrtVMHNGevWt5f9VLejM4hkqREe5s4xkWfGq2rYdFNMLQkMTJCEHKSeLq7CFuiRwj # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /src/main/java/cursors/SimpleCursor.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.Scalar 4 | import vec.macros.* 5 | import kotlin.coroutines.CoroutineContext 6 | 7 | /** 8 | * this provides a tiny bit of extra safety for zero-length Cursors 9 | * problem: colidx wants to get scalars from empty Cursor 10 | * 11 | 12 | 13 | * 14 | * 15 | */ 16 | class SimpleCursor @JvmOverloads constructor( 17 | val scalars: Vect0r, 18 | val data: Vect0r>, 19 | val o: Vect0r<() -> CoroutineContext> = scalars α { it.`⟲` }, 20 | val c: Pai2 Vect02 CoroutineContext>> = data α { it.zip(o) }, 21 | ) : Cursor /*by c*/ { 22 | override val first: Int by data::size 23 | override val second: (Int) -> Pai2 Pai2 CoroutineContext>> by c::second 24 | } 25 | -------------------------------------------------------------------------------- /src/test/java/cursors/ml/FeatureRangeKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors.ml 2 | 3 | import kotlin.test.* 4 | import vec.macros.Tw1n 5 | import vec.ml.deNormalize 6 | import vec.ml.normalize 7 | 8 | class FeatureRangeKtTest { 9 | 10 | @Test 11 | fun normalize() { 12 | 13 | (Tw1n(10.0, 100.0).normalize(40.0)) 14 | assertEquals(0.0, (Tw1n(10.0, 100.0).normalize(10.0))) 15 | run { 16 | val normalize = Tw1n(10.0, 100.0).normalize(100.0) 17 | assertEquals(1.0, normalize) 18 | } 19 | run { 20 | val deNormalize = Tw1n(10.0, 100.0).deNormalize(1.0) 21 | assertEquals(100.0, deNormalize) 22 | } 23 | run { 24 | val deNormalize = Tw1n(10.0, 100.0).deNormalize(0.0) 25 | assertEquals(10.0, deNormalize) 26 | } 27 | 28 | 29 | } 30 | } -------------------------------------------------------------------------------- /src/main/java/cursors/io/ImageRaster.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import cursors.context.Scalar.Companion.Scalar 5 | import vec.macros.t2 6 | import java.io.File 7 | import javax.imageio.ImageIO 8 | import kotlin.coroutines.CoroutineContext 9 | 10 | 11 | //todo: image IO blackboard states below 12 | fun crc16be(): CoroutineContext = Scalar(IOMemento.IoDouble, "gray16be") 13 | 14 | /** 15 | * returns a Cursor that traverses rows of double grayscale values from png 16be color format 16 | */ 17 | fun gray16bePixelCursor(tmpbmp: File): Cursor = ImageIO.read(tmpbmp).let { img -> 18 | img.height t2 { y: Int -> 19 | img.raster.getPixels(0, y, img.width, 1, IntArray(img.width)).let { ints: IntArray -> 20 | img.width t2 { 21 | ((ints[it] and 0xffff).toDouble() / 0xffff.toDouble()) t2 ::crc16be 22 | } 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/cursors/io/NioCursor.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.TypeMemento 4 | import cursors.context.CellDriver 5 | import vec.macros.* 6 | import vec.macros.Vect02_.left 7 | import java.nio.ByteBuffer 8 | import kotlin.coroutines.CoroutineContext 9 | 10 | typealias NioMeta = Tripl3, TypeMemento, Int> 11 | typealias NioCursor = Matrix Any?, (Any?) -> Unit, NioMeta>> 12 | typealias TableRoot = Pai2 13 | typealias ColMeta = Pai2 14 | typealias RowMeta = Vect0r 15 | typealias CellMeta = () -> CoroutineContext 16 | typealias RowVec = Vect02 17 | 18 | typealias writefn = Function2 19 | typealias readfn = Function1 20 | 21 | 22 | fun stringOf(it: RowVec): String = it.left.toList().map { any -> 23 | val isVec = any as? Vect0r<*> 24 | val any1 = isVec?.toList() ?: any 25 | any1 26 | }.toString() 27 | -------------------------------------------------------------------------------- /src/test/java/cursors/NegateColumnTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.TokenizedRow 4 | import cursors.effects.head 5 | import vec.macros.reverse 6 | import kotlin.test.* 7 | import vec.macros.size 8 | import vec.util._l 9 | 10 | internal class NegateColumnTest { 11 | @Test 12 | fun testNegate() { 13 | val csvLines11 = _l[ 14 | "One,Two,Three", 15 | "7,8,9", 16 | "1,2,3", 17 | "1,2,3", 18 | "1,2,3", 19 | "1,2,3", 20 | "1,2,3", 21 | "1,2,3", 22 | "7,6,5" 23 | ] 24 | val csvLines1 = 25 | TokenizedRow.CsvArraysCursor(csvLines11) 26 | csvLines1.head() 27 | csvLines1.reverse.head() 28 | csvLines1[-"One"].size shouldBe 8 29 | csvLines1[-"One", -"Three"].size shouldBe 8 30 | csvLines1[1].size shouldBe 8 31 | } 32 | 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/cursors/tbd/Cursor.kt: -------------------------------------------------------------------------------- 1 | package cursors.tbd 2 | 3 | import cursors.Cursor 4 | import cursors.at 5 | import vec.macros.Pai2 6 | import vec.macros.Vect02 7 | import vec.macros.Vect0r 8 | import kotlin.coroutines.CoroutineContext 9 | 10 | class MonoVal(val cell: Pai2 CoroutineContext>) { 11 | fun ctxt(): CoroutineContext = cell.second() 12 | val m: Any? get() = cell.first 13 | 14 | } 15 | 16 | class MonoRow(val row: Vect02 CoroutineContext>) : 17 | Vect0r { 18 | override val first: Int get() = row.first 19 | override val second: (Int) -> MonoVal 20 | get() = { ix: Int -> 21 | MonoVal( 22 | row.second(ix) 23 | ) 24 | } 25 | } 26 | 27 | class MonoCursor(val curs: Cursor) : 28 | Vect0r { 29 | override val first: Int get() = curs.first 30 | override val second: (Int) -> MonoRow 31 | get() = { iy: Int -> 32 | MonoRow((curs at (iy))) 33 | } 34 | } -------------------------------------------------------------------------------- /src/test/java/CirQlarTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import vec.CirQlar 4 | import vec.macros.f1rst 5 | import vec.macros.last 6 | import vec.macros.toList 7 | 8 | import kotlin.test.Test 9 | import kotlin.test.assertEquals 10 | 11 | class CirQlarTest { 12 | 13 | @Test 14 | fun testCirQtoList() { 15 | 16 | val cirQ = CirQlar(3) 17 | (-1 until 7).forEach { 18 | System.err.println(cirQ.toList()) 19 | cirQ.offer(it) 20 | } 21 | val list = cirQ.toList() 22 | System.err.println(list) 23 | assertEquals(6, list.last()) 24 | assertEquals(4, list.first()) 25 | 26 | } 27 | 28 | @Test 29 | fun testtoVect0r() { 30 | val cirQ = CirQlar(3) 31 | (-1 until 7).forEach { 32 | System.err.println(cirQ.toVect0r().toList()) 33 | cirQ.offer(it) 34 | } 35 | val vec = cirQ.toVect0r() 36 | System.err.println(vec.toList()) 37 | assertEquals(6, vec.last) 38 | assertEquals(4, vec.f1rst) 39 | } 40 | } -------------------------------------------------------------------------------- /src/main/java/org/slf4j/Logger.kt: -------------------------------------------------------------------------------- 1 | package org.slf4j 2 | 3 | interface Logger { 4 | fun info(s: String) 5 | fun debug(s: String) 6 | fun trace(s: String) 7 | fun warn(s: String) 8 | fun isInfoEnabled():Boolean=true 9 | fun isDebugEnabled():Boolean=true 10 | fun isTraceEnabled():Boolean=true 11 | fun isWarnEnabled():Boolean=true 12 | fun info(s: String, o: Any) = info("it $o") 13 | fun debug(s: String, o: Any) = debug("it $o") 14 | fun trace(s: String, o: Any) = trace("it $o") 15 | fun warn(s: String, o: Any) = warn("it $o") 16 | fun info(s: String, o: Any, o2: Any) = info("it $o $o2") 17 | fun debug(s: String, o: Any, o2: Any) = debug("it $o $o2") 18 | fun trace(s: String, o: Any, o2: Any) = trace("it $o $o2") 19 | fun warn(s: String, o: Any, o2: Any) = warn("it $o $o2") 20 | fun info(s: String, vararg o: Any) = info("it $o") 21 | fun debug(s: String, vararg o: Any) = debug("it $o") 22 | fun trace(s: String, vararg o: Any) = trace("it $o") 23 | fun warn(s: String, vararg o: Any) = warn("it $o") 24 | } 25 | -------------------------------------------------------------------------------- /src/test/resources/caven20.csv: -------------------------------------------------------------------------------- 1 | date,channel,deliver_qty,return_qty 2 | 2018-10-13,0101761/0101010207/13-14/01,88.0,0.0 3 | 2017-10-22,0102211/0101010212/13-14/01,80.0,0.0 4 | 2017-08-24,0500020/0101010106/13-14/05,4.0,0.0 5 | 2017-05-15,0102237/0101010803/10-14/01,820.0,0.0 6 | 2017-05-04,0200051/0101010106/13-14/02,2.0,0.0 7 | 2016-10-26,0101848/0101010212/13-14/01,260.0,0.0 8 | 2016-08-02,0100052/0101010106/13-14/01,100.0,0.0 9 | 2016-04-30,0101462/0101010503/13-14/01,4.0,0.0 10 | 2016-01-26,0100164/0101010106/13-14/01,2.0,0.0 11 | 2018-10-05,0500067/0101010106/13-14/05,4.0,0.0 12 | 2018-02-13,0200343/0101010106/13-16/02,80.0,0.0 13 | 2018-02-12,0500414/0101010106/13-13/05,20.0,0.0 14 | 2017-12-31,0101663/0101010209/13-14/01,20.0,0.0 15 | 2017-10-09,0300067/0101010106/13-14/03,8.0,0.0 16 | 2017-08-21,0102165/0101010803/10-13/01,20.0,0.0 17 | 2016-05-13,0101718/0101010106/13-14/01,220.0,0.0 18 | 2016-03-02,0500433/0101010206/13-14/05,88.0,0.0 19 | 2016-02-15,0700048/0101010106/13-14/07,1.0,0.0 20 | 2016-01-07,0101307/0101010802/20-15/01,80.0,0.0 21 | 2017-07-06,0100865/0101010106/13-14/01,4.0,0.0 22 | -------------------------------------------------------------------------------- /src/test/java/cursors/tbd/GilbertKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors.tbd 2 | 3 | import kotlin.math.abs 4 | import kotlin.test.* 5 | 6 | typealias IntIntPair = Pair 7 | 8 | /** TODO more exhasutive tests */ 9 | internal class GilbertCurveTest { 10 | @Test 11 | fun testSizes() { 12 | for (w in 1..8) for (h in 1..8) testGilbert(w, h) 13 | } 14 | 15 | companion object { 16 | private fun testGilbert(w: Int, h: Int) { 17 | 18 | val pointsOrdered = mutableListOf() 19 | gilbertCurve(w, h) { x, y -> pointsOrdered += (x to y) } 20 | 21 | assertEquals(w * h, pointsOrdered.distinct().size/*, "expected # of unique points"*/) 22 | 23 | //test that each sequential point has manhattan distance <= 2 from previous 24 | for (i in 0 until pointsOrdered.size - 1) { 25 | val a: IntIntPair = pointsOrdered[i] 26 | val b: IntIntPair = pointsOrdered[i + 1] 27 | assertTrue(abs(a.first - b.first) + abs(a.second - b.second) <= 2) 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/cursors/Resample.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.calendar.daySeq 4 | import cursors.calendar.featureRange 5 | import cursors.io.scalars 6 | import vec.macros.* 7 | import vec.macros.Vect02_.left 8 | import java.time.LocalDate 9 | 10 | 11 | /** 12 | * creates new cursor with synthetic rows combined at end with ordered missing index localdates 13 | * @param indexcol the column with dates to resample 14 | */ 15 | fun Cursor.resample(indexcol: Int): Cursor = let { 16 | val pai2 = combine(this[indexcol]).left α { it as LocalDate } 17 | val indexValues = pai2.`➤` 18 | val (min, max) = featureRange(indexValues, LocalDate.MAX t2 LocalDate.MIN) 19 | val scalars = this.scalars 20 | val sequence: Sequence = daySeq(min, max) - if(pai2.size>8)indexValues.toSet()else indexValues 21 | val indexVec = sequence.toVect0r() 22 | val cursor: Cursor = 23 | indexVec.size t2 { iy: Int -> 24 | scalars.size t2 { ix: Int -> 25 | ix.takeIf { it == indexcol }?.let { indexVec[iy] } t2 { scalars[ix] } 26 | } 27 | } 28 | combine(this, cursor) 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/cursors/ByColName.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.io.colIdx 4 | import vec.macros.Vect02 5 | import vec.macros.Vect02_.right 6 | import vec.macros.size 7 | import vec.macros.toList 8 | import javax.management.openmbean.InvalidKeyException 9 | 10 | 11 | /** 12 | * colIdx lands here 13 | */ 14 | inline operator fun Vect02.get(vararg s: T): IntArray = right.toList().let { list -> 15 | s.map { 16 | val indexOf = list.indexOf(it) 17 | if (-1 == indexOf) 18 | throw InvalidKeyException("$it not found in meta among $list") 19 | indexOf 20 | }.toIntArray() 21 | } 22 | 23 | /** 24 | * colIdx lands here 25 | */ 26 | inline operator fun Vect02.get(vararg s: NegateColumn): IntArray = 27 | ((0 until size).toSet() - get(*s.map { it.negated }.toTypedArray()).toSet()).toIntArray() 28 | 29 | fun Cursor.indexesForColumnNames(vararg s:String): IntArray =colIdx.get(*s) 30 | 31 | @JvmName("cursorViewByColumnNames") 32 | /** 33 | * cursor["f1","f2","f3"] to auto map the indexes 34 | */ 35 | operator fun Cursor.get(vararg s: String): Cursor = this[indexesForColumnNames(*s)] 36 | -------------------------------------------------------------------------------- /src/test/java/cursors/tbd/xyz.kt: -------------------------------------------------------------------------------- 1 | package cursors.tbd 2 | 3 | import kotlin.test.* 4 | import vec.macros.combine 5 | import vec.util._v 6 | 7 | 8 | class testXYZ { 9 | @Test 10 | fun tetstXyz() { 11 | val matrix = 12 | _v[_v[ 13 | _v[1, 2, 3], 14 | _v[4, 5, 6], 15 | _v[5, 8, 9], 16 | _v[6, 11, 12], 17 | ], _v[ 18 | _v[1, 2, 3], 19 | _v[4, 5, 6], 20 | _v[5, 8, 9], 21 | _v[6, 11, 12], 22 | ], _v[ 23 | _v[1, 2, 3], 24 | _v[4, 5, 6], 25 | _v[5, 8, 9], 26 | _v[6, 11, 12], 27 | ], _v[ 28 | _v[1, 2, 3], 29 | _v[4, 5, 6], 30 | _v[5, 8, 9], 31 | _v[6, 11, 12], 32 | ], _v[ 33 | _v[1, 2, 3], 34 | _v[4, 5, 6], 35 | _v[5, 8, 9], 36 | _v[6, 11, 12], 37 | ]] 38 | combine(combine(matrix)) 39 | 40 | 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/vec/Vect0rTakeTest.kt: -------------------------------------------------------------------------------- 1 | //package cursors.vec 2 | //TODO() 3 | // 4 | //import org.junit.jupiter.api.Test 5 | //import vec.macros.get 6 | //import vec.macros.size 7 | //import vec.macros.take 8 | //import vec.macros.takeLast 9 | // 10 | //import kotlin.test.assertEquals 11 | // 12 | ///** 13 | // * Unit test of Vect0r.take and Vect0r.takeLast 14 | // */ 15 | //class Vect0rTakeTest { 16 | // @Test 17 | // fun testTake() { 18 | // val v = vec.util._v[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 19 | // val v1 = v.take(5) 20 | // assertEquals(v1.size, 5) 21 | // assertEquals(v1[0], 1) 22 | // assertEquals(v1[1], 2) 23 | // assertEquals(v1[2], 3) 24 | // assertEquals(v1[3], 4) 25 | // assertEquals(v1[4], 5) 26 | // } 27 | // 28 | // @Test 29 | // fun testTakeLast() { 30 | // val v = vec.util._v[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 31 | // val v1 = v.takeLast(5) 32 | // assertEquals(v1.size, 5) 33 | // assertEquals(v1[0], 6) 34 | // assertEquals(v1[1], 7) 35 | // assertEquals(v1[2], 8) 36 | // assertEquals(v1[3], 9) 37 | // assertEquals(v1[4], 10) 38 | // } 39 | //} 40 | // 41 | // 42 | -------------------------------------------------------------------------------- /src/test/kotlin/vec/ThinSliceTest.kt: -------------------------------------------------------------------------------- 1 | package vec 2 | 3 | import kotlinx.coroutines.flow.Flow 4 | import kotlinx.coroutines.flow.flowOf 5 | import kotlinx.coroutines.flow.toList as flowToList // Alias to avoid conflict 6 | import kotlinx.coroutines.runBlocking 7 | import kotlinx.datetime.Clock 8 | import kotlin.test.Test 9 | import kotlin.test.assertEquals 10 | import kotlin.test.assertTrue 11 | 12 | class ThinSliceTest { 13 | 14 | @Test 15 | fun `flow should emit items correctly`() = runBlocking { 16 | val testFlow: Flow = flowOf(1, 2, 33) 17 | val collectedItems = testFlow.flowToList() // Use alias 18 | 19 | assertEquals(3, collectedItems.size, "Flow should emit 3 items") 20 | assertEquals(1, collectedItems[0], "First item should be 1") 21 | assertEquals(33, collectedItems[2], "Third item should be 33") 22 | } 23 | 24 | @Test 25 | fun `clock system now should return current time`() { 26 | val currentTime = Clock.System.now() 27 | // Basic check: epoch seconds should be positive and significant (e.g., after year 2000) 28 | assertTrue(currentTime.epochSeconds > 946684800L, "Current time should be after 2000-01-01") 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/cursors/macros/Operators.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNCHECKED_CAST") 2 | 3 | package cursors.macros 4 | 5 | import cursors.Cursor 6 | import cursors.at 7 | import cursors.io.RowVec 8 | import vec.macros.* 9 | import vec.macros.Vect02_.left 10 | import vec.macros.Vect02_.right 11 | 12 | /** 13 | * reducer func -- operator for sum/avg/mean etc. would be nice, but we have to play nice in a type-safe language so ∑'s just a hint of a reducer semantic 14 | */ 15 | infix fun Cursor.`∑`(reducer: (Any?, Any?) -> Any?): Cursor = Cursor(first) { iy: Int -> 16 | val aggcell: RowVec = second(iy) 17 | val al: Vect0r<*> = aggcell.left 18 | RowVec(aggcell.first) { ix: Int -> 19 | val ac = al[ix] 20 | val toList: List? = (ac as? Vect0r<*>)?.toList() 21 | val iterable: Iterable? = toList ?: (ac as? Iterable<*>) 22 | val any1 = iterable?.reduce(reducer) 23 | val any = any1 ?: ac 24 | any t2 aggcell[ix].second 25 | } 26 | } 27 | 28 | /** 29 | * tr func 30 | */ 31 | infix fun Cursor.α(unaryFunctor: (Any?) -> Any?): Cursor = run { 32 | size t2 { iy: Int -> 33 | val row: RowVec = (this at iy) 34 | (row.left α (unaryFunctor)).zip(row.right) 35 | } 36 | } 37 | 38 | inline val > V.`…`: List get() = this.toList() 39 | -------------------------------------------------------------------------------- /src/test/java/cursors/CategoriesKtTest.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("RemoveRedundantCallsOfConversionMethods") 2 | 3 | package cursors 4 | 5 | import cursors.context.TokenizedRow 6 | import cursors.effects.show 7 | import cursors.io.IOMemento 8 | import kotlin.test.* 9 | import vec.ml.DummySpec 10 | import vec.util._l 11 | 12 | internal class CategoriesKtTest { 13 | 14 | val curs = TokenizedRow.CsvArraysCursor( 15 | 16 | _l["something,notsomething,cbf", 17 | "john,0,below me", 18 | "john,2,below me", 19 | "john,4,below me", 20 | "john,8,below me", 21 | "john,16,below me", 22 | "john,32,below me", 23 | "john,64,below me", 24 | "john,128,below me", 25 | "phillip,256,below me"] , 26 | overrides = linkedMapOf("notsomething" to IOMemento.IoInt) 27 | ) 28 | 29 | 30 | @Test 31 | fun testCategories() { 32 | curs.categories().show() 33 | curs.categories(DummySpec.Last).show() 34 | curs.categories(1).show() 35 | } 36 | 37 | @Test 38 | fun testCategories1() { 39 | curs.categories().asBitSet().show() 40 | curs.categories(DummySpec.Last).asBitSet().show() 41 | curs.categories(1).asBitSet().show() 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/cursors/SimpleCursorTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.Scalar.Companion.Scalar 4 | import cursors.io.IOMemento 5 | import kotlin.test.* 6 | import vec.macros.Vect0r 7 | import vec.macros.t2 8 | import vec.macros.toList 9 | import vec.macros.toVect0r 10 | import vec.util._v 11 | import vec.util.logDebug 12 | 13 | class SimpleCursorTest { 14 | @Test 15 | fun testScalarsDispatch() { 16 | 17 | 18 | val vscalar = _v[ 19 | Scalar(IOMemento.IoString t2 "a"), 20 | Scalar(IOMemento.IoInt t2 "b"), 21 | Scalar(IOMemento.IoDouble t2 "c"), 22 | ] 23 | SimpleCursor( 24 | vscalar, _v[ 25 | _v["dog", 1, 0.0], 26 | _v["cat", 11, 0.01], 27 | _v["act", 111, 0.011], 28 | _v["lib", 1111, 0.0111], 29 | _v["nil", 11111, 0.1111], 30 | 31 | ] 32 | ).scalars.also { logDebug { it.toList().toString() } } 33 | val listOf: MutableList> = MutableList(0) { _v[null] } 34 | listOf.clear() 35 | val to = listOf.toVect0r() 36 | val simpleCursor: SimpleCursor = SimpleCursor(vscalar, to) 37 | simpleCursor.scalars.also { logDebug { it.toList().toString() } } 38 | 39 | } 40 | } -------------------------------------------------------------------------------- /src/main/java/cursors/io/CursorOf.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import cursors.context.Arity 5 | import cursors.context.Columnar 6 | import cursors.context.Scalar.Companion.Scalar 7 | import vec.macros.* 8 | import vec.macros.Vect02_.left 9 | import vec.macros.Vect02_.right 10 | import kotlin.collections.* 11 | import kotlin.collections.component2 12 | import kotlin.coroutines.CoroutineContext 13 | 14 | fun cursorOf(root: TableRoot): Cursor = root.let { (nioc: NioCursor, crt: CoroutineContext): TableRoot -> 15 | val (xy: IntArray, mapper: (IntArray) -> Tripl3<() -> Any?, (Any?) -> Unit, NioMeta>) = nioc 16 | val (xsize: Int, ysize: Int) = xy 17 | vec.macros.Vect0r(ysize) { iy -> 18 | vec.macros.Vect0r(xsize) { ix -> 19 | val (a: () -> Any?) = mapper(intArrayOf(ix, iy)) 20 | a() t2 { 21 | val cnar: Columnar = 22 | crt[Arity.arityKey] as Columnar 23 | //todo define spreadsheet context linkage; insert a matrix of (Any?)->Any? to crt as needed 24 | // and call in a cell through here 25 | val name = 26 | cnar.right[ix] ?: throw(InstantiationError("Tableroot's Columnar has no names")) 27 | val type = cnar.left[ix] 28 | Scalar(type, name) 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/cursors/hash/MD4.kt: -------------------------------------------------------------------------------- 1 | package cursors.hash 2 | 3 | import org.bouncycastle.crypto.digests.MD4Digest 4 | 5 | 6 | val Any?.md4a: ByteArray 7 | get() { 8 | val s = toString() 9 | val ba = s.toByteArray() 10 | val d: MD4Digest = MD4Digest() // this overhead doesn't look worth pooling or threadlocal 11 | d.update(ba, 0, ba.size) 12 | val o = ByteArray(d.digestSize) 13 | d.doFinal(o, 0) 14 | return o 15 | } 16 | 17 | val Any?.md4: String get() = this.md4a.hex 18 | fun hexLate(v: Int): Char = if (v < 0xa) '0' + v 19 | else 'a' + (v - 0xa) 20 | 21 | val ByteArray.hex: String 22 | get() { 23 | val res = CharArray(size shl 1) 24 | for (ix in indices) get(ix).toInt().also { 25 | var os = ix shl 1 26 | res[os++] = hexLate(it shr 4 and 0xf) 27 | res[os] = hexLate( it and 0xf) 28 | } 29 | return String(res) 30 | } 31 | 32 | 33 | /* 34 | object RosettaMD4 { 35 | @Throws(Exception::class) 36 | 37 | fun main(argv: Array) { 38 | val r = "test".toByteArray(charset("US-ASCII")) 39 | val d = MD4Digest() 40 | d.update(r, 0, r.size) 41 | val o = ByteArray(d.digestSize) 42 | d.doFinal(o, 0) 43 | 44 | Hex.encode(o, System.out) 45 | println() 46 | println(o.hex) 47 | } 48 | }*/ 49 | -------------------------------------------------------------------------------- /src/main/java/vec/macros/Combine.kt: -------------------------------------------------------------------------------- 1 | package vec.macros 2 | 3 | import kotlin.math.absoluteValue 4 | 5 | @JvmName("combine_Vect0r ") 6 | fun combine(vargs: Vect0r>): Vect0r = combine(*vargs.toArray()) 7 | 8 | 9 | @JvmName("combine_VecVa") 10 | fun combine(vararg vecArgs: Vect0r): Vect0r = vecArgs.filterNot { it.size == 0 }.let { vex -> 11 | when (vex.size) { 12 | 0 -> vecArgs[0] 13 | 1 -> vex.first() 14 | else -> vex.let { rows -> 15 | muxIndexes(rows).let { (isize, tails) -> 16 | isize t2 { ix: Int -> 17 | val (slot, i1) = demuxIndex(tails, ix) 18 | rows.get(slot).get(i1) 19 | } 20 | } 21 | } 22 | } 23 | } 24 | 25 | fun muxIndexes(vargs: Collection>): Pai2 = 26 | vargs.foldIndexed(0 t2 IntArray(vargs.size)) { vix, (acc: Int, srcVec: IntArray), (vecSize) -> 27 | (acc + vecSize).let { nsize -> 28 | srcVec[vix] = nsize 29 | nsize t2 srcVec 30 | } 31 | } 32 | 33 | fun demuxIndex(tails: IntArray, ix: Int): Tw1nt = 34 | (1 + tails.binarySearch(ix)).absoluteValue.let { source -> 35 | Tw1n( 36 | source, if (source != 0) (ix % tails[source]) - tails[source - 1] 37 | else ix % tails[0] 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/vec/FloatCirQlar.kt: -------------------------------------------------------------------------------- 1 | package vec 2 | 3 | import vec.macros.* 4 | import java.util.* 5 | 6 | /** 7 | 8 | stripped down circular 9 | 10 | only mutability is offer(T) 11 | 12 | has cheap direct toVect0r with live properties 13 | has more expensive toList/iterator by copy/concat 14 | */ 15 | class FloatCirQlar( 16 | val maxSize: Int, 17 | val al: FloatArray = FloatArray(maxSize), 18 | ) : AbstractQueue() { 19 | var tail: Int = 0 20 | val full: Boolean get() = maxSize <= tail 21 | 22 | override fun iterator(): MutableIterator = this.toVect0r(). iterator() as MutableIterator 23 | 24 | fun toList(): List = toVect0r().mapIndexedToList { _, t -> t } 25 | 26 | fun toVect0r(): Pai2 Float> = this@FloatCirQlar.size t2 { x: Int -> 27 | al[if (tail >= maxSize) { 28 | (tail + x) % maxSize 29 | } else x] 30 | } 31 | 32 | //todo: lockless dequeue here ? 33 | override fun offer(e: Float): Boolean = 34 | synchronized(this) { 35 | al[tail % maxSize] = e 36 | tail++ 37 | if (tail == 2 * maxSize) tail = maxSize 38 | true 39 | } 40 | 41 | override fun poll(): Float = TODO("Not yet implemented") 42 | override fun peek(): Float = TODO("Not yet implemented") 43 | override fun add(k: Float): Boolean = offer(k) 44 | override val size: Int get() = kotlin.math.min(tail, maxSize) 45 | } -------------------------------------------------------------------------------- /src/main/java/cursors/io/MappedFile.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import java.io.Closeable 4 | import java.io.RandomAccessFile 5 | import java.nio.ByteBuffer 6 | import java.nio.channels.FileChannel 7 | import kotlin.math.min 8 | 9 | /** 10 | * mapmodes READ_ONLY, READ_WRITE, or 11 | "r" Open for reading only. Invoking any of the write methods of the resulting object will cause an IOException to be thrown. 12 | "rw" Open for reading and writing. If the file does not already exist then an attempt will be made to create it. 13 | "rws" Open for reading and writing, as with "rw", and also require that every update to the file's content or metadata be written synchronously to the underlying storage device. 14 | "rwd" Open for reading and writing, as with "rw", and also require that every update to the file's content be written synchronously to the underlying storage device. 15 | */ 16 | open class MappedFile( 17 | filename: String, 18 | val mode: String = "r", 19 | val mapMode: FileChannel.MapMode = FileChannel.MapMode.READ_ONLY, 20 | val randomAccessFile: RandomAccessFile = RandomAccessFile( 21 | filename, 22 | mode 23 | ), 24 | val channel: FileChannel = randomAccessFile.channel, 25 | val length: Long = randomAccessFile.length(), 26 | val mappedByteBuffer: ByteBuffer = channel.map( 27 | mapMode, 28 | 0, 29 | min(Int.MAX_VALUE.toLong(), length) 30 | ), 31 | 32 | ) : FileAccess(filename), Closeable by randomAccessFile -------------------------------------------------------------------------------- /src/main/java/cursors/effects/Show.kt: -------------------------------------------------------------------------------- 1 | package cursors.effects 2 | 3 | import cursors.Cursor 4 | import cursors.at 5 | import cursors.io.colIdx 6 | import vec.macros.Vect02_.left 7 | import vec.macros.Vect02_.right 8 | import vec.macros.combine 9 | import vec.macros.size 10 | import vec.macros.toList 11 | import kotlin.math.max 12 | import kotlin.math.min 13 | import kotlin.random.Random 14 | 15 | //import kotlin.random.Random breaks graalvm 16 | val random: java.util.Random = java.util.Random() 17 | 18 | /*simple printout macro*/ 19 | fun Cursor.show(range: IntProgression = 0 until size) { 20 | println("rows:$size" to colIdx.right.toList()) 21 | showValues(range) 22 | } 23 | 24 | fun Cursor.showValues(range: IntProgression) { 25 | try { 26 | (range).forEach { 27 | val pai2 = this at it 28 | val combine = combine(pai2.left) 29 | println(combine.toList()) 30 | } 31 | } catch (e: NoSuchElementException) { 32 | System.err.println("cannot fully access range $range") 33 | } 34 | } 35 | 36 | /** 37 | * head default 5 rows 38 | * just like unix head - print default 5 lines from cursor contents to stdout 39 | */ 40 | @JvmOverloads fun Cursor.head(last: Int = 5): Unit = show(0 until (max(0, min(last, size)))) 41 | 42 | /** 43 | * run head starting at random index 44 | */ 45 | fun Cursor.showRandom(n: Int = 5) { 46 | head(0);repeat(n) { 47 | if (size > 0) showValues(Random.nextInt(0, size).let { it..it }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/cursors/CsvCursorTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.TokenizedRow.Companion.CsvArraysCursor 4 | import cursors.io.IOMemento 5 | import cursors.io.IOMemento.* 6 | import cursors.io.RowVec 7 | import kotlin.test.* 8 | import vec.macros.Vect02_.left 9 | import vec.macros.Vect0r 10 | import vec.macros.get 11 | import vec.util._v 12 | import java.nio.file.Files 13 | import java.nio.file.Path 14 | import java.nio.file.Paths 15 | import java.time.LocalDate 16 | import kotlin.streams.asSequence 17 | 18 | 19 | class CsvCursorTest { 20 | val dt: Vect0r< IOMemento> = _v[IoLocalDate, IoInt, IoString, IoString, IoString, IoString, IoString, IoString, 21 | IoString, IoString, IoString, IoInt, IoInt, IoInt] 22 | 23 | 24 | val path: Path = Paths.get("src/test/resources/calendar.csv") 25 | 26 | @Test 27 | fun HeapResidentCsvCursor() { 28 | val csvLines = Files.readAllLines(path) 29 | 30 | val curs = CsvArraysCursor(csvLines, dt) 31 | 32 | val testRow: RowVec = curs at 1 33 | 34 | val value = testRow.left[0] 35 | print(value) 36 | assertEquals(value, LocalDate.parse("2011-01-30")) 37 | } 38 | 39 | @Test 40 | fun ArrayCsvCursor() { 41 | 42 | val curs = CsvArraysCursor(Files.lines(path).asSequence().asIterable(), dt) 43 | 44 | val testRow: RowVec = curs at 1 45 | 46 | val value = testRow.left[0] 47 | print(value) 48 | assertEquals(value, LocalDate.parse("2011-01-30")) 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/cursors/io/Cursor.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import cursors.TypeMemento 5 | import cursors.at 6 | import cursors.context.Arity.Companion.arityKey 7 | import cursors.context.Scalar 8 | import vec.macros.* 9 | import vec.macros.Vect02_.right 10 | 11 | val Cursor.scalars: Vect0r 12 | get() = (this at 0).right α { it()[arityKey] as Scalar } 13 | 14 | val Cursor.width: Int get() = this.scalars.size 15 | val Cursor.colIdx: Vect02 get() = scalars α { sc: Scalar -> sc as Pai2 } 16 | 17 | fun networkCoords( 18 | ioMemos: Array, 19 | defaultVarcharSize: Int, 20 | varcharSizes: Map?, 21 | ): Vect02 = run { 22 | val sizes = networkSizes(ioMemos, defaultVarcharSize, varcharSizes) 23 | //todo: make IntArray Tw1nt Matrix 24 | var wrecordlen = 0 25 | val wcoords = Array(sizes.size) { ix -> 26 | (wrecordlen t2 (wrecordlen + sizes[ix]).also { wrecordlen = it }) 27 | } 28 | 29 | wcoords.map { tw1nt -> (-tw1nt).toList() }.flatten().toIntArray().let { ia -> 30 | Vect02(wcoords.size) { ix: Int -> 31 | val i = ix * 2 32 | val i1 = i + 1 33 | (ia[i] t2 ia[i1]) 34 | } 35 | } 36 | } 37 | 38 | fun networkSizes( 39 | ioMemos: Array, 40 | defaultVarcharSize: Int, 41 | varcharSizes: Map?, 42 | ): List = ioMemos.mapIndexed { ix, memento: TypeMemento -> 43 | val sz = varcharSizes?.get(ix) 44 | memento.networkSize ?: (sz ?: defaultVarcharSize) 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/vec/DoubleCirQlar.kt: -------------------------------------------------------------------------------- 1 | package vec 2 | 3 | import vec.macros.* 4 | import java.util.* 5 | 6 | /** 7 | 8 | stripped down circular 9 | 10 | only mutability is offer(T) 11 | 12 | has cheap direct toVect0r with live properties 13 | has more expensive toList/iterator by copy/concat 14 | */ 15 | class DoubleCirQlar( 16 | val maxSize: Int, 17 | val al: DoubleArray = DoubleArray(maxSize), 18 | ) : AbstractQueue() { 19 | 20 | var tail: Int = 0 21 | val full: Boolean get() = maxSize <= tail 22 | 23 | @Suppress("DeprecatedCallableAddReplaceWith") 24 | @Deprecated("gonna blow up on mutable ops") 25 | override fun iterator(): MutableIterator = this.toVect0r().iterator() as MutableIterator 26 | 27 | fun toList(): List = toVect0r().mapIndexedToList { _, t -> t } 28 | 29 | fun toVect0r(): Pai2 Double> = (this@DoubleCirQlar.size t2 { x: Int -> 30 | al[if (tail >= maxSize) { 31 | (tail + x) % maxSize 32 | } else x] 33 | }) 34 | 35 | //todo: lockless dequeue here ? 36 | override fun offer(e: Double): Boolean = 37 | synchronized(this) { 38 | al[tail % maxSize] = e 39 | tail++ 40 | if (tail == 2 * maxSize) tail = maxSize 41 | true 42 | } 43 | 44 | override fun poll(): Double = TODO("Not yet implemented") 45 | override fun peek(): Double = TODO("Not yet implemented") 46 | override fun add(k: Double): Boolean = offer(k) 47 | override val size: Int get() = kotlin.math.min(tail, maxSize) 48 | } -------------------------------------------------------------------------------- /src/main/java/vec/IntCirQlar.kt: -------------------------------------------------------------------------------- 1 | package vec 2 | 3 | import vec.macros.Pai2 4 | import vec.macros.`➤` 5 | import vec.macros.mapIndexedToList 6 | import vec.macros.t2 7 | import java.util.* 8 | 9 | /** 10 | 11 | stripped down circular 12 | 13 | only mutability is offer(T) 14 | 15 | has cheap direct toVect0r with live properties 16 | has more expensive toList/iterator by copy/concat 17 | */ 18 | class IntCirQlar( 19 | val maxSize: Int, 20 | val al: IntArray = IntArray(maxSize), 21 | ) : AbstractQueue() { 22 | var tail: Int = 0 23 | val full: Boolean get() = maxSize <= tail 24 | 25 | @Suppress("DeprecatedCallableAddReplaceWith") 26 | @Deprecated("gonna blow up on mutable ops") 27 | override fun iterator(): MutableIterator = this.toVect0r().`➤`.iterator() as MutableIterator 28 | 29 | fun toList(): List = toVect0r().mapIndexedToList { _, t -> t } 30 | 31 | fun toVect0r(): Pai2 Int> = this@IntCirQlar.size t2 { x: Int -> 32 | al[if (tail >= maxSize) { 33 | (tail + x) % maxSize 34 | } else x] 35 | } 36 | 37 | 38 | //todo: lockless dequeue here ? 39 | override fun offer(e: Int): Boolean = 40 | synchronized(this) { 41 | al[tail % maxSize] = e 42 | tail++ 43 | if (tail == 2 * maxSize) tail = maxSize 44 | true 45 | } 46 | 47 | override fun poll(): Int = TODO("Not yet implemented") 48 | override fun peek(): Int = TODO("Not yet implemented") 49 | override fun add(k: Int): Boolean = offer(k) 50 | override val size: Int get() = kotlin.math.min(tail, maxSize) 51 | } -------------------------------------------------------------------------------- /src/test/java/XPathTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import org.w3c.dom.Document 4 | import org.w3c.dom.NodeList 5 | import vec.macros.Vect0r 6 | import vec.macros.toList 7 | import java.io.FileInputStream 8 | import javax.xml.parsers.DocumentBuilderFactory 9 | import javax.xml.xpath.XPath 10 | import javax.xml.xpath.XPathConstants 11 | import javax.xml.xpath.XPathFactory 12 | import kotlin.test.Test 13 | 14 | 15 | /** 16 | * the simplest possible Vect0r array of an xpath query 17 | */ 18 | class XPathTest { 19 | @Test 20 | 21 | fun remap() { 22 | // curl -s 'https://en.wikipedia.org/wiki/List_of_largest_cities' >List_of_largest_cities.html 23 | // xmlstarlet sel -t -v '//*[@id="mw-content-text"]/div/table[2]/tbody/tr[position() > 2 ]/td[1]/a/text()' <(curl -s 'https://en.wikipedia.org/wiki/List_of_largest_cities') 24 | 25 | // val fileIS = (URL("https://en.wikipedia.org/wiki/List_of_largest_cities")).openConnection().getInputStream() 26 | val xmlDocument: Document = DocumentBuilderFactory.newInstance().newDocumentBuilder() 27 | .parse(FileInputStream("src/test/resources/List_of_largest_cities.html")) 28 | val xPath: XPath = XPathFactory.newInstance().newXPath() 29 | val expression = """//*[@id="mw-content-text"]/div/table[2]/tbody/tr[position() >= 3]/td[1]/a/text()""" 30 | val nodeList = xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET) as NodeList 31 | 32 | 33 | val cities = Vect0r(nodeList.length) { ix: Int -> 34 | nodeList.item(ix).textContent 35 | } 36 | 37 | System.err.println(cities.toList()) 38 | } 39 | } -------------------------------------------------------------------------------- /src/main/java/cursors/calendar/UnixTimeRemapper.kt: -------------------------------------------------------------------------------- 1 | package cursors.calendar 2 | 3 | //import cursors.io.left 4 | import cursors.Cursor 5 | import cursors.context.Arity 6 | import cursors.context.Scalar 7 | import cursors.context.Scalar.Companion.Scalar 8 | import cursors.get 9 | import cursors.io.IOMemento.IoInstant 10 | import cursors.io.RowVec 11 | import cursors.io.colIdx 12 | import cursors.macros.join 13 | import cursors.unaryMinus 14 | import vec.macros.size 15 | import vec.macros.t2 16 | import vec.macros.α 17 | import java.time.Instant 18 | 19 | class UnixTimeRemapper { 20 | companion object { 21 | /** 22 | * will select IoLong columns by name and project java.time.Instant column reordered into index 0..n joined 23 | * with the original cursor 24 | */ 25 | 26 | 27 | fun timestampFromIoLong(vararg timestampColumnNames: String): (Cursor) -> Cursor = { c0: Cursor -> 28 | val newKeys: IntArray = c0.colIdx.get(*timestampColumnNames) 29 | val thinned: IntArray = 30 | c0.colIdx.get(*timestampColumnNames.map(String::unaryMinus).toTypedArray()) //-"column" 31 | val leftovers: Cursor = c0[thinned] 32 | val c2: Cursor = c0[newKeys] α { rv: RowVec -> 33 | rv.size t2 rv.second α { (unixtime, desc) -> 34 | Instant.ofEpochMilli((unixtime as? Long) ?: unixtime.toString().toLongOrNull() ?: 0L) t2 { 35 | val (_, f) = desc()[Arity.arityKey] as Scalar 36 | Scalar(IoInstant, f) 37 | } 38 | } 39 | } 40 | join(c2, leftovers) 41 | } 42 | } 43 | } 44 | 45 | -------------------------------------------------------------------------------- /src/main/java/vec/util/Horizon.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNCHECKED_CAST") 2 | 3 | package vec.util 4 | 5 | import kotlin.math.PI 6 | import kotlin.math.max 7 | import kotlin.math.pow 8 | import kotlin.math.sin 9 | 10 | /** gradually compressed index accessor to underlying Cursor x values. 11 | */ 12 | 13 | @JvmOverloads 14 | fun horizon( 15 | index: Int, 16 | viewPoints: Int, 17 | datapoints: Int, 18 | dpDouble: Double = datapoints.toDouble(), 19 | vpDouble: Double = viewPoints.toDouble(), 20 | ): Int { 21 | return max( 22 | index, 23 | (dpDouble - 1 - sin((vpDouble - index) / vpDouble * (PI / 2.0)) * dpDouble - 1).toInt() 24 | ) 25 | } 26 | 27 | /** gradually compressed index accessor to underlying Cursor x values. */ 28 | @JvmOverloads 29 | fun hzInvSqr(index: Int, datapoints: Int, dpDub: Double = datapoints.toDouble(), vpDub: Double): Int = max( 30 | index, 31 | (1.0 / ((vpDub - index.toDouble()) * dpDub) * dpDub.pow(2.0)).toInt() 32 | ) 33 | 34 | fun main(args: Array) { 35 | 36 | val evens = args.map(String::toInt).zipWithNext() 37 | (0..evens.lastIndex step 2).map { x -> 38 | val (a, b) = evens[x] 39 | val arrayOfKFunctions: Array<(Int, Int, Int) -> Int> = _a[::horizon as (Int, Int, Int) -> Int , ::hzInvSqr as (Int, Int, Int) -> Int] 40 | arrayOfKFunctions.map { fn: (Int, Int, Int) -> Int -> 41 | println("--------$fn") 42 | println("showing $a of $b ${ 43 | (0 until a).map { 44 | val viewPoints = a 45 | val datapoints = b 46 | fn(it, viewPoints, datapoints) 47 | } 48 | }") 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /src/test/java/trie/TrieTest.kt: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | import kotlin.test.* 4 | 5 | 6 | /** 7 | * Created by kenny on 6/6/16. 8 | */ 9 | class TrieTest { 10 | val d: Int = Int.MIN_VALUE 11 | 12 | @Test 13 | fun stringTrie() { 14 | val trie = Trie() 15 | assertFalse(trie.contains(*emptyArray())) 16 | 17 | trie.add(d, "a", "b", "c") 18 | assertFalse(trie.contains("a")) 19 | assertFalse(trie.contains("a", "b")) 20 | assertTrue(trie.contains("a", "b", "c")) 21 | 22 | trie.add(d, "a", "b") 23 | assertTrue(trie.contains("a", "b")) 24 | 25 | trie.add(d, "a", "b", "d", "e") 26 | assertTrue(trie.contains("a", "b", "d", "e")) 27 | } 28 | /* 29 | @Test 30 | fun charTrie() { 31 | val trie = Trie() 32 | assertFalse(trie.contains(*emptyArray())) 33 | 34 | trie.add(d, 'a', 'b', 'c') 35 | assertFalse(trie.contains('a')) 36 | assertFalse(trie.contains('a', 'b')) 37 | assertTrue(trie.contains('a', 'b', 'c')) 38 | 39 | trie.add(d, 'a', 'b') 40 | assertTrue(trie.contains('a', 'b')) 41 | 42 | trie.add(d, 'a', 'b', 'd', 'e') 43 | assertTrue(trie.contains('a', 'b', 'd', 'e')) 44 | } 45 | 46 | @Test 47 | fun intTrie() { 48 | val trie = Trie() 49 | assertFalse(trie.contains(*emptyArray())) 50 | 51 | trie.add(d, 1, 2, 3) 52 | assertFalse(trie.contains(1)) 53 | assertFalse(trie.contains(1, 2)) 54 | assertTrue(trie.contains(1, 2, 3)) 55 | 56 | trie.add(d, 1, 2) 57 | assertTrue(trie.contains(1, 2)) 58 | 59 | trie.add(d, 1, 2, 4, 5) 60 | assertTrue(trie.contains(1, 2, 4, 5)) 61 | }*/ 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/vec/macros/SparseVect0r.kt: -------------------------------------------------------------------------------- 1 | package vec.macros 2 | 3 | import vec.macros.Vect02_.left 4 | import vec.macros.Vect02_.right 5 | import java.util.* 6 | 7 | 8 | /** 9 | * pay once for the conversion from a mutable map to an array map and all that implies 10 | */ 11 | inline fun Map.sparseVect0r(): SparseVect0r = run { 12 | val entries = 13 | ((this as? SortedMap)?.entries ?: entries.sortedBy { it.key }).toList() 14 | val sparse: Vect0r = bindSparse(entries α { (k, v) -> k t2 v }) 15 | SparseVect0r(sparse, entries) 16 | } 17 | 18 | inline fun bindSparse( 19 | driver: Vect02, 20 | ): Vect0r = driver.let { entres -> 21 | val k = driver.left.toIntArray() 22 | k.size t2 if (driver.size <= 16) //64 byte cacheline 23 | { x: Int -> 24 | var r: V? = null 25 | if (driver.size > 0) { 26 | var i = 0 27 | do { 28 | if (k[i] == x) 29 | r = entres.right[i] 30 | ++i 31 | } while (i < entres.size && r == null) 32 | } 33 | r 34 | } else { x: Int -> 35 | k.binarySearch(x).takeUnless { 36 | 0 > it 37 | }?.let { i -> entres.right[i] } 38 | } 39 | } 40 | 41 | /** 42 | * massive chimera 43 | */ 44 | class SparseVect0r( 45 | private val sparse: Vect0r, 46 | private val entries: List>, 47 | ) : Vect0r by sparse, Iterable> by (entries) { 48 | val left: Vect0r get() = entries α Map.Entry::key 49 | val right: Vect0r get() = entries α Map.Entry::value 50 | val keys: Vect0r by this::left 51 | val values: Vect0r by this::right 52 | } 53 | -------------------------------------------------------------------------------- /release.properties: -------------------------------------------------------------------------------- 1 | #release configuration 2 | #Thu Oct 07 04:09:37 SGT 2021 3 | projectVersionPolicyId=default 4 | project.scm.com.vsiwest.columnar\:columnar.tag=HEAD 5 | project.dev.com.vsiwest.columnar\:trie=1.0.2-SNAPSHOT 6 | project.rel.com.vsiwest.columnar\:exchange=1.0.1 7 | project.scm.com.vsiwest.columnar\:columnar.connection=scm\:git\:git@github.com\:jnorthrup/columnar.git 8 | project.rel.com.vsiwest.columnar\:cursor=1.0.1 9 | project.scm.com.vsiwest.columnar\:cursor.empty=true 10 | project.dev.com.vsiwest.columnar\:exchange=1.0.2-SNAPSHOT 11 | pushChanges=true 12 | project.rel.com.vsiwest.columnar\:lillypads=1.0.1 13 | project.scm.com.vsiwest.columnar\:columnar.developerConnection=scm\:git\:git@github.com\:jnorthrup/columnar.git 14 | project.dev.com.vsiwest.columnar\:meta-columnar=1.0.2-SNAPSHOT 15 | project.scm.com.vsiwest.columnar\:vector-like.empty=true 16 | remoteTagging=true 17 | scm.commentPrefix=[maven-release-plugin] 18 | project.scm.com.vsiwest.columnar\:lillypads.empty=true 19 | project.dev.com.vsiwest.columnar\:cursor=1.0.2-SNAPSHOT 20 | completedPhase=end-release 21 | scm.url=scm\:git\:git@github.com\:jnorthrup/columnar.git 22 | project.scm.com.vsiwest.columnar\:meta-columnar.url=git@github.com\:jnorthrup/columnar.git 23 | project.rel.com.vsiwest.columnar\:meta-columnar=1.0.1 24 | project.rel.com.vsiwest.columnar\:trie=1.0.1 25 | scm.tagNameFormat=@{project.artifactId}-@{project.version} 26 | project.scm.com.vsiwest.columnar\:trie.empty=true 27 | project.dev.com.vsiwest.columnar\:vector-like=1.0.2-SNAPSHOT 28 | project.dev.com.vsiwest.columnar\:lillypads=1.0.2-SNAPSHOT 29 | scm.tag=meta-columnar-1.0.1 30 | project.rel.com.vsiwest.columnar\:vector-like=1.0.1 31 | exec.snapshotReleasePluginAllowed=false 32 | preparationGoals=clean verify 33 | project.scm.com.vsiwest.columnar\:exchange.empty=true 34 | -------------------------------------------------------------------------------- /src/main/java/cursors/io/WriteCSV.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import vec.macros.Vect02_.left 5 | import vec.macros.Vect02_.right 6 | import vec.macros.size 7 | import vec.macros.toList 8 | import vec.util.FibonacciReporter 9 | import vec.util.debug 10 | import vec.util.logDebug 11 | import vec.util.path 12 | import java.nio.file.Files 13 | 14 | /** 15 | * writes a csv where true=1 and {null|false|0} ='' 16 | */ 17 | @JvmOverloads 18 | fun Cursor.writeCSV(filename: String, sep: Char = ',', eol: Char = '\n') { 19 | Files.newOutputStream(filename.path).bufferedWriter().use { fileWriter -> 20 | val reporter by lazy { FibonacciReporter(size, "Csv Rows") } 21 | val xsize = colIdx.size 22 | fileWriter.appendLine(colIdx.right.toList().joinToString(",")) 23 | for (iy in 0 until size) { 24 | val (_, row) = second(iy).left 25 | repeat(xsize) { ix -> 26 | if (0 < ix) fileWriter.append(sep) 27 | row(ix)?.also { cell1 -> 28 | //ignore 0's 29 | when (cell1) { 30 | is Boolean -> if (cell1) fileWriter.append('1') 31 | is Byte -> if (cell1 != 0.toByte()) fileWriter.append("$cell1") 32 | is Int -> if (cell1 != 0) fileWriter.append("$cell1") 33 | is Long -> if (cell1 != 0) fileWriter.append("$cell1") 34 | is Double -> if (cell1 != 0.0) fileWriter.append("$cell1") 35 | is Float -> if (cell1 != 0f) fileWriter.append("$cell1") 36 | else -> fileWriter.append("$cell1") 37 | } 38 | } 39 | } 40 | fileWriter.append(eol).debug { 41 | reporter.report(iy)?.debug { logDebug { it } } } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/test/java/vec/macros/VectorLikeKtTest.kt: -------------------------------------------------------------------------------- 1 | package vec.macros 2 | 3 | 4 | import vec.util._l 5 | import vec.util._v 6 | import vec.util.logDebug 7 | import kotlin.test.* 8 | 9 | class VectorLikeKtTest { 10 | 11 | @Test 12 | fun reverse() { 13 | assertTrue( 14 | _v[0, 1, 2, 3, 4, 5, 6].reverse.also { System.err.println(it.toList()) }.toList() 15 | .toIntArray().contentEquals(_l[0, 1, 2, 3, 4, 5, 6].reversed().toIntArray()) 16 | ) 17 | 18 | 19 | } 20 | 21 | @Test 22 | fun testTranspose() { 23 | val tme = _v[ 24 | _v[1, 2, 3, 4], //4, 5,100], 25 | _v[6, 7, 8, 9], //9, 10,101], 26 | _v[11, 12, 13, 14], //14, 15,102], 27 | ] 28 | val goal = _v[ 29 | _v[1, 6, 11], 30 | _v[2, 7, 12], 31 | _v[3, 8, 13], 32 | _v[4, 9, 14], 33 | ] 34 | val tist = tme.T 35 | (tme α Vect0r::toList).forEach { list: List -> 36 | System.err.println(list.toString()) 37 | } 38 | System.err.println("----") 39 | (goal α Vect0r::toList).forEach { list: List -> 40 | System.err.println(list.toString()) 41 | } 42 | System.err.println("----") 43 | (tist α Vect0r::toList).forEach { list: List -> 44 | System.err.println(list.toString()) 45 | } 46 | assertEquals(goal[1][0], tist[1][0]) 47 | assertEquals(goal[2][1], tist[2][1]) 48 | assertEquals(goal[3][2], tist[3][2]) 49 | assertEquals(goal[0][2], tist[0][2]) 50 | 51 | } 52 | 53 | @Test 54 | fun div() { 55 | 56 | val pai2 = (0 until 743 * 347 * 437) / 437 / 347 / 743 57 | logDebug { 58 | pai2[2] 59 | pai2.map { it.toList().toString() }.toList().toString() 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/vec/CirQlar.kt: -------------------------------------------------------------------------------- 1 | package vec 2 | 3 | import vec.macros.Pai2 4 | import vec.macros.size 5 | import vec.macros.t2 6 | import java.util.* 7 | 8 | /** 9 | 10 | stripped down circular 11 | 12 | only mutability is offer(T) 13 | 14 | has cheap direct toVect0r with live properties 15 | has more expensive toList/iterator by copy/concat 16 | */ 17 | open class CirQlar( 18 | val maxSize: Int, 19 | val evict: ((T) -> Unit)? =null 20 | ) : AbstractQueue() { 21 | private val al = arrayOfNulls(maxSize) 22 | var tail: Int = 0 23 | override val size: Int get() = kotlin.math.min(tail, maxSize) 24 | val full: Boolean get() = tail >= maxSize 25 | 26 | override fun offer(e: T): Boolean = synchronized(this) { 27 | val i = tail % maxSize 28 | val tmp =evict?.run { al.takeIf { it.size < i }?.get(i) } 29 | al[i] = e 30 | tail++ 31 | if (tail == 2 * maxSize) tail = maxSize 32 | tmp?.let {t-> evict?.invoke(t as T) } 33 | true 34 | 35 | } 36 | 37 | fun toList(): List { 38 | val iterator = iterator() 39 | return List(size) { 40 | val next = iterator.next() 41 | next 42 | } 43 | } 44 | 45 | override fun poll(): T = TODO("Not yet implemented") 46 | override fun peek(): T = TODO("Not yet implemented") 47 | override fun add(k: T): Boolean = offer(k) 48 | operator fun CirQlar.plus(k: T): Boolean = offer(k) 49 | operator fun CirQlar.plusAssign(k: T) { 50 | offer(k) 51 | } 52 | 53 | fun toVect0r(): Pai2 T> = ( size t2 { x: Int -> 54 | al[if (tail >= maxSize) { 55 | (tail + x) % maxSize 56 | } else x] as T 57 | }) 58 | 59 | override fun iterator(): MutableIterator = object : MutableIterator { 60 | val v = toVect0r() 61 | var i = 0 62 | override fun hasNext(): Boolean = i < v.size 63 | override fun next(): T = v.second(i++) 64 | override fun remove(): Unit = TODO("Not yet implemented") 65 | } 66 | } 67 | 68 | -------------------------------------------------------------------------------- /src/main/java/cursors/context/Arity.kt: -------------------------------------------------------------------------------- 1 | package cursors.context 2 | 3 | import cursors.TypeMemento 4 | import cursors.io.IOMemento 5 | import vec.macros.* 6 | import kotlin.coroutines.CoroutineContext 7 | 8 | sealed interface Arity : CoroutineContext.Element { 9 | override val key: CoroutineContext.Key get() = arityKey 10 | 11 | companion object { 12 | val arityKey: CoroutineContext.Key = object : 13 | CoroutineContext.Key {} 14 | } 15 | } 16 | 17 | interface Scalar : Arity, Pai2 { 18 | companion object { 19 | 20 | fun Scalar(p: Pai2): Scalar { 21 | val (a, b) = p 22 | return Scalar(a, b) 23 | } 24 | 25 | operator fun invoke(type: TypeMemento, name: String? = null): Scalar = Scalar(type, name) 26 | fun Scalar(type: TypeMemento, name: String? = null): Scalar = object : Scalar { 27 | override val first: TypeMemento 28 | get() = type 29 | override val second: String? 30 | get() = name 31 | } 32 | } 33 | 34 | val name: String 35 | get() = second 36 | ?: "generic${(first as? IOMemento)?.name ?: first::class.java.simpleName}:${first.networkSize}" 37 | } 38 | 39 | 40 | class Columnar(cols: Vect02) : 41 | Vect02 by cols, Arity { 42 | companion object { 43 | fun of(vararg type: TypeMemento): Columnar = Columnar(type α { t: TypeMemento -> t t2 null as String? }) 44 | 45 | @JvmName("fact2") 46 | fun of(scalars: Vect0r): Columnar { 47 | var c = 0 48 | val mapping: Vect02 = scalars α { (memento: TypeMemento, name: String?): Scalar -> 49 | val padStart = (c++).toString().padStart(6, '0') 50 | (name ?: "col$padStart") t2 memento 51 | } 52 | return Columnar(mapping.map { (a, b) -> b t2 a }) 53 | } 54 | } 55 | } 56 | 57 | class Variadic(val types: () -> Vect0r) : Arity 58 | -------------------------------------------------------------------------------- /src/main/java/trie/ArrayMap.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE") 2 | 3 | package trie 4 | 5 | import java.util.* 6 | 7 | 8 | class ArrayMap( 9 | val entre: Array>, 10 | val cmp: Comparator = Comparator { o1, o2 -> 11 | ((o1 as? Comparable)?.compareTo(o2 as K)) ?: o1.toString().compareTo(o2.toString()) 12 | }, 13 | ) : Map { 14 | override val entries: Set> 15 | get() = map { (k, v) -> 16 | object : Map.Entry { 17 | override val key get() = k 18 | override val value get() = v 19 | } 20 | }.toSet() 21 | override val size: Int get() = entre.size 22 | override val keys: Set get() = entre.map(Map.Entry::key).toSet() 23 | override val values: List get() = entre.map(Map.Entry::value) 24 | val comparator: Comparator> = entryComparator(cmp) 25 | 26 | override fun containsKey(key1: K): Boolean = 0 <= binIndexOf(key1) 27 | 28 | override fun containsValue(value: V): Boolean = entre.any { (_, v) -> Objects.equals(value, v) } 29 | override fun get(key1: K): V? = binIndexOf(key1).takeIf { it >= 0 }?.let { ix -> entre[ix].value } 30 | 31 | private fun binIndexOf(key1: K) = Arrays.binarySearch(entre, comparatorKeyShim(key1), comparator) 32 | 33 | fun comparatorKeyShim(key1: K): Map.Entry = ShimEntry(key1) 34 | 35 | override fun isEmpty(): Boolean = run(entre::isEmpty) 36 | 37 | companion object { 38 | fun entryComparator(comparator1: Comparator): Comparator> = 39 | Comparator> { (o1: K), (o2: K) -> comparator1.compare(o1, o2) } 40 | 41 | /** 42 | * if there aren't guarantees about ordered constructor entries, we can do a quick sort first on the comparator 43 | */ 44 | fun sorting( 45 | map: Map, 46 | cmp: Comparator = Comparator { o1, o2 -> o1.toString().compareTo(o2.toString()) }, 47 | ): ArrayMap = 48 | ArrayMap(map.entries.sortedWith(entryComparator(cmp)).toTypedArray(), cmp) 49 | } 50 | } 51 | 52 | class ShimEntry(private val key1: K) : Map.Entry { 53 | override val key: K get() = key1 54 | override val value: V get() = TODO("Not yet implemented") 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/cursors/macros/Join.kt: -------------------------------------------------------------------------------- 1 | package cursors.macros 2 | 3 | import cursors.Cursor 4 | import cursors.io.RowVec 5 | import cursors.io.scalars 6 | // Make sure all necessary vec.macros imports are present 7 | import cursors.io.CellMeta 8 | import vec.macros.* 9 | 10 | /** 11 | * Joins multiple Cursors column-wise. Assumes all cursors have the same number of rows. 12 | * This version takes a Vect0r of Cursors. 13 | */ 14 | fun join(vargs: Vect0r): Cursor { 15 | // Convert Vect0r to Array manually to avoid nullable spread operator issue 16 | val cursorArray = Array(vargs.size) { i -> vargs[i] } 17 | return join(*cursorArray) 18 | } 19 | 20 | /** 21 | * Joins multiple Cursors column-wise. Assumes all cursors have the same number of rows. 22 | * This version takes varargs Cursors. 23 | */ 24 | fun join(vararg vargs: Cursor): Cursor { 25 | // Check if there's anything to join 26 | if (vargs.isEmpty()) { 27 | throw IllegalArgumentException("Cannot join an empty list of cursors") 28 | } 29 | if (vargs.size == 1) { 30 | return vargs[0] // No need to join if only one cursor 31 | } 32 | 33 | // Original logic using muxIndexes 34 | return muxIndexes(vargs.map(Cursor::scalars)).let { (totalWidth, tails) -> 35 | val numRows = vargs[0].size // Assuming all cursors have the same number of rows 36 | 37 | object : Cursor { 38 | override val first: Int 39 | get() = numRows 40 | 41 | override val second: (Int) -> RowVec 42 | get() = { rowIndex: Int -> 43 | object : RowVec { 44 | override val first: Int 45 | get() = totalWidth 46 | 47 | override val second: (Int) -> Pai2 48 | get() = { columnIndex: Int -> 49 | val (sourceCursorIndex, indexInSourceCursor) = demuxIndex(tails, columnIndex) 50 | val sourceCursor = vargs[sourceCursorIndex] 51 | val sourceRow = sourceCursor.second(rowIndex) 52 | val cell = sourceRow.second(indexInSourceCursor) 53 | Pai2(cell.first, cell.second) 54 | } 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/vec/util/FibonacciReporter.kt: -------------------------------------------------------------------------------- 1 | @file:OptIn(kotlin.time.ExperimentalTime::class) 2 | 3 | package vec.util 4 | 5 | import kotlinx.datetime.Clock 6 | import kotlinx.datetime.TimeZone 7 | import kotlinx.datetime.* 8 | 9 | import kotlin.math.max 10 | import kotlin.time.* 11 | import kotlin.time.TimeSource 12 | 13 | /** 14 | * Reports progress using Fibonacci sequence for progress intervals. 15 | * @property totalSize Optional total size of the operation 16 | * @property itemName Name of the items being processed (default: "rows") 17 | */ 18 | class FibonacciReporter( 19 | private val totalSize: Int? = null, 20 | private val itemName: String = DEFAULT_ITEM_NAME, 21 | ) { 22 | private var sequencePosition: Int = 0 23 | private var remainingSteps: Int = 1 24 | private val startTime: TimeSource.Monotonic.ValueTimeMark = TimeSource.Monotonic.markNow() 25 | 26 | /** 27 | * Generates a progress report based on the current count. 28 | * @param currentCount Current number of processed items 29 | * @return Progress report string or null if it's not time to report 30 | */ 31 | fun report(currentCount: Int): String? { 32 | if (--remainingSteps > 0) return null 33 | 34 | remainingSteps = fib(++sequencePosition) 35 | val elapsedTime = startTime.elapsedNow() 36 | val timePerItem = elapsedTime / max(1, currentCount) 37 | 38 | return buildString { 39 | append(createBasicReport(currentCount, elapsedTime)) 40 | totalSize?.let { 41 | append(createEstimatedCompletion(currentCount, timePerItem)) 42 | } 43 | } 44 | } 45 | 46 | private fun createBasicReport(currentCount: Int, elapsedTime: Duration): String { 47 | val itemsPerSecond = (currentCount.toDouble() / elapsedTime.inWholeSeconds.toDouble()).toFloat() 48 | return "logged $currentCount $itemName in $elapsedTime $itemsPerSecond/s " 49 | } 50 | 51 | private fun createEstimatedCompletion(currentCount: Int, timePerItem: Duration): String { 52 | val remainingItems = totalSize!! - currentCount 53 | val estimatedTimeRemaining = timePerItem * remainingItems 54 | val estimatedCompletionTime = Clock.System.now() 55 | .plus(estimatedTimeRemaining) 56 | .toLocalDateTime(TimeZone.currentSystemDefault()) 57 | 58 | return "remaining: $estimatedTimeRemaining est $estimatedCompletionTime" 59 | } 60 | 61 | companion object { 62 | private const val DEFAULT_ITEM_NAME = "rows" 63 | } 64 | } -------------------------------------------------------------------------------- /src/test/resources/ninetyDegreesTest19334425141920886859.bin.meta: -------------------------------------------------------------------------------- 1 | # format: coords WS .. EOL names WS .. EOL TypeMememento WS .. 2 | # last coord is the recordlen 3 | 0 8 8 12 12 16 16 20 20 24 24 28 28 32 32 36 36 40 40 44 44 48 48 52 52 56 56 60 60 64 64 68 68 72 72 76 76 80 80 84 84 88 88 92 92 96 96 100 100 104 104 108 108 112 112 116 116 120 120 124 124 128 128 132 132 136 136 140 140 144 144 148 148 152 152 156 156 160 160 164 164 168 168 172 172 176 4 | date [channel=0101761/0101010207/13-14/01]:delivered [channel=0101761/0101010207/13-14/01]:ret [channel=0102211/0101010212/13-14/01]:delivered [channel=0102211/0101010212/13-14/01]:ret [channel=0500020/0101010106/13-14/05]:delivered [channel=0500020/0101010106/13-14/05]:ret [channel=0102237/0101010803/10-14/01]:delivered [channel=0102237/0101010803/10-14/01]:ret [channel=0200051/0101010106/13-14/02]:delivered [channel=0200051/0101010106/13-14/02]:ret [channel=0101848/0101010212/13-14/01]:delivered [channel=0101848/0101010212/13-14/01]:ret [channel=0100052/0101010106/13-14/01]:delivered [channel=0100052/0101010106/13-14/01]:ret [channel=0101462/0101010503/13-14/01]:delivered [channel=0101462/0101010503/13-14/01]:ret [channel=0100164/0101010106/13-14/01]:delivered [channel=0100164/0101010106/13-14/01]:ret [channel=0500067/0101010106/13-14/05]:delivered [channel=0500067/0101010106/13-14/05]:ret [channel=0200343/0101010106/13-16/02]:delivered [channel=0200343/0101010106/13-16/02]:ret [channel=0500414/0101010106/13-13/05]:delivered [channel=0500414/0101010106/13-13/05]:ret [channel=0101663/0101010209/13-14/01]:delivered [channel=0101663/0101010209/13-14/01]:ret [channel=0300067/0101010106/13-14/03]:delivered [channel=0300067/0101010106/13-14/03]:ret [channel=0102165/0101010803/10-13/01]:delivered [channel=0102165/0101010803/10-13/01]:ret [channel=0101718/0101010106/13-14/01]:delivered [channel=0101718/0101010106/13-14/01]:ret [channel=0500433/0101010206/13-14/05]:delivered [channel=0500433/0101010206/13-14/05]:ret [channel=0700048/0101010106/13-14/07]:delivered [channel=0700048/0101010106/13-14/07]:ret [channel=0101307/0101010802/20-15/01]:delivered [channel=0101307/0101010802/20-15/01]:ret [channel=0100865/0101010106/13-14/01]:delivered [channel=0100865/0101010106/13-14/01]:ret [channel=null]:delivered [channel=null]:ret 5 | IoLocalDate IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat IoFloat 6 | -------------------------------------------------------------------------------- /src/main/java/trie/Trie.kt: -------------------------------------------------------------------------------- 1 | package trie 2 | 3 | /** 4 | * Created by kenny on 6/6/16. 5 | */ 6 | class Trie(var root: Map = linkedMapOf()) { 7 | private var freeze: Boolean = false 8 | fun add(v: Int, vararg values: String) { 9 | var children = root 10 | for ((i, value) in values.withIndex()) { 11 | val isLeaf = i == values.size - 1 12 | // add new node 13 | if (!children.contains(value)) { 14 | val node = Node(value, isLeaf, v) 15 | (children as MutableMap)[value] = node 16 | children = node.children 17 | } else { 18 | // exist, so traverse current path + set isLeaf if needed 19 | val node = children[value]!! 20 | if (isLeaf != node.leaf) { 21 | node.leaf = isLeaf 22 | } 23 | children = node.children 24 | } 25 | } 26 | } 27 | 28 | fun contains(vararg values: String): Boolean = search(*values) != null 29 | 30 | operator fun get(vararg key: String): Int? = search(*key)?.payload 31 | 32 | fun frez(n: Node) { 33 | // if (n.leaf) return 34 | (n.children.entries).let { cnodes -> 35 | n.children = ArrayMap.sorting(n.children) 36 | for ((_, v) in cnodes) frez(v) 37 | } 38 | } 39 | 40 | fun freeze() { 41 | if (!freeze) { 42 | freeze = true 43 | root.values.forEach { frez(it) } 44 | root = ArrayMap.sorting(root) 45 | } 46 | } 47 | 48 | fun search(vararg segments: String): Node? { 49 | var children = root// exist, so traverse current path, ending if is last value, and is leaf node 50 | // not at end, continue traversing 51 | // add new node 52 | if (children.isNotEmpty()) { 53 | for ((i, value) in segments.withIndex()) { 54 | val atLeaf = i == segments.lastIndex 55 | // add new node 56 | if (children.contains(value)) { 57 | // exist, so traverse current path, ending if is last value, and is leaf node 58 | val node = children[value]!! 59 | if (atLeaf) return if (node.leaf) { 60 | node 61 | } else { 62 | null 63 | } 64 | // not at end, continue traversing 65 | children = node.children 66 | } else return null 67 | } 68 | throw IllegalStateException("Should not get here") 69 | } else return null 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/vec/ml/FeatureRange.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("NOTHING_TO_INLINE") 2 | 3 | package vec.ml 4 | 5 | import vec.macros.Tw1n 6 | import vec.macros.t2 7 | import kotlin.math.roundToInt 8 | import kotlin.math.roundToLong 9 | 10 | /** 11 | * returns min,max of Iterable given 12 | */ 13 | @JvmName("FRComparable") 14 | inline fun > featureRange(seq: Iterable, maxMinTwin: Tw1n): Tw1n = 15 | featureRange_(seq, maxMinTwin) 16 | 17 | /** 18 | * returns min,max of Iterable given 19 | */ 20 | @JvmName("FRComparable_") 21 | inline fun > featureRange_(seq: Iterable, maxMinTwin: Tw1n): Tw1n = 22 | seq.fold(maxMinTwin) { incumbent, candidate: T -> 23 | val (incumbentMin, incumbentMax) = incumbent 24 | val cmin = minOf(incumbentMin, candidate) 25 | val cmax = maxOf(candidate, incumbentMax) 26 | when { 27 | incumbentMax !== candidate && candidate === cmax || incumbentMin !== candidate && candidate === cmin -> Tw1n( 28 | cmin, 29 | cmax 30 | ) 31 | else -> incumbent 32 | } 33 | } 34 | 35 | /** 36 | * this is a min-max in that order 37 | */ 38 | inline fun Tw1n.normalize(d: Double): Double = let { (min, max) -> ((d - min) / (max - min)).toDouble() } 39 | inline fun Tw1n.normalize(d: Float): Double = let { (min, max) -> ((d - min) / (max - min)).toDouble() } 40 | inline fun Tw1n.normalize(d: Int): Double = let { (min, max) -> ((d - min) / (max - min)).toDouble() } 41 | inline fun Tw1n.normalize(d: Long): Double = let { (min, max) -> ((d - min) / (max - min)) .toDouble() } 42 | inline fun Tw1n.deNormalize(d: Double): Double = let { (min, max) -> (d * (max - min) + min) } 43 | inline fun Tw1n.deNormalize(d: Double): Float = let { (min, max) -> (d * (max - min) + min) }.toFloat() 44 | inline fun Tw1n.deNormalize(d: Double): Int = let { (min, max) -> (d * (max - min) + min) }.roundToInt() 45 | inline fun Tw1n.deNormalize(d: Double): Long = let { (min, max) -> (d * (max - min) + min) }.roundToLong() 46 | 47 | @JvmName("FrInt") 48 | fun featureRange( 49 | seq: Iterable, 50 | maxMinTwin: Tw1n = Int.MAX_VALUE t2 Int.MIN_VALUE, 51 | ): Tw1n = featureRange_(seq, maxMinTwin) 52 | 53 | 54 | @JvmName("FrLong") 55 | fun featureRange( 56 | seq: Iterable, 57 | maxMinTwin: Tw1n = Long.MAX_VALUE t2 Long.MIN_VALUE, 58 | ): Tw1n = featureRange_(seq, maxMinTwin) 59 | 60 | @JvmName("FrDouble") 61 | fun featureRange( 62 | seq: Iterable, 63 | maxMinTwin: Tw1n = Double.MAX_VALUE t2 Double.MIN_VALUE, 64 | ): Tw1n = featureRange_(seq, maxMinTwin) 65 | 66 | @JvmName("FrFloat") 67 | fun featureRange( 68 | seq: Iterable, 69 | maxMinTwin: Tw1n = Float.MAX_VALUE t2 Float.MIN_VALUE, 70 | ): Tw1n = featureRange_(seq, maxMinTwin) 71 | 72 | -------------------------------------------------------------------------------- /src/main/java/cursors/tbd/Gilbert.kt: -------------------------------------------------------------------------------- 1 | package cursors.tbd 2 | 3 | import java.lang.Integer.signum 4 | import kotlin.math.abs 5 | 6 | /** 7 | * 8 | * Generalized Hilbert ('gilbert') space-filling curve for arbitrary-sized 9 | * 2D rectangular grids. 10 | * 11 | * translated from: https://github.com/jakubcerveny/gilbert/blob/master/gilbert2d.py 12 | */ 13 | 14 | fun gilbertCurve(width: Int, height: Int, next: (Int, Int) -> Unit): Unit = 15 | if (width >= height) gilbertCurve2D(0, 0, width, 0, 0, height, next) else gilbertCurve2D( 16 | 0, 17 | 0, 18 | 0, 19 | height, 20 | width, 21 | 0, 22 | next 23 | ) 24 | 25 | fun gilbertCurve2D(xPrime: Int, yPrime: Int, ax: Int, ay: Int, bx: Int, by: Int, next: (Int, Int) -> Unit) { 26 | 27 | /** 28 | * performance may benefit from this being a Mutable Array passed in 29 | */ 30 | var x = xPrime 31 | var y = yPrime 32 | 33 | 34 | val w = abs(ax + ay) 35 | val h = abs(bx + by) 36 | val dax = signum(ax) 37 | val day = signum(ay) //unit major direction 38 | val dbx = signum(bx) 39 | val dby = signum(by) //unit orthogonal direction 40 | when { 41 | h == 1 -> { 42 | //trivial row fill 43 | for (i in 0 until w) { 44 | next(x, y) 45 | x += dax 46 | y += day 47 | } 48 | } 49 | w == 1 -> { 50 | //trivial column fill 51 | for (i in 0 until h) { 52 | next(x, y) 53 | x += dbx 54 | y += dby 55 | } 56 | } 57 | else -> { 58 | var ax2 = ax / 2 59 | var ay2 = ay / 2 60 | var bx2 = bx / 2 61 | var by2 = by / 2 62 | val w2 = abs(ax2 + ay2) 63 | val h2 = abs(bx2 + by2) 64 | when { 65 | 2 * w > 3 * h -> { 66 | if (w2 % 2 != 0 && w > 2) { 67 | //prefer even steps 68 | ax2 += dax 69 | ay2 += day 70 | } 71 | 72 | //long case: split in two parts only 73 | gilbertCurve2D(x, y, ax2, ay2, bx, by, next) 74 | gilbertCurve2D(x + ax2, y + ay2, ax - ax2, ay - ay2, bx, by, next) 75 | } 76 | else -> { 77 | if (h2 % 2 != 0 && h > 2) { 78 | //prefer even steps 79 | bx2 += dbx 80 | by2 += dby 81 | } 82 | 83 | //standard case: one step up, one long horizontal, one step down 84 | gilbertCurve2D(x, y, bx2, by2, ax2, ay2, next) 85 | gilbertCurve2D(x + bx2, y + by2, ax, ay, bx - bx2, by - by2, next) 86 | gilbertCurve2D( 87 | x + (ax - dax) + (bx2 - dbx), y + (ay - day) + (by2 - dby), 88 | -bx2, -by2, -(ax - ax2), -(ay - ay2), next 89 | ) 90 | } 91 | } 92 | } 93 | } 94 | } 95 | 96 | -------------------------------------------------------------------------------- /src/test/java/cursors/HeapCursorTest.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNCHECKED_CAST") 2 | 3 | package cursors 4 | 5 | import cursors.context.Scalar.Companion.Scalar 6 | import cursors.io.* 7 | import cursors.io.IOMemento.IoFloat 8 | import cursors.io.IOMemento.IoString 9 | import cursors.macros.`∑` 10 | import cursors.macros.join 11 | import kotlin.test.* 12 | import vec.macros.* 13 | import vec.util._a 14 | import vec.util._v 15 | import java.time.LocalDate 16 | 17 | data class row(val date: LocalDate, val channel: String, val delivered: Float, val ret: Float) 18 | 19 | class HeapCursorTest { 20 | val coords: Vect02 = _a[ 21 | 0, 10, 22 | 10, 84, 23 | 84, 124, 24 | 124, 164 25 | ].zipWithNext() 26 | 27 | val drivers: Vect0r = _v[ 28 | IOMemento.IoLocalDate, 29 | IoString, 30 | IoFloat, 31 | IoFloat 32 | ] 33 | 34 | 35 | val rows: Vect0r = _v[ 36 | 37 | row(LocalDate.of(2017, 10, 13), "0101761/0101010207/13-14/01", 88.0f, 0.0f), 38 | row(LocalDate.of(2017, 10, 22), "0102211/0101010212/13-14/01", 80.0f, 0.0f), 39 | row(LocalDate.of(2017, 10, 24), "0500020/0101010106/13-14/05", 4.0f, 0.0f), 40 | row(LocalDate.of(2017, 10, 22), "0500020/0101010106/13-14/05", 820.0f, 0.0f) 41 | ] 42 | 43 | val names: Vect0r = _v["date", "channel", "delivered", "ret"] 44 | 45 | 46 | val heapCursor: Cursor = Cursor(rows.size) { iy: Int -> 47 | rows[iy].run { 48 | RowVec(names.size) { ix: Int -> 49 | when (ix) { 50 | 0 -> component1() 51 | 1 -> component2() 52 | 2 -> component3() 53 | else -> component4() 54 | } as Any? t2 { Scalar(drivers[ix], names[ix]) } 55 | 56 | } 57 | } 58 | } 59 | 60 | @Test 61 | fun `resample+pivot+group+reduce+join`() { 62 | println("resample+group+reduce+join") 63 | val cursor: Cursor = heapCursor 64 | val resample = cursor.resample(0) 65 | resample.forEach { it -> 66 | println(it.map { vec -> 67 | "${ 68 | vec.component1().let { 69 | (it as? Vect0r<*>)?.toList() ?: it 70 | } 71 | }" 72 | }.toList()) 73 | } 74 | println("---") 75 | val grp = resample.group((1)) 76 | grp.forEach { it -> 77 | println(it.map { vec -> 78 | "${ 79 | vec.component1().let { 80 | (it as? Vect0r<*>)?.toList() ?: it 81 | } 82 | }" 83 | }.toList()) 84 | } 85 | println("---") 86 | val pai2 = grp[2, 3] 87 | val join: Cursor = join(grp[0, 1], pai2.`∑`(floatSum)) 88 | join.forEach { it -> 89 | println(it.map { vec -> 90 | "${ 91 | vec.component1().let { 92 | (it as? Vect0r<*>)?.toList() ?: it 93 | } 94 | }" 95 | }.toList()) 96 | } 97 | } 98 | 99 | } -------------------------------------------------------------------------------- /src/test/resources/caven20.fwf: -------------------------------------------------------------------------------- 1 | 2018-10-130101761/0101010207/13-14/01 88.000000000000 0E-12 2 | 2017-10-220102211/0101010212/13-14/01 80.000000000000 0E-12 3 | 2017-08-240500020/0101010106/13-14/05 4.000000000000 0E-12 4 | 2017-05-150102237/0101010803/10-14/01 820.000000000000 0E-12 5 | 2017-05-040200051/0101010106/13-14/02 2.000000000000 0E-12 6 | 2016-10-260101848/0101010212/13-14/01 260.000000000000 0E-12 7 | 2016-08-020100052/0101010106/13-14/01 100.000000000000 0E-12 8 | 2016-04-300101462/0101010503/13-14/01 4.000000000000 0E-12 9 | 2016-01-260100164/0101010106/13-14/01 2.000000000000 0E-12 10 | 2018-10-050500067/0101010106/13-14/05 4.000000000000 0E-12 11 | 2018-02-130200343/0101010106/13-16/02 80.000000000000 0E-12 12 | 2018-02-120500414/0101010106/13-13/05 20.000000000000 0E-12 13 | 2017-12-310101663/0101010209/13-14/01 20.000000000000 0E-12 14 | 2017-10-090300067/0101010106/13-14/03 8.000000000000 0E-12 15 | 2017-08-210102165/0101010803/10-13/01 20.000000000000 0E-12 16 | 2016-05-130101718/0101010106/13-14/01 220.000000000000 0E-12 17 | 2016-03-020500433/0101010206/13-14/05 88.000000000000 0E-12 18 | 2016-02-150700048/0101010106/13-14/07 1.000000000000 0E-12 19 | 2016-01-070101307/0101010802/20-15/01 80.000000000000 0E-12 20 | 2017-07-060100865/0101010106/13-14/01 4.000000000000 0E-12 21 | -------------------------------------------------------------------------------- /src/main/java/cursors/Categories.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.Scalar 4 | import cursors.io.IOMemento 5 | import cursors.io.RowVec 6 | import cursors.io.colIdx 7 | import cursors.io.scalars 8 | import cursors.macros.join 9 | import vec.macros.* 10 | import vec.macros.Vect02_.left 11 | import vec.macros.Vect02_.right 12 | import vec.ml.DummySpec 13 | import vec.util._a 14 | import vec.util._v 15 | import java.util.* 16 | import kotlin.coroutines.CoroutineContext 17 | 18 | 19 | /*** 20 | * 21 | * this creates a one-hot encoding set of categories for each value in each column. 22 | * 23 | * every distinct (column,value) permutation is reified as (as of this comment, expensive) in-place pivot 24 | * 25 | * TODO: staple the catx values to the cursor foreheads 26 | * 27 | * 28 | * maybe this is faster if each (column,value) pair was a seperate 1-column cursor. first fit for now. 29 | * 30 | * allocates IntArray per column, row length 31 | * 32 | */ 33 | fun Cursor.categories(dummySpec: Any? = null): Cursor = let { (psize, _) -> 34 | val colIdx = this.colIdx 35 | val (csize) = colIdx 36 | val typeMementos: Vect0r = colIdx.left.map { it } 37 | colIdx.right 38 | val alwaysZero = _a[0] 39 | 40 | 41 | Array(csize) { ix: Int -> 42 | 43 | val narrowed = this[ix] 44 | val (osize, oval) = narrowed.ordered(alwaysZero, IOMemento.listComparator(_v[typeMementos[ix]])) 45 | 46 | (0 until osize).map { oval(it).second(0).first }.distinct()./*toTypedArray().*/let { v -> 47 | 48 | Cursor(psize) { iy: Int -> 49 | 50 | val element = (narrowed at (iy))[0].first 51 | val useIndex = v.indexOf(element) 52 | 53 | RowVec(v.size) { vx: Int -> 54 | (useIndex == vx) as Any? t2 { 55 | @Suppress("USELESS_CAST") 56 | Scalar( 57 | IOMemento.IoBoolean, 58 | "${narrowed.scalars[0].second}=${v[vx]}" 59 | ) as CoroutineContext 60 | } 61 | } 62 | } 63 | }.let { curs: Cursor -> 64 | val s = curs.colIdx.size 65 | when { 66 | s == 1 || dummySpec == DummySpec.KeepAll -> curs 67 | else -> curs[(0 until s) - 68 | when (dummySpec) { 69 | is String -> curs.colIdx[dummySpec][0] 70 | is Int -> dummySpec 71 | DummySpec.Last -> curs.scalars.size - 1 72 | else -> 0 73 | }] 74 | } 75 | } 76 | }.let { join(it.size t2 it::get) } 77 | } 78 | 79 | fun Cursor.asBitSet(): Cursor { 80 | val xsize = colIdx.size 81 | val r = BitSet(size * xsize) 82 | repeat(size) { iy -> 83 | val rowVec = this at iy 84 | repeat(xsize) { ix -> 85 | if (true == (rowVec.second(ix).first as? Boolean)) 86 | r.set(iy * xsize + ix) 87 | } 88 | } 89 | val prep = this.scalars α { (Scalar(IOMemento.IoBoolean, it.name) as CoroutineContext).`⟲` } 90 | return this.size t2 { iy: Int -> 91 | xsize t2 { ix: Int -> 92 | r[iy * xsize + ix] t2 prep[ix] 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/cursors/calendar/JvmCal.kt: -------------------------------------------------------------------------------- 1 | package cursors.calendar 2 | 3 | import cursors.Cursor 4 | import cursors.at 5 | import cursors.context.Scalar.Companion.Scalar 6 | import cursors.io.IOMemento 7 | import cursors.io.RowVec 8 | import vec.macros.* 9 | import vec.macros.Vect02_.left 10 | import java.time.LocalDate 11 | import java.time.ZoneId 12 | import java.time.ZoneOffset 13 | import java.time.ZonedDateTime 14 | import java.time.chrono.* 15 | import java.time.temporal.ChronoField 16 | import java.time.temporal.UnsupportedTemporalTypeException 17 | import java.util.* 18 | 19 | /** 20 | * jvm supported calndars https://en.wikipedia.org/wiki/Category:Specific_calendars 21 | */ 22 | enum class JvmCal(val jvmProxy: Chronology) { 23 | Iso(IsoChronology.INSTANCE), 24 | ThaiBuddhist(ThaiBuddhistChronology.INSTANCE), 25 | Minguo(MinguoChronology.INSTANCE), 26 | Japanese(JapaneseChronology.INSTANCE), 27 | Hijrah(HijrahChronology.INSTANCE) 28 | ; 29 | 30 | data class dinfo( 31 | val id: String,//jvmProxy.id, 32 | val calendarType: String,// this.jvmProxy.calendarType, 33 | 34 | val dateEpochDay: ChronoLocalDate,// jvmProxy.dateEpochDay( 0) , 35 | val dateNow: ChronoLocalDate,// jvmProxy.dateNow(), 36 | val eras: MutableList, // jvmProxy.eras() 37 | ) 38 | 39 | fun info(): dinfo = dinfo( 40 | jvmProxy.id, 41 | this.jvmProxy.calendarType, 42 | jvmProxy.dateEpochDay(0), 43 | jvmProxy.dateNow(), 44 | jvmProxy.eras() 45 | ) 46 | 47 | fun date(localDate: LocalDate): ChronoLocalDate = jvmProxy.date( 48 | ZonedDateTime.ofInstant( 49 | localDate.atStartOfDay().toInstant( 50 | ZoneOffset.UTC 51 | ), ZoneId.systemDefault() 52 | ) 53 | ) 54 | 55 | 56 | /** 57 | * certain calendars should override this list 58 | * to reduce exception overheads 59 | */ 60 | open 61 | val dateCat: EnumSet 62 | get() = EnumSet.of( 63 | ChronoField.YEAR, 64 | ChronoField.MONTH_OF_YEAR, 65 | ChronoField.DAY_OF_MONTH, 66 | ChronoField.DAY_OF_WEEK, 67 | ChronoField.ALIGNED_WEEK_OF_MONTH 68 | ) 69 | 70 | open 71 | fun dateWiseCategories( 72 | localDate: LocalDate, 73 | ): Pai2 Pai2> = 74 | date(localDate).let { it: ChronoLocalDate? -> 75 | 76 | dateCat.mapNotNull { chronoField: ChronoField -> 77 | try { 78 | chronoField.name t2 (it!![chronoField]) 79 | } catch (t: UnsupportedTemporalTypeException) { 80 | null 81 | } 82 | }.toVect0r() 83 | } 84 | 85 | /** 86 | * returns a cursor the length of the passed in param1 87 | * @param calendarCurs source timeseries cursor 88 | * @param localdateIndex #calendarCurs index of LocalDate 89 | * @param catSize the length of categories to use defaulting to all avail 90 | */ 91 | fun inflate( 92 | calendarCurs: Cursor, 93 | localdateIndex: Int = 0, 94 | catSize: Int = dateWiseCategories(LocalDate.now()).size, 95 | ): Cursor = 96 | Cursor(calendarCurs.size) { iy: Int -> 97 | val rvec: RowVec = (calendarCurs at iy) 98 | val localDate: LocalDate = rvec.left[localdateIndex] as LocalDate 99 | val row: RowVec = dateWiseCategories(localDate) α { (nama: String, theVal: Int) -> 100 | theVal t2 { Scalar(IOMemento.IoInt, "$name:$nama") } 101 | } 102 | RowVec(row.size) { ix: Int -> row[ix] } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/cursors/context/Ordering.kt: -------------------------------------------------------------------------------- 1 | package cursors.context 2 | 3 | import cursors.io.TableRoot 4 | import vec.macros.Vect02 5 | import vec.macros.Vect02_.right 6 | import vec.macros.`→` 7 | import vec.macros.get 8 | import vec.macros.t2 9 | import java.nio.ByteBuffer 10 | import kotlin.coroutines.CoroutineContext 11 | 12 | /** 13 | * ordering arranges the row and column IO chunking to tailor the io access patterns. 14 | * 15 | * The composable options are 16 | * * Indexable Addressability Consumer 17 | * * linear Addressability provider 18 | * * coordinate system translations for such as pivot and inversion 19 | */ 20 | sealed class Ordering : CoroutineContext.Element { 21 | override val key: CoroutineContext.Key get() = orderingKey 22 | 23 | companion object { 24 | val orderingKey: CoroutineContext.Key = object : CoroutineContext.Key {} 25 | } 26 | } 27 | 28 | /** 29 | * [x++,y,z] 30 | * [x,y++,z] 31 | */ 32 | class RowMajor : Ordering() { 33 | companion object { 34 | fun fixedWidthOf( 35 | nio: NioMMap, 36 | coords: Vect02, 37 | defaulteol: () -> Byte = '\n'.code::toByte, 38 | ): FixedWidth = fixedWidth(nio, coords, defaulteol) 39 | 40 | fun fixedWidth( 41 | nio: NioMMap, 42 | coords: Vect02, 43 | defaulteol: () -> Byte, 44 | ): FixedWidth { 45 | return FixedWidth(recordLen = defaulteol() `→` { endl: Byte -> 46 | nio.mappedFile.mappedByteBuffer.duplicate().clear().run { 47 | while (get() != endl){} 48 | position() 49 | } 50 | }, coords = coords) 51 | } 52 | 53 | 54 | fun indexableOf( 55 | nio: NioMMap, fixedWidth: FixedWidth, 56 | mappedByteBuffer: ByteBuffer = nio.mappedFile.mappedByteBuffer, 57 | ): Indexable = 58 | Indexable(size = (nio.mappedFile.randomAccessFile.length() / fixedWidth.recordLen)::toInt) { recordIndex -> 59 | mappedByteBuffer.limit(fixedWidth.recordLen).slice().position(recordIndex * fixedWidth.recordLen) 60 | } 61 | 62 | //todo: move to rowMajor 63 | fun TableRoot.name(xy: IntArray): String? = this.let { (_, rootContext) -> 64 | (rootContext[Arity.arityKey]!! as Columnar).let { cnar -> 65 | cnar.right[(rootContext[orderingKey]!! as? ColumnMajor)?.let { xy[1] } ?: xy[0]] 66 | } 67 | } 68 | } 69 | 70 | /** 71 | * this builds a context and launches a cursor in the given NioMMap frame of reference 72 | */ 73 | fun fromFwf( 74 | fixedWidth: FixedWidth, 75 | indexable: Indexable, 76 | nio: NioMMap, 77 | columnarArity: Columnar, 78 | 79 | ): TableRoot = 80 | (this + fixedWidth + indexable + nio + columnarArity).let { coroutineContext -> nio.values(coroutineContext) t2 coroutineContext } 81 | 82 | /** 83 | * this builds a context and launches a cursor in the given NioMMap frame of reference 84 | */ 85 | fun fromBinary( 86 | fixedWidth: FixedWidth, 87 | indexable: Indexable, 88 | nio: NioMMap, 89 | columnarArity: Columnar, 90 | 91 | ): TableRoot = 92 | (this + fixedWidth + indexable + nio + columnarArity).let { coroutineContext -> nio.values(coroutineContext) t2 coroutineContext } 93 | 94 | } 95 | 96 | /** 97 | * [x,y++] 98 | * [x++,y] 99 | */ 100 | abstract class ColumnMajor : Ordering() 101 | 102 | /** 103 | * {x,y,z}+-(1|n|n^?)] 104 | */ 105 | abstract class Hilbert : Ordering() 106 | 107 | /**for seh 108 | * Associative||Abstract, Variadic, 109 | */ 110 | abstract class RTree : Ordering() 111 | -------------------------------------------------------------------------------- /src/main/java/cursors/io/RowVecExtensions.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNCHECKED_CAST") 2 | 3 | package cursors.io 4 | 5 | import cursors.io.IOMemento 6 | import cursors.context.Scalar 7 | import cursors.context.Arity 8 | import vec.macros.Vect02 9 | import kotlin.coroutines.CoroutineContext 10 | import kotlin.reflect.KClass 11 | 12 | /** 13 | * Extension functions for RowVec to provide type-safe access to cell values. 14 | * These functions use the metadata from CellMeta to ensure type safety before casting. 15 | * Client code is expected to inline type casts in inner loops for performance. 16 | * GraalVM specializations are proposed for lazy conversions and iterable promotion. 17 | */ 18 | // Removed duplicate typealias CellMeta to avoid redeclaration conflict 19 | 20 | /** 21 | * Get an Int value at the specified index with type safety. 22 | * Returns null if the type does not match or casting fails. 23 | * @GraalSpecialization: Lazy conversion can be optimized with GraalVM to defer casting until value is accessed. 24 | */ 25 | fun RowVec.getInt(index: Int): Int? { 26 | val (value, metaFn) = this.second(index) 27 | val type = metaFn().getTypeMemento() 28 | return if (type == IOMemento.IoInt) value as? Int else null 29 | } 30 | 31 | /** 32 | * Get a String value at the specified index with type safety. 33 | * Returns null if the type does not match or casting fails. 34 | * @GraalSpecialization: Lazy conversion can be optimized with GraalVM to defer casting until value is accessed. 35 | */ 36 | fun RowVec.getString(index: Int): String? { 37 | val (value, metaFn) = this.second(index) 38 | val type = metaFn().getTypeMemento() 39 | return if (type == IOMemento.IoString) value as? String else null 40 | } 41 | 42 | /** 43 | * Get a Float value at the specified index with type safety. 44 | * Returns null if the type does not match or casting fails. 45 | * @GraalSpecialization: Lazy conversion can be optimized with GraalVM to defer casting until value is accessed. 46 | */ 47 | fun RowVec.getFloat(index: Int): Float? { 48 | val (value, metaFn) = this.second(index) 49 | val type = metaFn().getTypeMemento() 50 | return if (type == IOMemento.IoFloat) value as? Float else null 51 | } 52 | 53 | /** 54 | * Get a Double value at the specified index with type safety. 55 | * Returns null if the type does not match or casting fails. 56 | * @GraalSpecialization: Lazy conversion can be optimized with GraalVM to defer casting until value is accessed. 57 | */ 58 | fun RowVec.getDouble(index: Int): Double? { 59 | val (value, metaFn) = this.second(index) 60 | val type = metaFn().getTypeMemento() 61 | return if (type == IOMemento.IoDouble) value as? Double else null 62 | } 63 | 64 | /** 65 | * Generic method to get a typed value at the specified index. 66 | * Returns null if the type does not match the expected type or casting fails. 67 | * @GraalSpecialization: Lazy conversion can be optimized with GraalVM for generic type casting. 68 | */ 69 | fun RowVec.getTyped(index: Int, expectedType: KClass, typeMemento: IOMemento): T? { 70 | val (value, metaFn) = this.second(index) 71 | val actualTypeMemento = metaFn().getTypeMemento() 72 | return if (actualTypeMemento == typeMemento) value as? T else null 73 | } 74 | 75 | /** 76 | * Helper function to extract TypeMemento from CoroutineContext. 77 | * This assumes the Scalar key contains the type information. 78 | */ 79 | fun CoroutineContext.getTypeMemento(): IOMemento? { 80 | val scalar = this[Arity.arityKey] as? Scalar 81 | return scalar?.first as? IOMemento 82 | } 83 | 84 | /** 85 | * Promote RowVec to an Iterable of typed values for a specific column type. 86 | * This method is intended for client code to use in inner loops with inline casting. 87 | * @GraalSpecialization: GraalVM can optimize iterable promotion for efficient iteration over large datasets. 88 | */ 89 | inline fun RowVec.asIterable(typeMemento: IOMemento): Iterable { 90 | return Iterable { 91 | object : Iterator { 92 | var index = 0 93 | override fun hasNext(): Boolean = index < first 94 | override fun next(): T? { 95 | return getTyped(index++, T::class, typeMemento) 96 | } 97 | } 98 | } 99 | } -------------------------------------------------------------------------------- /src/main/java/vec/util/BloomFilter.kt: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * credit to lovasoa https://github.com/lovasoa/bloomfilter/blob/master/src/main/java/BloomFilter.java 4 | package com.github.lovasoa.bloomfilter 5 | */ 6 | 7 | package vec.util 8 | 9 | import java.lang.Math.round 10 | import java.util.* 11 | import kotlin.math.roundToLong 12 | 13 | /** 14 | * bloom filter. 15 | * @param n Expected number of elements 16 | * @param m Desired size of the container in bits 17 | */ 18 | 19 | class BloomFilter(n: Int, m: Int = n * 11 /* good for 31 bit ints*/) : Cloneable { 20 | private var k // Number of hash functions 21 | : Int = (LN2 * m / n).roundToLong().toInt().let { if (it <= 0) 1 else it } 22 | private val hashes: BitSet = BitSet(m).also { logDebug { "bloomfilter for $n using $m bits" } } 23 | private val prng: RandomInRange = RandomInRange(m, k) 24 | 25 | 26 | /** 27 | * Add an element to the container 28 | */ 29 | fun add(o: Any) { 30 | prng.init(o) 31 | for (r in prng) hashes.set(r.value) 32 | } 33 | 34 | /** 35 | * If the element is in the container, returns true. 36 | * If the element is not in the container, returns true with a probability ≈ e^(-ln(2)² * m/n), otherwise false. 37 | * So, when m is large enough, the return value can be interpreted as: 38 | * - true : the element is probably in the container 39 | * - false : the element is definitely not in the container 40 | */ 41 | operator fun contains(o: Any): Boolean { 42 | prng.init(o) 43 | for (r in prng) if (!hashes[r.value]) return false 44 | return true 45 | } 46 | 47 | /** 48 | * Removes all of the elements from this filter. 49 | */ 50 | fun clear() { 51 | hashes.clear() 52 | } 53 | 54 | /** 55 | * Create a copy of the current filter 56 | */ 57 | @Throws(CloneNotSupportedException::class) 58 | public override fun clone(): BloomFilter = super.clone() as BloomFilter 59 | 60 | /** 61 | * Generate a unique hash representing the filter 62 | */ 63 | override fun hashCode(): Int = hashes.hashCode() xor k 64 | 65 | /** 66 | * Test if the filters have equal bitsets. 67 | * WARNING: two filters may contain the same elements, but not be equal 68 | * (if the filters have different size for example). 69 | */ 70 | fun equals(other: BloomFilter): Boolean = hashes == other.hashes && k == other.k 71 | 72 | /** 73 | * Merge another bloom filter into the current one. 74 | * After this operation, the current bloom filter contains all elements in 75 | * other. 76 | */ 77 | fun merge(other: BloomFilter) { 78 | require(!(other.k != k || other.hashes.size() != hashes.size())) { "Incompatible bloom filters" } 79 | hashes.or(other.hashes) 80 | } 81 | 82 | inner class RandomInRange internal constructor( 83 | // Maximum value returned + 1 84 | private val max: Int, // Number of random elements to generate 85 | private val count: Int, val seed: Long = Runtime.getRuntime().freeMemory(), 86 | ) : Iterable, MutableIterator { 87 | private val prng: Random = Random(seed).also { 88 | logDebug { "using randomSeeed $seed" } 89 | } 90 | private var i = 0 // Number of elements generated 91 | var value: Int // The current value 92 | = 0 93 | 94 | fun init(o: Any) { 95 | prng.setSeed(o.hashCode().toLong()) 96 | } 97 | 98 | override fun iterator(): Iterator { 99 | i = 0 100 | return this 101 | } 102 | 103 | override fun next(): RandomInRange { 104 | i++ 105 | value = prng.nextInt() % max 106 | if (value < 0) value = -value 107 | return this 108 | } 109 | 110 | override fun hasNext(): Boolean = i < count 111 | 112 | override fun remove() { 113 | throw UnsupportedOperationException() 114 | } 115 | 116 | 117 | } 118 | 119 | companion object { 120 | private const val LN2 = 0.6931471805599453 // ln(2) 121 | } 122 | 123 | } -------------------------------------------------------------------------------- /src/main/java/cursors/io/IOMemento.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.TypeMemento 4 | import vec.macros.Vect0r 5 | import vec.macros.get 6 | import vec.macros.map 7 | import vec.macros.α 8 | import java.nio.ByteBuffer 9 | import java.time.Instant 10 | import java.time.LocalDate 11 | import kotlin.time.ExperimentalTime 12 | 13 | const val SPACE: Byte = 32.toByte() 14 | const val ZERO: Float = 0.toFloat() 15 | 16 | val xInsertString: (ByteBuffer, String?) -> Unit = { a: ByteBuffer, b: String? -> 17 | a.put(b?.toByteArray(Charsets.UTF_8)) 18 | while (a.hasRemaining()) { 19 | a.put(SPACE) 20 | } 21 | } 22 | val dateMapper: (String) -> LocalDate = { s: String -> 23 | s.let { 24 | val res = try { 25 | LocalDate.parse(it) 26 | } catch (e: RuntimeException) { 27 | } catch (e: Throwable) { 28 | } finally { 29 | } 30 | res as? LocalDate 31 | } ?: LocalDate.EPOCH 32 | } 33 | val instantMapper: (String) -> Instant = { s: String -> 34 | s.let { 35 | val res = try { 36 | Instant.parse(it) 37 | } catch (e: RuntimeException) { 38 | } catch (e: Throwable) { 39 | } finally { 40 | } 41 | res as? Instant 42 | } ?: Instant.EPOCH 43 | } 44 | 45 | enum class IOMemento(override val networkSize: Int? = null) : TypeMemento { 46 | IoBoolean(1), 47 | IoByte(1), 48 | IoInt(4), 49 | IoLong(8), 50 | IoFloat(4), 51 | IoDouble(8), 52 | IoString, 53 | IoLocalDate(8), 54 | IoInstant(12), 55 | IoNothing 56 | ; 57 | 58 | companion object { 59 | var cmpMap: MutableMap Int> = linkedMapOf( 60 | IoLocalDate to { o1, o2 -> (o1 as LocalDate).compareTo(o2 as LocalDate) }, 61 | IoInstant to { o1, o2 -> (o1 as Instant).compareTo(o2 as Instant) }, 62 | IoBoolean to { o1, o2 -> (o1 as Boolean).compareTo(o2 as Boolean) }, 63 | IoByte to { o1, o2 -> (o1 as Int and 0xff).compareTo(o2 as Int and 0xff) }, 64 | IoInt to { o1, o2 -> (o1 as Int).compareTo(o2 as Int) }, 65 | IoLong to { o1, o2 -> (o1 as Long).compareTo(o2 as Long) }, 66 | IoFloat to { o1, o2 -> (o1 as Float).compareTo(o2 as Float) }, 67 | IoDouble to { o1, o2 -> (o1 as Double).compareTo(o2 as Double) } 68 | ) 69 | 70 | fun cmp(t: TypeMemento): (Any?, Any?) -> Int = cmpMap[t] ?: { o1: Any?, o2: Any? -> 71 | o1.toString().compareTo(o2.toString()) 72 | } 73 | 74 | fun listComparator(progression: Vect0r): java.util.Comparator> = Comparator> { o1, o2 -> 75 | val comp = progression.map(::cmp) 76 | var res = 0 77 | var idx = 0 78 | while (res == 0 && idx < o1.size) { 79 | val compare = comp[idx] 80 | res = compare(o1[idx], o2[idx]) 81 | idx++ 82 | } 83 | res 84 | } 85 | 86 | 87 | } 88 | } 89 | 90 | 91 | fun floatFillNa(fill: Float): (Any?) -> Any? { 92 | lateinit var x: (Any?) -> Any? 93 | @Suppress("UNCHECKED_CAST") val bound: (Any?) -> Any? = { testMe: Any? -> 94 | (testMe as? Vect0r<*>)?.α(x) ?: (testMe as? Iterable<*>)?.α(x) ?: (testMe as? Array<*>)?.α(x) ?: when (testMe) { 95 | Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, null -> fill 96 | else -> testMe 97 | } 98 | } 99 | x = bound 100 | return bound 101 | } 102 | 103 | 104 | val floatSum: (Any?, Any?) -> Any? = { acc: Any?, any2: Any? -> 105 | val fl = (acc as? Float) ?: ZERO 106 | val fl1 = (any2 as? Float) ?: ZERO 107 | fl + fl1 108 | } 109 | 110 | 111 | val sumReducer: Map Any?> = mapOf( 112 | IOMemento.IoInt to { acc: Any?, any2: Any? -> ((acc as? Int) ?: 0) + ((any2 as? Int) ?: 0) }, 113 | IOMemento.IoLong to { acc: Any?, any2: Any? -> 114 | ((acc as? Long) ?: 0.toLong()) + ((any2 as? Long) ?: 0.toLong()) 115 | }, 116 | IOMemento.IoFloat to floatSum, 117 | IOMemento.IoDouble to { acc: Any?, any2: Any? -> 118 | ((acc as? Double) ?: 0.toDouble()) + ((any2 as? Double) ?: 0.toDouble()) 119 | }, 120 | IOMemento.IoString to { acc: Any?, any2: Any? -> 121 | ((acc as? String) ?: 0.toString()) + ((any2 as? String) ?: 0.toString()) 122 | } 123 | ) -------------------------------------------------------------------------------- /src/main/java/cursors/context/RecordBoundary.kt: -------------------------------------------------------------------------------- 1 | package cursors.context 2 | 3 | import cursors.Cursor 4 | import cursors.TypeMemento 5 | import cursors.context.Scalar.Companion.Scalar 6 | import cursors.io.IOMemento 7 | import vec.macros.* 8 | import java.nio.ByteBuffer 9 | import kotlin.coroutines.CoroutineContext 10 | import kotlin.math.max 11 | 12 | /** 13 | * context-level support class for Fixed or Tokenized Record boundary conditions. 14 | * 15 | * 16 | * This does not extend guarantees to cell-level definitions-- 17 | * FWF cells are parsed strings, thus tokenized within fixed records, and may carry lineendings 18 | * csv records with fixed length fields is rare but has its place in aged messaging formats. 19 | * even fixed-record fixed-cell (e.g. purer ISAM) formats have to use external size variables for varchar 20 | * 21 | * it is assumed that record-level granularity and above are efficiently held in context details to perform outer loops 22 | * once the context variables are activated and conjoined per thier roles. 23 | * 24 | */ 25 | sealed class RecordBoundary : CoroutineContext.Element { 26 | companion object { 27 | val boundaryKey: CoroutineContext.Key = object : CoroutineContext.Key {} 28 | } 29 | 30 | override val key: CoroutineContext.Key get() = boundaryKey 31 | } 32 | 33 | 34 | class TokenizedRow(val tokenizer: (String) -> List) : RecordBoundary() { 35 | companion object { 36 | /** 37 | * same as CsvLinesCursor but does the splits at creationtime. 38 | * this does no quote escapes, handles no padding, and assumes row 0 is the header names. 39 | * 40 | * This Cursor scalar meta contains a compound CoroutineContext holding the max-length of Strings found in each 41 | * column. FixedWidth sealed class 42 | */ 43 | @JvmStatic 44 | @JvmOverloads 45 | fun CsvArraysCursor( 46 | csvLines1: Iterable, 47 | dt1: Vect0r = Vect0r(Int.MAX_VALUE) { IOMemento.IoString }, 48 | overrides: Map? = null, 49 | delimRegexFragment: Regex = ("\\s*,\\s*").toRegex(), 50 | ): Cursor { 51 | lateinit var longest: IntArray 52 | var colnames: Array 53 | lateinit var dt: Array 54 | 55 | val iter = csvLines1.iterator() 56 | 57 | run { 58 | val s = iter.next() 59 | val res = s.split(delimRegexFragment).map { "$it" }.toTypedArray() 60 | longest = IntArray(res.size) 61 | colnames = res 62 | dt = Array(colnames.size) { i -> 63 | overrides?.let { it[colnames[i]] } ?: dt1[i] 64 | } 65 | } 66 | 67 | val csvArrays = mutableListOf>() 68 | if (iter.hasNext()) 69 | do { 70 | val s = iter.next() 71 | csvArrays += s.split(delimRegexFragment, dt.size).map { it }.toTypedArray().mapIndexed { i, s -> 72 | when (val ioMemento = dt[i]) { 73 | IOMemento.IoString -> { 74 | longest[i] = max(longest[i], s.length) 75 | s 76 | } 77 | else -> Tokenized.mapped[ioMemento]!!.read(ByteBuffer.wrap(s.toByteArray()).rewind()) 78 | } 79 | }.toVect0r() 80 | } while (iter.hasNext()) 81 | 82 | val sdt = dt.mapIndexed { ix, dt -> 83 | (if (IOMemento.IoString == dt) { 84 | Scalar(type = dt, name = colnames[ix]) + FixedWidth( 85 | recordLen = longest[ix], 86 | coords = dummy, 87 | endl = { null }, 88 | pad = { null } 89 | ) //TODO: review whether using FixedWidth here is is a bad thing and we need a new Context Class for this feature. 90 | 91 | } else Scalar(dt, colnames[ix])).`⟲` 92 | }.toVect0r() 93 | 94 | return csvArrays α { it zip sdt } 95 | } 96 | } 97 | } 98 | 99 | private val dummy = vect0rOf>() 100 | 101 | class FixedWidth( 102 | val recordLen: Int, 103 | val coords: Vect02, 104 | val endl: () -> Byte? = { '\n'.code.toByte() }, 105 | val pad: () -> Byte? = { ' '.code.toByte() }, 106 | ) : RecordBoundary() 107 | -------------------------------------------------------------------------------- /src/test/java/VectorLikeKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | 4 | 5 | import kotlin.test.* 6 | 7 | import vec.macros.* 8 | import vec.util._a 9 | import vec.util._l 10 | 11 | 12 | infix fun Any?.shouldBe(that: Any?) { 13 | assertEquals(that, this) 14 | } 15 | 16 | class VectorLikeKtTest { 17 | @Test 18 | fun testCombineVec() { 19 | val c: Vect0r = combine( 20 | (0..2).toVect0r(), 21 | (3..5).toVect0r(), 22 | (6..9).toVect0r() 23 | ) 24 | val toList = c.toList() 25 | assertEquals("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]", toList.toString()) 26 | System.err.println(toList) 27 | } 28 | 29 | @Test 30 | fun testDiv() { 31 | val intRange = 0..11 32 | fun IntRange.split(nSubRanges: Int) = run { 33 | val subSize = (last - first + (1 - first)) / nSubRanges 34 | sequence { 35 | for (i in this@split step subSize) yield(i..minOf(last, i + subSize - 1)) 36 | } 37 | } 38 | System.err.println(intRange.toList()) 39 | System.err.println(intRange.last) 40 | val toList = intRange.split(3).toList() 41 | val toList1 = (intRange / 3).toList() 42 | System.err.println(toList to toList1) 43 | assertEquals(toList1, toList) 44 | } 45 | 46 | 47 | @Test 48 | fun WhatCanVect0rDo() { 49 | 50 | // we can create an array from bikeshed util 51 | val ar = _a["0", "1", "2"] 52 | 53 | //we can make a lazy vector from the array of strings 54 | val strVec = ar.toVect0r() 55 | 56 | // ## "convertables" 57 | // we have conversion code for sequences, flows, arrays, and List in both directions. 58 | strVec.toSequence() 59 | strVec.toFlow() 60 | strVec.toArray() 61 | strVec.toList() 62 | 63 | //we can make a lazy vector of ints from the array of strings. 64 | val intVec = strVec α String::toInt 65 | 66 | intVec.last shouldBe 2 67 | 68 | //we can combine them to create new indexed vet0r's 69 | val doubleLength = combine(intVec, intVec) 70 | doubleLength.size shouldBe 6 71 | 72 | 73 | // we can print them. there is no toString so we get an identity from an inline class 74 | System.err.println("double length vector is cold: $doubleLength") 75 | System.err.println() 76 | 77 | // we can reify them and then print that. 78 | System.err.println("double length vector is reified: ${doubleLength.toList()}") 79 | System.err.println() 80 | 81 | //we can destructure them to reach under the hood 82 | val (a: Int, _: (Int) -> Int) = doubleLength 83 | (a == doubleLength.size) shouldBe (doubleLength.size == doubleLength.first) 84 | 85 | //we can reorder them as a new Vect0r, lazily. the bikeshed util extends this to Any convertables above 86 | val reordered1 = doubleLength[2, 1, 3, 4, 4, 4, 4, 4] 87 | System.err.println(reordered1.toList()) 88 | System.err.println() 89 | 90 | //#### zip 91 | // same as the stdlib zip except the Vect0r result uses an interface instead of a data class 92 | // the pairwise construction uses the simplest possible `pair` interface named `Pai2` 93 | val zip: Vect02 = reordered1.zip(reordered1) 94 | 95 | // these are pure functional interfaces without toString, but have a pair conversion 96 | val z1 = zip α Pai2::pair 97 | System.err.println("z1: zip" + z1.toList()) 98 | 99 | //#### Vect0r.zipWithNext() 100 | //also similar to stdlib with a modified 101 | val zwn = combine(strVec, strVec).zipWithNext() 102 | val z2 = zwn α Pai2::pair 103 | System.err.println("z2: zwn " + z2.toList()) 104 | 105 | System.err.println() 106 | 107 | //## isomorphic pair interfaces 108 | // we use a simple inheritable Pair interface called Pai2 109 | val theInts = 0 t2 1 110 | val theIntPai = theInts.pair 111 | System.err.println("theIntPair:$theIntPai") 112 | 113 | //### union type casting 114 | // we get a free retype operation by having typalias of pair work across multiple sets of 115 | // extension functions... 116 | System.err.println(" theTw1n:" + theInts.pair) 117 | 118 | // we can still specialize based on type semantics in typealiases. powerful language to say the least. 119 | val theTw1n2: Tw1nt = Tw1n(0, 1) 120 | System.err.println(" theTw1n:" +/* */_l[theTw1n2.toString()]) 121 | 122 | 123 | // ### twins 124 | //as suggested there is a Twin in gscollections now called Eclipse-Collecitons 125 | //we name this Tw1n 126 | 127 | Tw1n("0", 1) 128 | 129 | //specialized overloads exist for primitive types 130 | 131 | Tw1n(0, 1) 132 | Tw1nt(_a[0, 1]) 133 | 134 | 135 | // #### isomorphisms: Vect0r>,Vect0r>, Vect02 136 | 137 | // these are all interchangable casts of the same Pai2: 138 | 139 | reordered1.zip(reordered1) 140 | reordered1.zip(reordered1) 141 | reordered1.zip(reordered1) 142 | // val vectTwint: Vect0r = reordered1.zip(reordered1) 143 | 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/test/java/vec/util/BloomFilterTest.kt: -------------------------------------------------------------------------------- 1 | package vec.util 2 | 3 | import kotlin.test.* 4 | import java.lang.management.ManagementFactory 5 | import java.lang.management.ThreadMXBean 6 | import java.util.* 7 | import kotlin.math.exp 8 | import kotlin.math.ln 9 | 10 | 11 | class TestBloomFilter { 12 | var elements: Int = 1000000 13 | var bitsize: Int = 10000000 14 | var filter: BloomFilter 15 | var prng: Random 16 | var bean: ThreadMXBean 17 | 18 | @Test 19 | fun testCorrectness() { 20 | println( 21 | """ 22 | Testing correctness. 23 | Creating a Set and filling it together with our filter... 24 | """.trimIndent() 25 | ) 26 | filter.clear() 27 | val inside: MutableSet = HashSet((elements / 0.75).toInt()) 28 | while (inside.size < elements) { 29 | val v = prng.nextInt() 30 | inside.add(v) 31 | filter.add(v) 32 | assert(filter.contains(v)) { "There should be no false negative" } 33 | } 34 | 35 | // testing 36 | var found = 0 37 | var total = 0 38 | var rate = 0.0 39 | while (total < elements) { 40 | val v = prng.nextInt() 41 | if (inside.contains(v)) continue 42 | total++ 43 | found += if (filter.contains(v)) 1 else 0 44 | rate = found.toFloat() / total.toDouble() 45 | if (total % 1000 == 0 || total == elements) { 46 | System.out.format( 47 | "\rElements incorrectly found to be inside: %8d/%-8d (%3.2f%%)", 48 | found, total, 100 * rate 49 | ) 50 | } 51 | } 52 | println("\n") 53 | val ln2 = ln(2.0) 54 | val expectedRate = exp(-ln2 * ln2 * bitsize / elements) 55 | assert(rate <= expectedRate * 1.10) { "error rate p = e^(-ln2^2*m/n)" } 56 | } 57 | 58 | @Test 59 | 60 | fun testInsertion() { 61 | println("Testing insertion speed...") 62 | filter.clear() 63 | val start = bean.currentThreadCpuTime 64 | for (i in 0 until elements) filter.add(prng.nextInt()) 65 | val end = bean.currentThreadCpuTime 66 | val time = end - start 67 | System.out.format( 68 | """ 69 | Inserted %d elements in %d ns. 70 | Insertion speed: %g elements/second 71 | 72 | 73 | """.trimIndent(), 74 | elements, 75 | time, 76 | elements / (time * 1e-9) 77 | ) 78 | } 79 | 80 | @Test 81 | 82 | fun testQuery() { 83 | println("Testing query speed...") 84 | filter.clear() 85 | for (i in 0 until elements) filter.add(prng.nextInt()) 86 | var xor = true // Make sure our result isn’t optimized out 87 | val start = bean.currentThreadCpuTime 88 | for (i in 0 until elements) xor = xor xor filter.contains(prng.nextInt()) 89 | val end = bean.currentThreadCpuTime 90 | val time = end - start 91 | System.out.format( 92 | """ 93 | Queried %d elements in %d ns. 94 | Query speed: %g elements/second 95 | 96 | 97 | """.trimIndent(), 98 | elements, 99 | time, 100 | elements / (time * 1e-9) 101 | ) 102 | } 103 | 104 | @Test 105 | @Throws(CloneNotSupportedException::class) 106 | fun testMerge() { 107 | print("Testing merge... ") 108 | filter.clear() 109 | val filter2 = filter.clone() 110 | for (i in 0 until elements) { 111 | var a: Int 112 | var b: Int 113 | filter.add(prng.nextInt().also { a = it }) 114 | filter2.add(prng.nextInt().also { b = it }) 115 | val concat = filter.clone() 116 | concat.merge(filter2) 117 | assert(concat.contains(a) && concat.contains(b)) { "merged filters don't lose elements" } 118 | } 119 | val concat1 = filter.clone() 120 | concat1.merge(filter2) 121 | val concat2 = filter2.clone() 122 | concat2.merge(filter) 123 | assert(concat1.equals(concat2)) { "a.merge(b) = b.merge(a)" } 124 | println("Done.\n") 125 | } 126 | 127 | /* 128 | companion object { 129 | 130 | fun main(args: Array) { 131 | val test = TestBloomFilter () 132 | if (args.size >= 1) test.elements = args[0].toInt() 133 | if (args.size >= 2) test.bitsize = args[1].toInt() 134 | test.testCorrectness() 135 | test.testInsertion() 136 | test.testQuery() 137 | } 138 | } 139 | 140 | */ 141 | init { 142 | System.out.format( 143 | """ 144 | Testing a bloom filter containing n=%d elements in a bit array of m=%d bits (=%.1fMib) 145 | 146 | 147 | """.trimIndent(), 148 | elements, bitsize, bitsize.toFloat() / (1024 * 1024 * 8) 149 | ) 150 | bean = ManagementFactory.getThreadMXBean() 151 | prng = Random() 152 | prng.setSeed(0) 153 | filter = BloomFilter(elements, bitsize) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/test/java/cursors/NinetyDegreeTest.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | 4 | import cursors.context.* 5 | import cursors.context.RowMajor.Companion.fixedWidthOf 6 | import cursors.context.RowMajor.Companion.indexableOf 7 | import cursors.io.* 8 | import cursors.io.IOMemento.* 9 | import cursors.macros.join 10 | import cursors.macros.α 11 | import kotlin.test.* 12 | import vec.macros.* 13 | import vec.macros.Vect02_.left 14 | import vec.macros.Vect02_.right 15 | import vec.util.toArray 16 | import java.io.File 17 | import java.nio.channels.FileChannel 18 | import java.nio.file.Paths 19 | import java.nio.file.StandardOpenOption 20 | 21 | class NinetyDegreeTest { 22 | 23 | 24 | /** 25 | * wrtie a fixed length networkendian binary fwf of a cursor. 26 | */ 27 | @Test 28 | fun rewriteBinFwfRowMajor() { 29 | val coords = intArrayOf( 30 | 0, 10, 31 | 10, 84, 32 | 84, 124, 33 | 124, 164 34 | ).zipWithNext() 35 | val mf = MappedFile("src/test/resources/caven20.fwf") 36 | val nio = NioMMap(mf) 37 | val fixedWidth = fixedWidthOf(nio, coords) 38 | val indexable: Addressable = indexableOf(nio, fixedWidth) 39 | //we resample and pivot a source cursor 40 | val piv: Cursor = cursorOf( 41 | RowMajor().fromFwf( 42 | fixedWidth, indexable as Indexable, nio, Columnar( 43 | vect0rOf( 44 | IoLocalDate as TypeMemento, 45 | IoString, 46 | IoFloat, 47 | IoFloat 48 | ).zip( 49 | vect0rOf("date", "channel", "delivered", "ret") 50 | ) 51 | ) 52 | ) 53 | ).resample(0).pivot(0.toArray(), 1.toArray(), intArrayOf(2, 3)) α floatFillNa(0f) 54 | val defaultVarcharSize = 64 55 | 56 | /** 57 | * open destination file 58 | */ 59 | val createTempFile = File.createTempFile("ninetyDegreesTest1", ".bin") 60 | System.err.println("tmpfile is " + createTempFile.toPath()) 61 | piv.writeISAM(createTempFile.absolutePath, defaultVarcharSize) 62 | } 63 | 64 | /** 65 | * wrtie a fixed length networkendian binary fwf of a cursor. 66 | */ 67 | @Test 68 | fun shardCursor() { 69 | val coords = intArrayOf( 70 | 0, 10, 71 | 10, 84, 72 | 84, 124, 73 | 124, 164 74 | ).zipWithNext() 75 | val mf = MappedFile("src/test/resources/caven20.fwf") 76 | val nio = NioMMap(mf) 77 | val fixedWidth = fixedWidthOf(nio, coords) 78 | val indexable: Addressable = indexableOf(nio, fixedWidth) 79 | //we resample and pivot a source cursor 80 | val piv: Cursor = cursorOf( 81 | RowMajor().fromFwf( 82 | fixedWidth, indexable as Indexable, nio, Columnar( 83 | vect0rOf( 84 | IoLocalDate as TypeMemento, 85 | IoString, 86 | IoFloat, 87 | IoFloat 88 | ).zip( 89 | vect0rOf("date", "channel", "delivered", "ret") 90 | ) 91 | ) 92 | ) 93 | ).resample(0).pivot(0.toArray(), 1.toArray(), intArrayOf(2, 3)) α floatFillNa(0f) 94 | 95 | val fnList = piv.scalars.mapIndexedToList { i, scalar -> 96 | 97 | 98 | System.err.println("writing " + scalar.name) 99 | /** 100 | * open destination file 101 | */ 102 | val createTempFile = File.createTempFile( 103 | "shardedPiv-" + scalar.name.replace( 104 | "\\W+".toRegex(), 105 | "_" 106 | ), 107 | ".bin" 108 | ) 109 | System.err.println("tmpfile is " + createTempFile.toPath()) 110 | val pathname = createTempFile.absolutePath 111 | piv[i].writeISAM(pathname) 112 | pathname 113 | } 114 | System.err.println(fnList) 115 | 116 | val handleMapThing: Vect02 = fnList.α { 117 | val fc = FileChannel.open(Paths.get(it), StandardOpenOption.READ) 118 | fc t2 ISAMCursor(Paths.get(it), fc) 119 | } 120 | 121 | try { 122 | val c: Vect0r = handleMapThing.right 123 | val piv2 = join(c) 124 | 125 | 126 | for (i in 0 until piv.size) for (j in 0 until piv.scalars.size) 127 | (piv at i)[j].first shouldBe (piv2 at (i))[j].first 128 | 129 | } finally { 130 | try { 131 | handleMapThing.left.forEach { it.close() } 132 | } catch (_: Throwable) { 133 | } 134 | } 135 | 136 | } 137 | 138 | @Test 139 | fun readBinary() { 140 | val s1 = "ninetyDegreesTest19334425141920886859.bin" 141 | "$s1.meta" 142 | val binpath = Paths.get( 143 | "src", 144 | "test", 145 | "resources", s1 146 | ) 147 | 148 | FileChannel.open(binpath, StandardOpenOption.READ).use { mf -> 149 | val cursr = ISAMCursor(binpath, mf) 150 | 151 | System.err.println((cursr at (0)).left.toList()) 152 | System.err.println((cursr at (1)).left.toList()) 153 | System.err.println((cursr at (2)).left.toList()) 154 | System.err.println((cursr at (3)).left.toList()) 155 | } 156 | } 157 | 158 | } 159 | -------------------------------------------------------------------------------- /src/test/java/cursors/io/ISAMCursorKtTest.kt: -------------------------------------------------------------------------------- 1 | package cursors.io 2 | 3 | import cursors.Cursor 4 | import cursors.at 5 | import cursors.context.Scalar.Companion.Scalar 6 | import kotlin.test.* 7 | import vec.macros.`⟲` 8 | import vec.macros.get 9 | import vec.macros.size 10 | import vec.macros.t2 11 | import vec.util._v 12 | import vec.util.path 13 | import java.nio.channels.FileChannel 14 | import java.time.Instant 15 | import kotlin.random.Random 16 | 17 | class ISAMCursorKtTest { 18 | @Test 19 | fun testIoInstant() { 20 | 21 | val testInstant = Instant.now() 22 | System.err.println(testInstant) 23 | val c0: Cursor = _v[_v[testInstant t2 Scalar(IOMemento.IoInstant, "ASDF").`⟲` 24 | ]] 25 | val fname = "target/IoInstant.isam" 26 | c0.also { c: Cursor -> 27 | c.writeISAM(fname) 28 | } 29 | FileChannel.open(fname.path).use { fc -> 30 | val c2 = ISAMCursor(fname.path, fc) 31 | assertEquals((c0 at 0)[0].first as Instant, (c2 at 0)[0].first as Instant) 32 | assertEquals((c0.size), (c2.size)) 33 | System.err.println((c2 at 0)[0].first as Instant) 34 | } 35 | } 36 | 37 | @Test 38 | fun testIoInstant2() { 39 | 40 | val testInstant = Instant.now() 41 | System.err.println(testInstant) 42 | val c0: Cursor = cursors.Cursor(2) { y: Int -> 43 | RowVec(7) { x: Int -> 44 | testInstant t2 Scalar(IOMemento.IoInstant, "ASDF").`⟲` 45 | } 46 | } 47 | 48 | val fname = "target/IoInstant.isam" 49 | c0.also { c: Cursor -> 50 | c.writeISAM(fname) 51 | } 52 | FileChannel.open(fname.path).use { fc -> 53 | val c2 = ISAMCursor(fname.path, fc) 54 | assertEquals((c0 at 0)[0].first as Instant, (c2 at 0)[0].first as Instant) 55 | assertEquals((c0.size), (c2.size)) 56 | assertEquals((c0 at 0)[1].first as Instant, (c2 at 0)[1].first as Instant) 57 | assertEquals((c0 at 0)[2].first as Instant, (c2 at 0)[2].first as Instant) 58 | assertEquals((c0 at 0)[3].first as Instant, (c2 at 0)[3].first as Instant) 59 | assertEquals((c0 at 0)[4].first as Instant, (c2 at 0)[4].first as Instant) 60 | assertEquals((c0 at 0)[5].first as Instant, (c2 at 0)[5].first as Instant) 61 | assertEquals((c0 at 0)[6].first as Instant, (c2 at 0)[6].first as Instant) 62 | assertEquals((c0 at 1)[1].first as Instant, (c2 at 1)[1].first as Instant) 63 | assertEquals((c0 at 1)[2].first as Instant, (c2 at 1)[2].first as Instant) 64 | assertEquals((c0 at 1)[3].first as Instant, (c2 at 1)[3].first as Instant) 65 | assertEquals((c0 at 1)[4].first as Instant, (c2 at 1)[4].first as Instant) 66 | assertEquals((c0 at 1)[5].first as Instant, (c2 at 1)[5].first as Instant) 67 | assertEquals((c0 at 1)[6].first as Instant, (c2 at 1)[6].first as Instant) 68 | System.err.println((c2 at 0)[5].first as Instant) 69 | } 70 | } 71 | 72 | @Test 73 | fun testIoInstant3() { 74 | 75 | val testInstant = Instant.now() 76 | val testDouble = Random.nextDouble() 77 | System.err.println(testInstant) 78 | System.err.println(testDouble) 79 | val c0: Cursor = cursors.Cursor(2) { y: Int -> 80 | RowVec(7) { x: Int -> 81 | when (x.rem(2)) { 82 | 0 -> testInstant t2 Scalar(IOMemento.IoInstant, "ASDF").`⟲` 83 | else -> testDouble t2 Scalar(IOMemento.IoDouble, "JOID").`⟲` 84 | 85 | 86 | } 87 | } 88 | } 89 | 90 | val fname = "target/IoInstant.isam" 91 | c0.also { c: Cursor -> 92 | c.writeISAM(fname) 93 | } 94 | FileChannel.open(fname.path).use { fc -> 95 | val c2 = ISAMCursor(fname.path, fc) 96 | assertEquals((c0 at 0)[0].first as Instant, (c2 at 0)[0].first as Instant) 97 | assertEquals((c0.size), (c2.size)) 98 | assertEquals((c0 at 0)[1].first, (c2 at 0)[1].first) 99 | assertEquals((c0 at 0)[2].first, (c2 at 0)[2].first) 100 | assertEquals((c0 at 0)[3].first, (c2 at 0)[3].first) 101 | assertEquals((c0 at 0)[4].first, (c2 at 0)[4].first) 102 | assertEquals((c0 at 0)[5].first, (c2 at 0)[5].first) 103 | assertEquals((c0 at 0)[6].first, (c2 at 0)[6].first) 104 | assertEquals((c0 at 1)[1].first, (c2 at 1)[1].first) 105 | assertEquals((c0 at 1)[2].first, (c2 at 1)[2].first) 106 | assertEquals((c0 at 1)[3].first, (c2 at 1)[3].first) 107 | assertEquals((c0 at 1)[4].first, (c2 at 1)[4].first) 108 | assertEquals((c0 at 1)[5].first, (c2 at 1)[5].first) 109 | assertEquals((c0 at 1)[6].first, (c2 at 1)[6].first) 110 | System.err.println((c2 at 1)[5].first) 111 | } 112 | } 113 | 114 | @Test 115 | fun testIoInstant4() { 116 | 117 | val testInstant = Instant.now() 118 | val testDouble = Random.nextDouble() 119 | val testInt = Random.nextInt() 120 | System.err.println(testInstant) 121 | System.err.println(testDouble) 122 | System.err.println(testInt) 123 | val c0: Cursor = cursors.Cursor(2) { y: Int -> 124 | RowVec(7) { x: Int -> 125 | when (x.rem(2)) { 126 | 0 -> testInstant t2 Scalar(IOMemento.IoInstant, "ASDF").`⟲` 127 | 1 -> testInt t2 Scalar(IOMemento.IoInt, "ESXA").`⟲` 128 | else -> testDouble t2 Scalar(IOMemento.IoDouble, "JOID").`⟲` 129 | 130 | 131 | } 132 | } 133 | } 134 | 135 | val fname = "target/IoInstant.isam" 136 | c0.also { c: Cursor -> 137 | c.writeISAM(fname) 138 | } 139 | FileChannel.open(fname.path).use { fc -> 140 | val c2 = ISAMCursor(fname.path, fc) 141 | assertEquals((c0 at 0)[0].first as Instant, (c2 at 0)[0].first as Instant) 142 | assertEquals((c0.size), (c2.size)) 143 | assertEquals((c0 at 0)[1].first, (c2 at 0)[1].first) 144 | assertEquals((c0 at 0)[2].first, (c2 at 0)[2].first) 145 | assertEquals((c0 at 0)[3].first, (c2 at 0)[3].first) 146 | assertEquals((c0 at 0)[4].first, (c2 at 0)[4].first) 147 | assertEquals((c0 at 0)[5].first, (c2 at 0)[5].first) 148 | assertEquals((c0 at 0)[6].first, (c2 at 0)[6].first) 149 | assertEquals((c0 at 1)[1].first, (c2 at 1)[1].first) 150 | assertEquals((c0 at 1)[2].first, (c2 at 1)[2].first) 151 | assertEquals((c0 at 1)[3].first, (c2 at 1)[3].first) 152 | assertEquals((c0 at 1)[4].first, (c2 at 1)[4].first) 153 | assertEquals((c0 at 1)[5].first, (c2 at 1)[5].first) 154 | assertEquals((c0 at 1)[6].first, (c2 at 1)[6].first) 155 | System.err.println((c2 at 1)[5].first) 156 | } 157 | } 158 | } -------------------------------------------------------------------------------- /src/main/java/cursors/Clusters.kt: -------------------------------------------------------------------------------- 1 | package cursors 2 | 3 | import cursors.context.Scalar 4 | import cursors.hash.md4 5 | import cursors.io.* 6 | import trie.ArrayMap 7 | import trie.Trie 8 | import vec.macros.* 9 | import vec.macros.Vect02_.left 10 | import vec.macros.Vect02_.right 11 | import vec.util.BloomFilter 12 | import vec.util._m 13 | import java.util.* 14 | import kotlin.math.acos 15 | import kotlin.math.max 16 | import kotlin.math.sqrt 17 | 18 | 19 | fun bloomAccess(groupClusters: Iterable): Vect02 = groupClusters α { 20 | val n = it.size 21 | val bloomFilter = BloomFilter(n/*, n * 11*/).apply { for (i in it) ::add } 22 | bloomFilter t2 it 23 | } 24 | 25 | 26 | /** 27 | * group on indexes created using the columnNames of the cursor passed in 28 | * 29 | */ 30 | fun Cursor.group( 31 | /**these columns will be preserved as the cluster key. 32 | * the remaining indexes will be aggregate. 33 | * 34 | * setting the precedent here where curs[[-]foo"] is adequate to convey a longer scala2s extraction 35 | */ 36 | axis: Cursor, 37 | ): Cursor = group(*colIdx.get(*axis.colIdx.right.toList().filterNotNull().toTypedArray()).toTypedArray().toIntArray()) 38 | 39 | 40 | fun Cursor.group( 41 | /**these columns will be preserved as the cluster key. 42 | * the remaining indexes will be aggregate 43 | */ 44 | vararg axis: Int, 45 | ): Cursor = let { orig -> 46 | val clusters = groupClusters(axis) 47 | val masterScalars = orig.scalars 48 | Cursor(clusters.size) { cy -> 49 | val cluster = clusters[cy] 50 | val cfirst = cluster.first() 51 | RowVec(masterScalars.first) { ix: Int -> 52 | when (ix) { 53 | in axis -> (orig at (cfirst))[ix] 54 | else -> vec.macros.Vect0r(cluster.size) { iy: Int -> (orig at (cluster[iy]))[ix].first } t2 { masterScalars[ix] } 55 | } 56 | } 57 | } 58 | } 59 | 60 | inline fun Cursor.group( 61 | /**these columns will be preserved as the cluster key. 62 | * the remaining indexes will be aggregate 63 | */ 64 | axis: IntArray, 65 | crossinline reducer: ((Any?, Any?) -> Any?), 66 | ): Cursor = run { 67 | val clusters = groupClusters(axis) 68 | val masterScalars = scalars 69 | val xSize = masterScalars.first 70 | val ac =if(axis.size>16)axis.toSet()else axis.toList() 71 | Cursor(clusters.size) { cy -> 72 | val acc1 = arrayOfNulls(xSize) 73 | val cluster = clusters[cy] 74 | val keyIndex = cluster.first() 75 | val valueIndices = acc1.indices - ac 76 | 77 | for (i in cluster) { 78 | val value = (this at i).left 79 | for (valueIndex in valueIndices) 80 | acc1[valueIndex] = reducer(acc1[valueIndex], value[valueIndex]) 81 | } 82 | RowVec(masterScalars.first) { ix: Int -> 83 | if (ix in axis) { 84 | (this at (keyIndex))[ix] 85 | } else acc1[ix] t2 masterScalars[ix].`⟲` 86 | } 87 | } 88 | } 89 | 90 | 91 | /** 92 | * Performs [Cursor.keyClusters] and returns trimmed values for group clusters 93 | */ 94 | fun Cursor.groupClusters( 95 | axis: IntArray, 96 | clusters: MutableMap, MutableList> = linkedMapOf(), 97 | ): Pai2 IntArray> = run { 98 | System.err.println("--- groupClusters") 99 | keyClusters(axis, clusters) 100 | clusters.values α MutableList::toIntArray 101 | } 102 | 103 | /** 104 | * grows a list for each cluster key using a bit of a guess on median capacity -- does not trim the clusters. 105 | */ 106 | fun Cursor.keyClusters( 107 | axis: IntArray, 108 | clusters: MutableMap, MutableList>, 109 | ): MutableMap, MutableList> = clusters.apply { 110 | val cap = max(8, sqrt(size.toDouble()).toInt()) 111 | forEachIndexed { iy: Int, row: RowVec -> row[axis].left.toList().let { getOrPut(it) { ArrayList(cap) } += iy } } 112 | // logDebug { "cap: $cap keys:${clusters.size to clusters.keys}" } been stable for a while 113 | } 114 | 115 | /** 116 | * ordered keys of ordered cluster indexes trimmed. 117 | */ 118 | fun Cursor.mapClusters(axis: IntArray) = 119 | _m[keyClusters(axis, linkedMapOf()).entries α { (a,b)-> a t2 b.toIntArray()}] 120 | 121 | 122 | /** 123 | * /primary/ key mapping. collision behaviors are map-defined 124 | * produces more idealized hash bucket distributions 125 | */ 126 | fun Cursor.mapOnColumnsMd4(vararg colNames: String): Map = run { 127 | val kix = colIdx.get(*colNames) 128 | val index = this[kix] 129 | Array(size) { 130 | (index at it).run { 131 | index.left.toList().md4 to it 132 | } 133 | }.toMap() 134 | } 135 | 136 | /** 137 | * /primary/ key mapping. collision behaviors are map-defined 138 | * 139 | */ 140 | fun Cursor.mapOnColumns(vararg colNames: String): Map, Int> = let { 141 | val kix = colIdx.get(*colNames) 142 | val index = this[kix] 143 | val scalars: Vect0r = index.scalars 144 | (scalars as Vect02).left.toArray().map { IOMemento::cmp } 145 | Array(size) { iy -> 146 | (index at iy).let { row: RowVec -> 147 | row.left.toList() to iy 148 | } 149 | }.toMap() 150 | } 151 | 152 | /** 153 | * /primary/ key mapping. collision behaviors are map-defined 154 | * 155 | */ 156 | fun Cursor.arrayMapOnColumns(vararg colNames: String): ArrayMap, Int> = let { 157 | val kix = colIdx.get(*colNames) 158 | val index = this[kix] 159 | val scalars: Vect0r = index.scalars 160 | val map = (scalars as Vect02).left.toArray().map { IOMemento.cmp(it) } 161 | val cmp = Comparator { l1: List<*>, l2: List<*> -> 162 | var res = 0 163 | var ix = 0 164 | while (res == 0 && ix < map.size) { 165 | res = map[ix].invoke(l1[ix], l2[ix]) 166 | ix++ 167 | } 168 | res 169 | } 170 | val comparator = Comparator, Int>> { (l1, _), (l2, _) -> 171 | cmp.compare(l1, l2) 172 | } 173 | 174 | val toTypedArray: Array, Int>> = Array(size) { iy -> 175 | (index at iy).let { row: RowVec -> 176 | row.left.toList() to iy 177 | } 178 | }.toSortedSet(comparator).toTypedArray() 179 | 180 | 181 | val entre: Array, Int>> = toTypedArray.map { (a, b) -> 182 | object : Map.Entry, Int> { 183 | override val key get() = a 184 | override val value get() = b 185 | } 186 | }.toTypedArray() 187 | ArrayMap(entre, cmp) 188 | } 189 | 190 | /** 191 | * /primary/ key mapping. collision behaviors are map-defined 192 | * 193 | */ 194 | fun Cursor.trieOnColumns(vararg colNames: String): Trie = let { 195 | val kix = colIdx.get(*colNames) 196 | val index = this[kix] 197 | Trie().apply { 198 | repeat(size) { iy -> 199 | add(iy, *(index at iy).left.α(Any?::toString).toArray()) 200 | } 201 | }/* only for short paths...*/ 202 | .also { it.freeze() } 203 | } 204 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | <# : batch portion 2 | @REM ---------------------------------------------------------------------------- 3 | @REM Licensed to the Apache Software Foundation (ASF) under one 4 | @REM or more contributor license agreements. See the NOTICE file 5 | @REM distributed with this work for additional information 6 | @REM regarding copyright ownership. The ASF licenses this file 7 | @REM to you under the Apache License, Version 2.0 (the 8 | @REM "License"); you may not use this file except in compliance 9 | @REM with the License. You may obtain a copy of the License at 10 | @REM 11 | @REM http://www.apache.org/licenses/LICENSE-2.0 12 | @REM 13 | @REM Unless required by applicable law or agreed to in writing, 14 | @REM software distributed under the License is distributed on an 15 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | @REM KIND, either express or implied. See the License for the 17 | @REM specific language governing permissions and limitations 18 | @REM under the License. 19 | @REM ---------------------------------------------------------------------------- 20 | 21 | @REM ---------------------------------------------------------------------------- 22 | @REM Apache Maven Wrapper startup batch script, version 3.3.2 23 | @REM 24 | @REM Optional ENV vars 25 | @REM MVNW_REPOURL - repo url base for downloading maven distribution 26 | @REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven 27 | @REM MVNW_VERBOSE - true: enable verbose log; others: silence the output 28 | @REM ---------------------------------------------------------------------------- 29 | 30 | @IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) 31 | @SET __MVNW_CMD__= 32 | @SET __MVNW_ERROR__= 33 | @SET __MVNW_PSMODULEP_SAVE=%PSModulePath% 34 | @SET PSModulePath= 35 | @FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( 36 | IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) 37 | ) 38 | @SET PSModulePath=%__MVNW_PSMODULEP_SAVE% 39 | @SET __MVNW_PSMODULEP_SAVE= 40 | @SET __MVNW_ARG0_NAME__= 41 | @SET MVNW_USERNAME= 42 | @SET MVNW_PASSWORD= 43 | @IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) 44 | @echo Cannot start maven from wrapper >&2 && exit /b 1 45 | @GOTO :EOF 46 | : end batch / begin powershell #> 47 | 48 | $ErrorActionPreference = "Stop" 49 | if ($env:MVNW_VERBOSE -eq "true") { 50 | $VerbosePreference = "Continue" 51 | } 52 | 53 | # calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties 54 | $distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl 55 | if (!$distributionUrl) { 56 | Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" 57 | } 58 | 59 | switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { 60 | "maven-mvnd-*" { 61 | $USE_MVND = $true 62 | $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" 63 | $MVN_CMD = "mvnd.cmd" 64 | break 65 | } 66 | default { 67 | $USE_MVND = $false 68 | $MVN_CMD = $script -replace '^mvnw','mvn' 69 | break 70 | } 71 | } 72 | 73 | # apply MVNW_REPOURL and calculate MAVEN_HOME 74 | # maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ 75 | if ($env:MVNW_REPOURL) { 76 | $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } 77 | $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" 78 | } 79 | $distributionUrlName = $distributionUrl -replace '^.*/','' 80 | $distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' 81 | $MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" 82 | if ($env:MAVEN_USER_HOME) { 83 | $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" 84 | } 85 | $MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' 86 | $MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" 87 | 88 | if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { 89 | Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" 90 | Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" 91 | exit $? 92 | } 93 | 94 | if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { 95 | Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" 96 | } 97 | 98 | # prepare tmp dir 99 | $TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile 100 | $TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" 101 | $TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null 102 | trap { 103 | if ($TMP_DOWNLOAD_DIR.Exists) { 104 | try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } 105 | catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } 106 | } 107 | } 108 | 109 | New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null 110 | 111 | # Download and Install Apache Maven 112 | Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." 113 | Write-Verbose "Downloading from: $distributionUrl" 114 | Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" 115 | 116 | $webclient = New-Object System.Net.WebClient 117 | if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { 118 | $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) 119 | } 120 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 121 | $webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null 122 | 123 | # If specified, validate the SHA-256 sum of the Maven distribution zip file 124 | $distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum 125 | if ($distributionSha256Sum) { 126 | if ($USE_MVND) { 127 | Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." 128 | } 129 | Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash 130 | if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { 131 | Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." 132 | } 133 | } 134 | 135 | # unzip and move 136 | Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null 137 | Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null 138 | try { 139 | Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null 140 | } catch { 141 | if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { 142 | Write-Error "fail to move MAVEN_HOME" 143 | } 144 | } finally { 145 | try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } 146 | catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } 147 | } 148 | 149 | Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" 150 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is one of many restarts from a pristine testing columnar 2018 implementation. It instantly specializes in a domain and the tests suffer 2 | 3 | ### High-Level Summary 4 | 5 | This project is a **custom columnar data processing library for the JVM**, analogous to **Pandas (Python)**, **data.table (R)**, or **Apache Spark (Scala/Java)**. 6 | 7 | Its key characteristics are: 8 | 1. **Lazy Evaluation:** The core data structure, `Cursor`, is not an in-memory collection but a lazy representation of a dataset, defined by its size and an accessor function. This is a powerful pattern for handling data larger than memory. 9 | 2. **Functional & Expressive DSL:** It uses symbolic operators (`α` for map, `∑` for reduce) and a functional style, reminiscent of languages like APL or Scala's more functional libraries. 10 | 3. **High-Performance I/O:** It has first-class support for memory-mapped files, both for text (fixed-width) and a custom binary `ISAM` format, designed for very fast access. 11 | 4. **Rich Metadata Context:** It uses Kotlin's `CoroutineContext` in a novel way as a "property bag" or "blackboard" to carry rich metadata about data types, storage layout, and schema, making this information available throughout the system. 12 | 13 | --- 14 | 15 | ### Feature-by-Feature Analysis and Overlaps 16 | 17 | #### 1. Core Abstraction: The `Cursor` as a Lazy DataFrame 18 | 19 | * **In this library:** `Cursor` is a `typealias` for `Vect0r`, where `Vect0r` is essentially a `(size, accessorFunction)` pair. This means that data isn't loaded until an accessor function is called for a specific row index. This is a core design decision for performance and memory management. 20 | * **Overlaps & Analogues:** 21 | * **Python:** The API surface (slicing, grouping, pivoting) is very similar to **Pandas**. However, Pandas is generally *eager*, loading data into memory. The lazy nature of `Cursor` is much more like **Dask DataFrame** or **Polars**. Polars, in particular, uses a lazy expression system that builds a query plan and executes it all at once, which is a very similar philosophy. 22 | * **Scala/Java:** This is the fundamental model of **Apache Spark**. A Spark DataFrame is a lazily evaluated, distributed collection of rows with a known schema. Operations on a DataFrame build up a logical plan, which is only executed when an "action" (like `collect()` or `save()`) is called. 23 | * **R:** The `data.table` and `tidyverse` (`dplyr`) packages provide a similar high-level API but are primarily in-memory. For lazy evaluation, R users typically connect to a database using a library like `dbplyr`, which translates R code into SQL queries. 24 | 25 | #### 2. Functional & Expressive DSL 26 | 27 | * **In this library:** 28 | * `cursor α { ... }`: An element-wise transformation (map/apply). The use of the Greek letter alpha is stylistic, evoking lambda calculus's alpha-conversion. 29 | * `cursor ∑ { ... }`: A reduction operation over aggregated data. 30 | * `c1[-"colA", -"colB"]`: Column selection by negation, a very convenient feature. 31 | * Fluent chaining of operations like `resample(...).pivot(...).group(...)`. 32 | * **Overlaps & Analogues:** 33 | * **Python (Pandas/Polars):** The fluent chaining style is identical. `df.resample(...).pivot_table(...).groupby(...)`. 34 | * **R (tidyverse):** The "pipe" operator (`%>%`) achieves the same fluent, readable style: `data %>% group_by(col) %>% summarize(...)`. 35 | * **Scala:** The use of symbolic operators is very common in the Scala ecosystem (e.g., in libraries like **Cats** or **ZIO**) to create concise, powerful DSLs. 36 | * **APL/J/K:** These are array-oriented programming languages built almost entirely on terse, symbolic operators for data manipulation. This library's DSL borrows philosophical inspiration from them. 37 | 38 | #### 3. High-Performance I/O & Persistence (NioMMap, ISAMCursor) 39 | 40 | * **In this library:** `NioMMap` and `ISAMCursor` provide direct access to data on disk via memory-mapping, bypassing much of the standard JVM I/O stack. The library defines its own binary `ISAM` (Indexed Sequential Access Method) format, complete with a `.meta` file for schema, which is a classic database technique. The `CellDriver` classes (`Tokenized`, `Fixed`) are codecs for reading/writing types to/from `ByteBuffer`. 41 | * **Overlaps & Analogues:** 42 | * **Databases:** This is the core competency of columnar databases. The approach is very similar to how **DuckDB** (in-process) or **ClickHouse** manage data on disk. Creating a custom binary format is what high-performance systems do. 43 | * **Python:** **Apache Arrow** is the modern standard for language-agnostic, zero-copy, columnar in-memory data. Libraries like **Polars** and **DuckDB** are built on it. The memory-mapping concept is available in **NumPy** (`memmap`) and **Arrow**. 44 | * **Java:** **Chronicle Map** is a well-known Java library for creating off-heap, low-latency key-value stores that are memory-mapped. This library implements similar ideas from scratch using standard `java.nio`. 45 | 46 | #### 4. Metadata via `CoroutineContext` 47 | 48 | * **In this library:** The `context` package (`Arity`, `RecordBoundary`, `Ordering`, `Scalar`) uses `CoroutineContext.Element` to create a type-safe, extensible "property bag". This context is attached to each cell's value (`RowVec` is a `Vect02`), allowing any function to retrieve metadata about a cell's type, name, storage format, etc., without passing it explicitly. 49 | * **Overlaps & Analogues:** This is a very creative use of a Kotlin-specific feature. The general pattern is known as a **Context Object** or **Property Bag**. 50 | * **Functional Programming:** This pattern is analogous to using a **Reader Monad**. A Reader Monad allows you to "inject" a shared environment or configuration into a computation, which can then be accessed by any function within that computation without being an explicit parameter. 51 | * **Java:** A less elegant way to achieve this would be with `ThreadLocal` variables holding a context object. 52 | * **Web Frameworks:** In frameworks like Flask (Python) or Express (Node.js), a global-like `request` object is available during the lifecycle of a request, serving a similar purpose. 53 | 54 | #### 5. Data Structures (`Vect0r`, `Trie`, `ArrayMap`) 55 | 56 | * **In this library:** 57 | * `Vect0r` is a functional, lazy list. 58 | * `Pai2`, `Tripl3`, etc., are custom tuple interfaces. 59 | * `Trie` and `ArrayMap` are specialized data structures for fast lookups. `ArrayMap` is an immutable, sorted map backed by an array, enabling O(log n) lookups via binary search. 60 | * **Overlaps & Analogues:** 61 | * **Custom Collections:** Many high-performance libraries create their own collection types to escape the overhead or semantic limitations of standard library collections. **Eclipse Collections** (Java) and `boost::container` (C++) are prime examples. 62 | * **Immutable Data Structures:** Libraries like **Vavr** (Java) or **Immutable.js** (JavaScript) provide persistent, immutable data structures, which is a core tenet of functional programming and a feature of this library's design. The `Trie` is a classic example of such a structure. 63 | 64 | ### Conclusion 65 | 66 | This library is a fascinating and powerful piece of engineering. It doesn't just copy one tool; it synthesizes ideas from several domains: 67 | 68 | * It has the user-friendly, high-level **API of Pandas**. 69 | * It has the lazy evaluation model of **Apache Spark** or **Dask**. 70 | * It has the low-level, high-performance I/O and custom storage formats of a **columnar database**. 71 | * It uses a terse, functional **DSL reminiscent of APL or Scala**. 72 | * It leverages a unique Kotlin feature (`CoroutineContext`) to implement a **clean, extensible metadata system** akin to what one might build with a Reader Monad in Haskell. 73 | 74 | This is not a general-purpose replacement for Pandas or Spark but appears to be a highly specialized "boutique" library designed for extreme performance on specific, large-scale data processing tasks on the JVM, where control over memory layout and I/O is critical. 75 | -------------------------------------------------------------------------------- /src/test/java/vec/util/HorizonTest.kt: -------------------------------------------------------------------------------- 1 | package vec.util 2 | 3 | import kotlin.test.* 4 | import vec.macros.* 5 | import kotlin.time.Duration.Companion.days 6 | import kotlin.time.ExperimentalTime 7 | 8 | /*@ExperimentalTime*/ 9 | class HorizonTest { 10 | @Test 11 | fun horizonTest() { 12 | val v = Vect0r(5000) { x: Int -> x } 13 | val v2size = 100 14 | val v2: Vect0r = v2size t2 { x: Int -> horizon(x, v2size, v.size) } 15 | System.err.println(v2.toList()) 16 | horizonRatioHunt(v2size, v.size) 17 | } 18 | 19 | fun horizonRatioHunt(viewSize: Int, modelSize: Int): List = run { 20 | var acc = 0 21 | for (i in 0 until viewSize) { 22 | val horizon = horizon(i, viewSize, modelSize)//.also { logDebug { "$it,$acc" } } 23 | if ((horizon - acc) > 1) break 24 | acc = i 25 | 26 | } 27 | val res = _l[ 28 | acc.toDouble() / viewSize.toDouble(), 29 | acc.toDouble() / modelSize.toDouble(), 30 | viewSize.toDouble() / modelSize.toDouble(), 31 | ] 32 | val res2 = res.let { 33 | res + _l[ 34 | it[0] * it[1], 35 | it[1] * it[2], 36 | acc.toDouble() * it[1], 37 | (viewSize - acc).toDouble() * it[1], 38 | (viewSize - acc).toDouble() * it[2], 39 | ] 40 | } 41 | logDebug { ("ratio ascends at($acc, $viewSize,${modelSize})=${res2}") } 42 | 43 | res2 44 | } 45 | 46 | @Test 47 | fun horizonTest2() { 48 | val v = Vect0r(50000) { x: Int -> x } 49 | val v2size = 1000 50 | val v2: Vect0r = v2size t2 { x: Int -> 51 | horizon(x, v2size, v.size) 52 | } 53 | System.err.println(v2[0 until 100].toList()) 54 | horizonRatioHunt(v2size, v.size) 55 | } 56 | 57 | @Test 58 | fun horizonTest3() { 59 | val v = Vect0r(2034190) { x: Int -> x } 60 | val v2size = 5000 61 | val v2: Vect0r = v2size t2 { x: Int -> 62 | horizon(x, v2size, v.size) 63 | } 64 | System.err.println(v2[0 until 200].toList()) 65 | System.err.println("...") 66 | System.err.println(v2[(v2.size - 50) until v2.size].toList()) 67 | horizonRatioHunt(v2size, v.size) 68 | } 69 | 70 | @Test 71 | fun horizonTest4() { 72 | val v = Vect0r(3_000_000) { x: Int -> x } 73 | val v2size = 20 74 | val v2: Vect0r = v2size t2 { x: Int -> 75 | horizon(x, v2size, v.size) 76 | } 77 | System.err.println(v2[0 until 20].toList()) 78 | horizonRatioHunt(v2size, v.size) 79 | horizonRatioHunt(100, 14.days.inWholeMinutes.toInt()) 80 | horizonRatioHunt(200, 14.days.inWholeMinutes.toInt()) 81 | horizonRatioHunt(300, 14.days.inWholeMinutes.toInt()) 82 | horizonRatioHunt(500, 14.days.inWholeMinutes.toInt()) 83 | horizonRatioHunt(3000, 14.days.inWholeMinutes.toInt()) 84 | } 85 | } 86 | /* 87 | wc -l `find -name *csv`|sort -n 88 | 52 ./BTC/GYEN/final-BTC-GYEN-1m.csv 89 | 5161 ./ETH/UAH/final-ETH-UAH-1m.csv 90 | 25321 ./DOGE/BIDR/final-DOGE-BIDR-1m.csv 91 | 51361 ./DOGE/RUB/final-DOGE-RUB-1m.csv 92 | 65327 ./BNB/UAH/final-BNB-UAH-1m.csv 93 | 105647 ./ADA/RUB/final-ADA-RUB-1m.csv 94 | 125597 ./ADA/AUD/final-ADA-AUD-1m.csv 95 | 125597 ./DOT/BRL/final-DOT-BRL-1m.csv 96 | 135677 ./ADA/BRL/final-ADA-BRL-1m.csv 97 | 135677 ./ADA/GBP/final-ADA-GBP-1m.csv 98 | 135677 ./ADA/TRY/final-ADA-TRY-1m.csv 99 | 135677 ./DOT/GBP/final-DOT-GBP-1m.csv 100 | 160078 ./DOGE/GBP/final-DOGE-GBP-1m.csv 101 | 160078 ./DOT/TRY/final-DOT-TRY-1m.csv 102 | 167278 ./BTC/VAI/final-BTC-VAI-1m.csv 103 | 170158 ./DOGE/AUD/final-DOGE-AUD-1m.csv 104 | 170158 ./DOGE/BRL/final-DOGE-BRL-1m.csv 105 | 175798 ./DOGE/EUR/final-DOGE-EUR-1m.csv 106 | 175798 ./DOGE/TRY/final-DOGE-TRY-1m.csv 107 | 246094 ./SUSHI/DOWNUSDT/final-SUSHI-DOWNUSDT-1m.csv 108 | 247552 ./SUSHI/UPUSDT/final-SUSHI-UPUSDT-1m.csv 109 | 286447 ./ADA/EUR/final-ADA-EUR-1m.csv 110 | 296527 ./BNB/BRL/final-BNB-BRL-1m.csv 111 | 306607 ./DOT/EUR/final-DOT-EUR-1m.csv 112 | 306607 ./ETH/BRL/final-ETH-BRL-1m.csv 113 | 331087 ./BTC/BRL/final-BTC-BRL-1m.csv 114 | 375658 ./DOT/DOWNUSDT/final-DOT-DOWNUSDT-1m.csv 115 | 377092 ./DOT/UPUSDT/final-DOT-UPUSDT-1m.csv 116 | 391507 ./SUSHI/BNB/final-SUSHI-BNB-1m.csv 117 | 391507 ./SUSHI/BTC/final-SUSHI-BTC-1m.csv 118 | 391507 ./SUSHI/BUSD/final-SUSHI-BUSD-1m.csv 119 | 391508 ./SUSHI/USDT/final-SUSHI-USDT-1m.csv 120 | 397448 ./DOT/BIDR/final-DOT-BIDR-1m.csv 121 | 409948 ./DOT/BNB/final-DOT-BNB-1m.csv 122 | 409948 ./DOT/BTC/final-DOT-BTC-1m.csv 123 | 409948 ./DOT/BUSD/final-DOT-BUSD-1m.csv 124 | 409948 ./DOT/USDT/final-DOT-USDT-1m.csv 125 | 421048 ./BNB/DAI/final-BNB-DAI-1m.csv 126 | 421048 ./BTC/DAI/final-BTC-DAI-1m.csv 127 | 421048 ./ETH/DAI/final-ETH-DAI-1m.csv 128 | 426474 ./XTZ/UPUSDT/final-XTZ-UPUSDT-1m.csv 129 | 426493 ./XTZ/DOWNUSDT/final-XTZ-DOWNUSDT-1m.csv 130 | 426749 ./BNB/DOWNUSDT/final-BNB-DOWNUSDT-1m.csv 131 | 427048 ./BNB/AUD/final-BNB-AUD-1m.csv 132 | 428188 ./BNB/UPUSDT/final-BNB-UPUSDT-1m.csv 133 | 436648 ./BTC/AUD/final-BTC-AUD-1m.csv 134 | 436649 ./ETH/AUD/final-ETH-AUD-1m.csv 135 | 438388 ./DOGE/BUSD/final-DOGE-BUSD-1m.csv 136 | 456655 ./ADA/DOWNUSDT/final-ADA-DOWNUSDT-1m.csv 137 | 458095 ./ADA/UPUSDT/final-ADA-UPUSDT-1m.csv 138 | 462688 ./ETH/UPUSDT/final-ETH-UPUSDT-1m.csv 139 | 462689 ./ETH/DOWNUSDT/final-ETH-DOWNUSDT-1m.csv 140 | 481408 ./BNB/BIDR/final-BNB-BIDR-1m.csv 141 | 481408 ./BTC/BIDR/final-BTC-BIDR-1m.csv 142 | 481408 ./ETH/BIDR/final-ETH-BIDR-1m.csv 143 | 489958 ./BTC/UAH/final-BTC-UAH-1m.csv 144 | 496918 ./BNB/GBP/final-BNB-GBP-1m.csv 145 | 496918 ./BTC/GBP/final-BTC-GBP-1m.csv 146 | 496918 ./ETH/GBP/final-ETH-GBP-1m.csv 147 | 532722 ./BNB/ZAR/final-BNB-ZAR-1m.csv 148 | 532722 ./BTC/ZAR/final-BTC-ZAR-1m.csv 149 | 532722 ./ETH/ZAR/final-ETH-ZAR-1m.csv 150 | 548740 ./BTC/DOWNUSDT/final-BTC-DOWNUSDT-1m.csv 151 | 548757 ./BTC/UPUSDT/final-BTC-UPUSDT-1m.csv 152 | 587848 ./BNB/IDRT/final-BNB-IDRT-1m.csv 153 | 587848 ./BTC/IDRT/final-BTC-IDRT-1m.csv 154 | 682166 ./XTZ/BUSD/final-XTZ-BUSD-1m.csv 155 | 738266 ./BNB/EUR/final-BNB-EUR-1m.csv 156 | 738266 ./BTC/EUR/final-BTC-EUR-1m.csv 157 | 738266 ./ETH/EUR/final-ETH-EUR-1m.csv 158 | 751169 ./ETH/TRY/final-ETH-TRY-1m.csv 159 | 758306 ./BNB/TRY/final-BNB-TRY-1m.csv 160 | 758306 ./BTC/TRY/final-BTC-TRY-1m.csv 161 | 784046 ./BNB/RUB/final-BNB-RUB-1m.csv 162 | 784046 ./BTC/RUB/final-BTC-RUB-1m.csv 163 | 784046 ./ETH/RUB/final-ETH-RUB-1m.csv 164 | 800306 ./ADA/BUSD/final-ADA-BUSD-1m.csv 165 | 839973 ./BTC/NGN/final-BTC-NGN-1m.csv 166 | 844563 ./ETH/BUSD/final-ETH-BUSD-1m.csv 167 | 883263 ./XTZ/BNB/final-XTZ-BNB-1m.csv 168 | 883263 ./XTZ/BTC/final-XTZ-BTC-1m.csv 169 | 883263 ./XTZ/USDT/final-XTZ-USDT-1m.csv 170 | 890521 ./BTC/BUSD/final-BTC-BUSD-1m.csv 171 | 890522 ./BNB/BUSD/final-BNB-BUSD-1m.csv 172 | 999363 ./DOGE/BTC/final-DOGE-BTC-1m.csv 173 | 999363 ./DOGE/USDT/final-DOGE-USDT-1m.csv 174 | 1143182 ./ADA/USDC/final-ADA-USDC-1m.csv 175 | 1163342 ./BNB/TUSD/final-BNB-TUSD-1m.csv 176 | 1163343 ./BTC/TUSD/final-BTC-TUSD-1m.csv 177 | 1163344 ./ETH/TUSD/final-ETH-TUSD-1m.csv 178 | 1283940 ./ADA/TUSD/final-ADA-TUSD-1m.csv 179 | 1289741 ./ETH/USDC/final-ETH-USDC-1m.csv 180 | 1289751 ./BNB/USDC/final-BNB-USDC-1m.csv 181 | 1289751 ./BTC/USDC/final-BTC-USDC-1m.csv 182 | 1312200 ./BNB/PAX/final-BNB-PAX-1m.csv 183 | 1312200 ./BTC/PAX/final-BTC-PAX-1m.csv 184 | 1312202 ./ETH/PAX/final-ETH-PAX-1m.csv 185 | 1636388 ./ADA/BNB/final-ADA-BNB-1m.csv 186 | 1636388 ./ADA/USDT/final-ADA-USDT-1m.csv 187 | 1832340 ./ADA/ETH/final-ADA-ETH-1m.csv 188 | 1832341 ./ADA/BTC/final-ADA-BTC-1m.csv 189 | 1867416 ./BNB/USDT/final-BNB-USDT-1m.csv 190 | 1983630 ./ETH/USDT/final-ETH-USDT-1m.csv 191 | 1983631 ./BTC/USDT/final-BTC-USDT-1m.csv 192 | 1994848 ./BNB/ETH/final-BNB-ETH-1m.csv 193 | 2033587 ./BNB/BTC/final-BNB-BTC-1m.csv 194 | 2034190 ./ETH/BTC/final-ETH-BTC-1m.csv 195 | 71438298 total 196 | 197 | */ -------------------------------------------------------------------------------- /src/main/java/cursors/io/ISAMCursor.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("OVERRIDE_BY_INLINE", "UNCHECKED_CAST") 2 | 3 | package cursors.io 4 | 5 | import cursors.Cursor 6 | import cursors.TypeMemento 7 | import cursors.at 8 | import cursors.context.* 9 | import kotlinx.datetime.Clock 10 | import kotlinx.datetime.Instant 11 | import vec.macros.* 12 | import vec.macros.Vect02_.left 13 | import vec.macros.Vect02_.right 14 | import vec.util.FibonacciReporter 15 | import vec.util.debug 16 | import vec.util.logDebug 17 | import vec.util.span 18 | import java.nio.ByteBuffer 19 | import java.nio.channels.FileChannel 20 | import java.nio.file.Files 21 | import java.nio.file.Path 22 | import java.nio.file.Paths 23 | import java.nio.file.StandardOpenOption 24 | import java.util.* 25 | 26 | /** A cursor that reads from a file in ISAM format. 27 | */ 28 | class ISAMCursor( 29 | binpath: Path, 30 | val fc: FileChannel, 31 | metapath: Path = Paths.get("$binpath.meta".debug { logDebug { "using default Isam meta: $it" } }), 32 | ) : Cursor { 33 | override inline val first: Int get() = size 34 | override val second: (Int) -> RowVec 35 | val width: Int get() = drivers.size 36 | val drivers: Array> by lazy { NioMMap.binary(scalars.map(Scalar::first) as Vect0r) } 37 | val recordlen: Int by lazy { rcoords.last.second } 38 | val rcoords: Vect02 39 | val scalars: Vect0r 40 | val size: Int get() = (fc.size() / recordlen).toInt() 41 | 42 | init { 43 | val lines = Files.readAllLines(metapath).apply { removeIf { it.startsWith("# ") || it.isNullOrBlank() } } 44 | rcoords = (lines[0].split("\\s+".toRegex()) α (String::toInt)).zipWithNext() 45 | val typeVec = run { 46 | val s = lines[2] 47 | val split = s.split("\\s+".toRegex()) 48 | val res = split.α(IOMemento::valueOf) 49 | 50 | res 51 | } 52 | val rnames = lines[1].split("\\s+".toRegex()).toVect0r() 53 | scalars = (typeVec).zip(rnames, Scalar.Companion::invoke) /*as Vect0r*/ 54 | 55 | fc.let { fileChannel -> 56 | 57 | second = { iy: Int -> 58 | /** 59 | * this is a departure from the original windowed 60 | * ByteBuffer Nio harnasss 61 | */ 62 | 63 | val row = ByteBuffer.allocate(recordlen) 64 | val read = fc.read(row.clear(), iy.toLong() * recordlen.toLong()) 65 | 66 | RowVec(drivers.size) { ix: Int -> 67 | rcoords[ix].let { (b, e) -> 68 | val slice = row.limit(e).position(b).slice() 69 | val read1 = drivers[ix].read(slice) 70 | read1 t2 { this@ISAMCursor.scalars[ix] } 71 | } 72 | } 73 | } 74 | } 75 | } 76 | } 77 | 78 | /** 79 | * this writes a cursor's values to a single Network-endian ByteBuffer translation and writes an additional filename+".meta" 80 | * file containing commented meta description strings 81 | */ 82 | fun Cursor.writeISAM( 83 | pathname: String, 84 | defaultVarcharSize: Int = 128, 85 | varcharSizes: Map< 86 | /** 87 | column*/ 88 | Int, 89 | /**length*/ 90 | Int>? = null, 91 | ) { 92 | var logger: FibonacciReporter? = null 93 | val mementos = scalars.map(Scalar::first).toArray() 94 | val vec = Columnar.of(scalars) t2 mementos 95 | 96 | /** create context columns */ 97 | val (_: Arity, ioMemos) = vec 98 | val sizes = varcharSizes ?: let { curs -> 99 | 100 | linkedMapOf().apply { 101 | (curs at 0).right.`➤`.withIndex().onEach { (index, it) -> 102 | 103 | //our blackboard CoroutineCOntext metadata function. 104 | val cc = it.invoke() 105 | (cc[Arity.arityKey] as? Scalar)?.let { (a) -> 106 | if (a == IOMemento.IoString) 107 | (cc[RecordBoundary.boundaryKey] as? FixedWidth)?.let { fw: FixedWidth -> 108 | this.entries.add(AbstractMap.SimpleEntry(index, fw.recordLen)) 109 | } 110 | } 111 | } 112 | } 113 | } 114 | val wcoords: Vect02 = networkCoords(ioMemos, defaultVarcharSize, sizes) 115 | val reclen = wcoords.right.last 116 | writeISAMMeta(pathname, wcoords) 117 | val rowBuf = ByteBuffer.allocateDirect(reclen + 1) 118 | val bounceyBuff = ByteBuffer.allocate(reclen + 1) 119 | val colIdx = colIdx 120 | val drivers = (colIdx.left α Fixed.mapped::get).`➤`.filterNotNull().toTypedArray() 121 | 122 | lateinit var closeTimer: Instant 123 | try { 124 | FileChannel.open( 125 | Paths.get(pathname), 126 | // ExtendedOpenOption.DIRECT, Current location of the bytebuffer (0) is not a multiple of the block size (4096) 127 | StandardOpenOption.TRUNCATE_EXISTING, 128 | StandardOpenOption.CREATE, 129 | StandardOpenOption.WRITE 130 | ) 131 | } catch (x: Exception) { 132 | logDebug { "falling back to non-direct file open. " + x.localizedMessage } 133 | FileChannel.open( 134 | Paths.get(pathname), 135 | StandardOpenOption.TRUNCATE_EXISTING, 136 | StandardOpenOption.CREATE, 137 | StandardOpenOption.WRITE 138 | ) 139 | }.use { fchannel -> 140 | 141 | val xsize = width 142 | val ysize = size 143 | debug { 144 | logger = FibonacciReporter( 145 | ysize, "total is $ysize rows" 146 | ) 147 | } 148 | for (y in 0 until ysize) { 149 | val row = this at y 150 | rowBuf.clear() 151 | for (x in 0 until xsize) { 152 | val cellData = row.left[x] 153 | val writeFn = drivers[x].write 154 | (writeFn as (ByteBuffer, Any?) -> Unit)(bounceyBuff.clear().limit(wcoords[x].span), cellData) 155 | rowBuf.put(bounceyBuff.flip()) 156 | } 157 | fchannel.write(rowBuf.flip()) 158 | logger?.let { it.report(y)?.let { System.err.println(it) } } 159 | } 160 | debug { 161 | closeTimer = Clock.System.now() 162 | logDebug { "file channel $fchannel going out of scope, closing, writing" } 163 | } 164 | } 165 | debug { 166 | val instant = Clock.System.now() - closeTimer 167 | logDebug { "close took $instant" } 168 | } 169 | } 170 | 171 | 172 | /** writes a meta file for the ISAM file */ 173 | 174 | fun Cursor.writeISAMMeta( 175 | pathname: String, 176 | //wrecordlen: Int, 177 | wcoords: Vect02, 178 | ) { 179 | Files.newOutputStream( 180 | Paths.get("$pathname.meta") 181 | ).bufferedWriter().use { 182 | 183 | fileWriter -> 184 | 185 | val s = this.scalars as Vect02 186 | 187 | val coords = wcoords.toList() 188 | .map { listOf(it.first, it.second) }.flatten().joinToString(" ") 189 | val nama = s.right.map { s1: String? -> s1!!.replace(' ', '_') }.toList().joinToString(" ") 190 | val mentos = s.left.toArray().mapIndexed { ix, it -> 191 | if (it is IOMemento) 192 | it.name else { 193 | val pai2 = wcoords[ix] 194 | pai2.span 195 | } 196 | }.toList() 197 | 198 | .joinToString(" ") 199 | listOf( 200 | "# format: coords WS .. EOL names WS .. EOL TypeMememento WS ..", 201 | "# last coord is the recordlen", 202 | coords, 203 | nama, 204 | mentos 205 | ).forEach { line -> 206 | fileWriter.write(line) 207 | fileWriter.newLine() 208 | } 209 | } 210 | } 211 | 212 | /** 213 | * this is a handle for a simple cursor that reads a file in the format written by [Cursor.writeISAM] 214 | */ 215 | typealias CursorHandle = Pai2 216 | 217 | /** 218 | * this opens a cursor isam file for Cursor access and returns the cursor and channel in a handle for manual closing 219 | */ 220 | fun openIsamCursor(resolve: Path): CursorHandle { 221 | val channel1 = FileChannel.open(resolve) 222 | val cursor = ISAMCursor(resolve, channel1) 223 | return (cursor t2 channel1) 224 | } -------------------------------------------------------------------------------- /src/main/java/vec/util/BikeShed.kt: -------------------------------------------------------------------------------- 1 | /** 2 | * what library doesn't have at least one util for all the evils of getting work done outside the elegant showroom code? 3 | */ 4 | @file:Suppress( 5 | "NOTHING_TO_", 6 | "UNCHECKED_CAST", 7 | "ClassName", 8 | "HasPlatformType", 9 | "NOTHING_TO_INLINE", 10 | "UnclearPrecedenceOfBinaryExpression" 11 | ) 12 | @file:OptIn(ExperimentalUnsignedTypes::class) 13 | 14 | package vec.util 15 | 16 | import vec.macros.* 17 | import java.io.BufferedReader 18 | import java.io.InputStreamReader 19 | import java.lang.ref.SoftReference 20 | import java.nio.ByteBuffer 21 | import java.nio.file.Path 22 | import java.nio.file.Paths 23 | import java.util.* 24 | import kotlin.math.min 25 | import kotlin.text.Charsets.UTF_8 26 | 27 | val Pair.span: Int get() = let { (a: Int, b: Int) -> b - a } 28 | val Pai2.span: Int get() = let { (a: Int, b: Int) -> b - a } 29 | fun Int.toArray(): IntArray = _a[this] 30 | fun btoa(ba: ByteArray): String = String(ba, UTF_8) 31 | fun trim(it: String): String = it.trim() 32 | infix fun ByteBuffer.at(start: Int): ByteBuffer = apply { (if (limit() > start) clear() else this).position(start) } 33 | operator fun ByteBuffer.get(start: Int, end: Int): ByteBuffer = apply { this.at(start).limit(end) } 34 | fun bb2ba(bb: ByteBuffer): ByteArray = ByteArray(bb.remaining()).also { bb[it] } 35 | 36 | //infix fun Any?.debug(message: String) = kotlin.io.println(message) 37 | 38 | 39 | val IntProgression.indices: List 40 | get() = map { it } 41 | 42 | var logReuseCountdown: Int = 0 43 | 44 | /** 45 | * missing stdlib list operator https://github.com/Kotlin/KEEP/pull/112 46 | */ 47 | object _v { 48 | inline operator fun get(vararg t: T): Vect0r = t.size t2 t::get 49 | } 50 | 51 | /** 52 | * missing stdlib list operator https://github.com/Kotlin/KEEP/pull/112 53 | */ 54 | object _l { 55 | inline operator fun get(vararg t: T): List = if (t.size == 1) Collections.singletonList(t[0]) else listOf(*t) 56 | } 57 | 58 | /** 59 | * missing stdlib array operator https://github.com/Kotlin/KEEP/pull/112 60 | */ 61 | object _a { 62 | inline operator fun get(vararg t: Boolean): BooleanArray = t 63 | inline operator fun get(vararg t: Byte): ByteArray = t 64 | inline operator fun get(vararg t: UByte): UByteArray = t 65 | inline operator fun get(vararg t: Char): CharArray = t 66 | inline operator fun get(vararg t: Short): ShortArray = t 67 | inline operator fun get(vararg t: UShort): UShortArray = t 68 | inline operator fun get(vararg t: Int): IntArray = t 69 | inline operator fun get(vararg t: UInt): UIntArray = t 70 | inline operator fun get(vararg t: Long): LongArray = t 71 | inline operator fun get(vararg t: ULong): ULongArray = t 72 | inline operator fun get(vararg t: Float): FloatArray = t 73 | inline operator fun get(vararg t: Double): DoubleArray = t 74 | inline operator fun get(vararg t: T): Array = t as Array 75 | } 76 | 77 | /** 78 | * missing stdlib set operator https://github.com/Kotlin/KEEP/pull/112 79 | */ 80 | object _s { 81 | inline operator fun get(vararg t: T): Set = if (t.size == 1) Collections.singleton(t[0]) else setOf(*t) 82 | } 83 | 84 | /** 85 | * missing stdlib map convenience operator 86 | */ 87 | object _m { 88 | operator fun > get(p: List

): Map = (p).toMap() 89 | operator fun > get(vararg p: P): Map = mapOf(*p) 90 | operator fun get(p: Vect02): Map = p.`➤`.associate(Pai2::pair) 91 | } 92 | 93 | fun logDebug(debugTxt: () -> String) { 94 | if (`assertionsEnableDebug`) System.err.println(debugTxt()) 95 | } 96 | 97 | @Suppress("ObjectPropertyName") 98 | val assertionsEnableDebug = try { 99 | assert(false) 100 | false 101 | } catch (e: AssertionError) { 102 | true 103 | } 104 | //@[JvmSynthetic JvmField] 105 | //val `debug status matching -ea jvm switch` = Pai2::class.java.desiredAssertionStatus() 106 | 107 | fun T.debug(block: (T) -> Unit): T = also { lmbda -> 108 | if (`assertionsEnableDebug`) block(lmbda) 109 | } 110 | 111 | /** 112 | * ternary --- (B) % t ?: f 113 | */ 114 | inline infix operator fun Boolean.rem(noinline block: () -> T): T? = block.takeIf { this }?.invoke() 115 | 116 | fun main() { 117 | logDebug { "this ought not be visible" } 118 | } 119 | 120 | val eol: String = System.getProperty("line.separator") 121 | 122 | fun fileSha256Sum(pathname: String): String { 123 | val command = ProcessBuilder().command("sha256sum", pathname) 124 | val process = command.start() 125 | val reader = BufferedReader(InputStreamReader(process.inputStream)) 126 | val builder = StringBuilder() 127 | var line: String? = null 128 | while (reader.readLine().also { line = it } != null) { 129 | builder.append(line) 130 | builder.append(eol) 131 | } 132 | return builder.toString() 133 | } 134 | 135 | val String.path: Path get() = Paths.get(this) 136 | 137 | infix fun Any?.println(x: Any?) { 138 | kotlin.io.println("$x") 139 | } 140 | 141 | @JvmOverloads 142 | tailrec fun fib(n: Int, a: Int = 0, b: Int = 1): Int = when (n) { 143 | 0 -> a 144 | 1 -> b 145 | else -> fib(n - 1, b, a + b) 146 | } 147 | 148 | //the values repeat until the lower limit is reached providing cheap dummy row context. 149 | @JvmOverloads 150 | fun moireValues( 151 | inVec: Vect0r, 152 | limit: Int, 153 | initialOneOrMore: Int = inVec.first, 154 | x: (Int) -> T = inVec.second, 155 | ): Pai2 T> = min(initialOneOrMore, limit).let { min -> 156 | combine(min t2 x, (limit - min) t2 { i: Int -> 157 | x(i.rem(min)) 158 | } //dummy loop rows 159 | ) 160 | } 161 | 162 | @JvmName("todubNeg") 163 | fun todubneg(d: Any?): Double = todub(d, -1e300) 164 | 165 | @JvmName("todub") 166 | fun todub(d: Any?): Double = todub(d, .0) 167 | 168 | @JvmName("tof0") 169 | fun tofneg(f: Any?): Float = tof(f, (-1e53).toFloat()) 170 | 171 | @JvmName("tof1") 172 | fun tof(f: Any?): Float = tof(f, .0f) 173 | 174 | 175 | val cheapDubCache: WeakHashMap>> = 176 | WeakHashMap>>(0) 177 | 178 | val cheapFCache: WeakHashMap>> = 179 | WeakHashMap>>(0) 180 | 181 | /** really really wants to produce a Double 182 | */ 183 | @JvmName("todubd") 184 | fun todub(f: Any?, default: Double): Double = ((f as? Double ?: (f as? Number)?.toDouble()) ?: "$f".let { 185 | cheapDubCache.getOrPut(it) { SoftReference(it t2 it.toDoubleOrNull()) } 186 | }.get()?.second)?.takeUnless { it.isNaN || it.isInfinite } ?: default 187 | 188 | @JvmName("todubf") 189 | fun tof(f: Any?, default: Float): Float = ((f as? Float ?: (f as? Number)?.toFloat()) ?: "$f".let { 190 | cheapFCache.getOrPut(it) { SoftReference(it t2 it.toFloatOrNull()) } 191 | }.get()?.second)?.takeUnless { it.isNaN || it.isInfinite } ?: default 192 | 193 | 194 | @JvmName("utodubd") 195 | fun utodub(f: Any?, default: Double = 0.0): Double = ((f as? Double ?: (f as? Number)?.toDouble())) 196 | ?: "$f".toDoubleOrNull()?.takeUnless { it.isNaN || it.isInfinite } ?: default 197 | 198 | @JvmName("utodubf") 199 | fun utof(f: Any?, default: Float = 0f): Float = ((f as? Float ?: f as? Number)?.toFloat()) 200 | ?: ("$f".toFloatOrNull()?.takeUnless { it.isNaN || it.isInfinite } ?: default) 201 | 202 | 203 | fun MutableCollection.flip(flipV: E) { 204 | if (flipV in this) remove(flipV) 205 | else add(flipV) 206 | } 207 | 208 | inline val Double.isNaN get() = this != this 209 | inline val Float.isNaN get() = this != this 210 | inline val Double.isInfinite get() = (this == Double.POSITIVE_INFINITY) || (this == Double.NEGATIVE_INFINITY) 211 | inline val Float.isInfinite get() = (this == Float.POSITIVE_INFINITY) || (this == Float.NEGATIVE_INFINITY) 212 | inline val Double.sane get() = if (isNaN || isInfinite) 0.0 else this 213 | inline val Float.sane get() = if (isNaN || isInfinite) 0.0f else this 214 | fun ubyteHex2(toUByte: UByte) = ("0" + toUByte.toString(16)).takeLast(2) 215 | 216 | fun SortedSet.subSet(indirectHiddenRange: ClosedRange): SortedSet = 217 | subSet(indirectHiddenRange.start, indirectHiddenRange.endInclusive.inc()) 218 | 219 | fun SortedMap.subMap(range: ClosedRange): SortedMap = 220 | this.subMap(range.start, range.endInclusive.inc()) 221 | 222 | fun Random.nextInt(range: IntRange): Int = nextInt(range.first, range.last) -------------------------------------------------------------------------------- /src/test/java/cursors/CursorKtTest.kt: -------------------------------------------------------------------------------- 1 | @file:Suppress("UNCHECKED_CAST") 2 | 3 | package cursors 4 | 5 | import cursors.context.Columnar 6 | import cursors.context.FixedWidth 7 | import cursors.context.NioMMap 8 | import cursors.context.RowMajor 9 | import cursors.context.RowMajor.Companion.fixedWidthOf 10 | import cursors.io.* 11 | import cursors.io.IOMemento.IoFloat 12 | import cursors.io.IOMemento.IoString 13 | import cursors.macros.join 14 | import cursors.macros.`∑` 15 | import kotlin.test.* 16 | import vec.macros.* 17 | import vec.macros.Vect02_.left 18 | import vec.macros.Vect02_.right 19 | import vec.ml.DummySpec 20 | import vec.util._a 21 | import vec.util._v 22 | 23 | 24 | 25 | class CursorKtTest { 26 | val coords: Vect02 = _a[ 27 | 0, 10, 28 | 10, 84, 29 | 84, 124, 30 | 124, 164 31 | ].zipWithNext() 32 | 33 | val drivers: Vect0r = _v[ 34 | IOMemento.IoLocalDate, 35 | IoString, 36 | IoFloat, 37 | IoFloat 38 | ] 39 | 40 | val names: Vect0r = _v["date", "channel", "delivered", "ret"] 41 | val mf: MappedFile = MappedFile("src/test/resources/caven4.fwf") 42 | val nio: NioMMap = NioMMap(mf) 43 | val fixedWidth: FixedWidth 44 | get() = fixedWidthOf(nio = nio, coords = coords) 45 | 46 | @Suppress("UNCHECKED_CAST") 47 | val root: TableRoot = RowMajor().fromFwf( 48 | fixedWidth, 49 | RowMajor.indexableOf(nio, fixedWidth), 50 | nio, 51 | Columnar(drivers.zip(names) as Vect02 /* = vec.macros.Pai2 vec.macros.Pai2> */) 52 | ) 53 | 54 | 55 | @Test 56 | fun div() { 57 | val pai21 = (0..2800000) / Runtime.getRuntime().availableProcessors() 58 | System.err.println(pai21.toList().toString()) 59 | 60 | } 61 | 62 | @Test 63 | fun resample() { 64 | val cursor: Cursor = cursorOf(root) 65 | cursor.toList()[3][2].first shouldBe 820f 66 | System.err.println(combine(cursor).left.toList()) 67 | } 68 | 69 | @Test 70 | fun oneHot() { 71 | val cursor: Cursor = cursorOf(root) 72 | var categories = cursor[0].categories() 73 | var scalars = categories.scalars as Vect02 74 | System.err.println(scalars.right.toList()) 75 | var toList = combine(categories).left.toList() 76 | toList.forEach { System.err.println(it) } 77 | categories = cursor[0].categories(DummySpec.Last) 78 | scalars = categories.scalars as Vect02 79 | System.err.println(scalars.right.toList()) 80 | toList = combine(categories).left.toList() 81 | toList.forEach { System.err.println(it) } 82 | } 83 | 84 | @Test 85 | fun `resample+ordered`() { 86 | val cursor: Cursor = cursorOf(root) 87 | run { 88 | System.err.println("unordered\n\n") 89 | 90 | val resample = cursor.resample(0) 91 | val toList = combine( 92 | resample 93 | /*.ordered(intArrayOf(0), Comparator { o1, o2 -> o1.toString().compareTo(o2.toString()) })*/ 94 | ).left 95 | .toList() 96 | resample.toList()[3][2].first shouldBe 820f 97 | toList.forEach { System.err.println(it) } 98 | } 99 | System.err.println("ordered\n\n") 100 | run { 101 | val ordered = cursor.resample(0) 102 | .ordered(intArrayOf(0)/*, Comparator { o1, o2 -> o1.toString().compareTo(o2.toString()) }*/) 103 | 104 | val toList = combine(ordered).left.toList() 105 | 106 | toList.forEach { System.err.println(it) } 107 | val pai2 = ordered.toList()[9] 108 | val pai21 = pai2[1] 109 | val first = pai21.first 110 | first shouldBe "0102211/0101010212/13-14/01" 111 | ordered.toList()[10][1].first shouldBe "0500020/0101010106/13-14/05" 112 | } 113 | } 114 | 115 | @Test 116 | fun `resample+join`() { 117 | val cursor: Cursor = cursorOf(root) 118 | 119 | 120 | val resample = cursor.resample(0) 121 | val join = join(_v[resample[0, 1], resample[2, 3]]) 122 | for (i in 0 until resample.first) { 123 | (resample at (i)).left.toList() shouldBe (join at (i)).left.toList() 124 | println( 125 | (resample at (i)).left.toList() to (join at (i)).left.toList() 126 | ) 127 | } 128 | 129 | 130 | } 131 | 132 | @Test 133 | fun whichKey() { 134 | val fanOut_size = 2 135 | val lhs_size = 2 136 | fun whichKey(ix: Int) = (ix - lhs_size) / fanOut_size 137 | whichKey(702) shouldBe 350 138 | assertEquals(349, whichKey(700)) 139 | } 140 | 141 | @Test 142 | fun whichValue() { 143 | 144 | val fanOut_size = 2 145 | val lhs_size = 2 146 | 147 | fun whichValue(ix: Int) = (ix - lhs_size) % fanOut_size 148 | whichValue(3) shouldBe 1 149 | whichValue(33) shouldBe 1 150 | 151 | whichValue(3) shouldBe 1 152 | whichValue(4) shouldBe 0 153 | whichValue(0) shouldBe 0 154 | } 155 | 156 | @Test 157 | fun pivot() { 158 | val cursor: Cursor = cursorOf(root) 159 | println(combine(cursor).left.toList()) 160 | val piv = cursor.pivot(intArrayOf(0), intArrayOf(1), intArrayOf(2, 3)) 161 | val toArray = piv.scalars.toArray() 162 | val map = toArray.map { it.second } 163 | println(map) 164 | piv.forEach { 165 | val left = it.left.toList() 166 | println("" + left) 167 | 168 | } 169 | } 170 | 171 | @Test 172 | fun group() { 173 | 174 | val cursor: Cursor = cursorOf(root) 175 | println(combine(cursor).left.toList()) 176 | val piv = cursor.group((0)) 177 | cursor.forEach { it -> 178 | println(it.map { pai2 -> 179 | "${ 180 | pai2.component1().let { 181 | (it as? Vect0r<*>)?.toList() ?: it 182 | } 183 | }" 184 | }.toList()) 185 | } 186 | piv.forEach { it -> 187 | println(it.map { pai2 -> 188 | "${ 189 | pai2.component1().let { 190 | (it as? Vect0r<*>)?.toList() ?: it 191 | } 192 | }" 193 | }.toList()) 194 | } 195 | } 196 | 197 | @Test 198 | fun `pivot+group`() { 199 | System.err.println("pivot+group ") 200 | val cursor: Cursor = cursorOf(root) 201 | println("from:\n" + combine(cursor).left.toList()) 202 | val piv = cursor.pivot(intArrayOf(0), intArrayOf(1), intArrayOf(2, 3)).group((0)) 203 | println() 204 | piv.forEach { it -> 205 | println(it.map { vec -> 206 | "${ 207 | vec.component1().let { 208 | (it as? Vect0r<*>)?.toList() ?: it 209 | } 210 | }" 211 | }.toList()) 212 | } 213 | } 214 | 215 | @Test 216 | fun `pivot+group+reduce`() { 217 | System.err.println("pivot+group+reduce") 218 | val cursor: Cursor = cursorOf(root) 219 | println(combine(cursor).left.toList()) 220 | val piv = cursor.pivot( 221 | intArrayOf(0), 222 | intArrayOf(1), 223 | intArrayOf(2, 3) 224 | ).group((0)).`∑`(sumReducer[IoFloat]!!) 225 | 226 | piv.forEach { it -> 227 | println(it.map { vec -> 228 | "${ 229 | vec.component1().let { 230 | (it as? Vect0r<*>)?.toList() ?: it 231 | } 232 | }" 233 | }.toList()) 234 | } 235 | } 236 | 237 | @Test 238 | fun `resample+pivot+group+reduce+join`() { 239 | println("resample+group+reduce+join") 240 | val cursor: Cursor = cursorOf(root) 241 | val resample = cursor.resample(0) 242 | resample.forEach { it -> 243 | println(it.map { vec -> 244 | "${ 245 | vec.component1().let { 246 | (it as? Vect0r<*>)?.toList() ?: it 247 | } 248 | }" 249 | }.toList()) 250 | } 251 | println("---") 252 | val grp = resample.group((1)) 253 | grp.forEach { it -> 254 | println(it.map { vec -> 255 | "${ 256 | vec.component1().let { 257 | (it as? Vect0r<*>)?.toList() ?: it 258 | } 259 | }" 260 | }.toList()) 261 | } 262 | println("---") 263 | val pai2 = grp[2, 3] 264 | val join: Cursor = join(_v[grp[0, 1], pai2.`∑`(floatSum)]) 265 | join.forEach { it -> 266 | println(it.map { vec -> 267 | "${ 268 | vec.component1().let { 269 | (it as? Vect0r<*>)?.toList() ?: it 270 | } 271 | }" 272 | }.toList()) 273 | } 274 | } 275 | 276 | @Test fun castness(){ 277 | 278 | } 279 | } 280 | -------------------------------------------------------------------------------- /src/main/java/vec/macros/Twop13.kt: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | In mathematics, a tuple is a finite ordered list (sequence) of elements. An n-tuple is a sequence (or ordered list) of n elements, where n is a non-negative integer. There is only one 0-tuple, an empty sequence, or empty tuple, as it is referred to. An n-tuple is defined inductively using the construction of an ordered pair. 4 | 5 | Mathematicians usually write 'tuples' by listing the elements within parentheses "{\displaystyle ({\text{ }})}(\text{ })" and separated by commas; for example, {\displaystyle (2,7,4,1,7)}(2, 7, 4, 1, 7) denotes a 5-tuple. Sometimes other symbols are used to surround the elements, such as square brackets "[ ]" or angle brackets "⟨ ⟩". Braces "{ }" are only used in defining arrays in some programming languages such as C++ and Java, but not in mathematical expressions, as they are the standard notation for sets. The term tuple can often occur when discussing other mathematical objects, such as vectors. 6 | 7 | In computer science, tuples come in many forms. In dynamically typed languages, such as Lisp, lists are commonly used as tuples.[citation needed] Most typed functional programming languages implement tuples directly as product types,[1] tightly associated with algebraic data types, pattern matching, and destructuring assignment.[2] Many programming languages offer an alternative to tuples, known as record types, featuring unordered elements accessed by label.[3] A few programming languages combine ordered tuple product types and unordered record types into a single construct, as in C structs and Haskell records. Relational databases may formally identify their rows (records) as tuples. 8 | 9 | Tuples also occur in relational algebra; when programming the semantic web with the Resource Description Framework (RDF); in linguistics;[4] and in philosophy.[5] 10 | */ 11 | 12 | 13 | @file:Suppress("OVERRIDE_BY_", "OVERRIDE_BY_INLINE", "NOTHING_TO_INLINE", "FunctionName") 14 | 15 | package vec.macros 16 | 17 | import vec.util._a 18 | 19 | /**inheritable version of pair */ 20 | interface Pai2 { 21 | val first: F 22 | val second: S 23 | operator fun component1(): F = first 24 | operator fun component2(): S = second 25 | 26 | /** 27 | * for println and serializable usecases, offload that stuff using this method. 28 | */ 29 | val pair: Pair get() = let { first to second } 30 | 31 | companion object { 32 | operator fun invoke(first: F, second: S): Pai2 = object : Pai2 { 33 | override val first get() = first 34 | override val second get() = second 35 | } 36 | 37 | 38 | /** 39 | * Pair copy ctor conversion 40 | */ 41 | 42 | operator fun , R : Pai2> invoke(p: P): Pai2 = 43 | object : Pai2 { 44 | override val first: F by p::first 45 | override val second: S by p::second 46 | } 47 | 48 | operator fun , R : Pai2> invoke(p: P): Pai2 = object : Pai2 { 49 | override val first: F by p::key 50 | override val second: S by p::value 51 | } 52 | 53 | } 54 | } 55 | 56 | fun Pai2<*, *>.toString(): String = pair.toString() 57 | 58 | 59 | /**inheritable version of triple 60 | */ 61 | interface Tripl3/* : Pai2 */ { 62 | val first: F 63 | val second: S 64 | val third: T 65 | 66 | /** 67 | * for println and serializable usecases, offload that stuff using this method. 68 | */ 69 | val triple: Triple get() = Triple(first, second, third) 70 | operator fun component1(): F = first 71 | operator fun component2(): S = second 72 | operator fun component3(): T = third 73 | 74 | companion object { 75 | 76 | inline operator fun invoke(first: F, second: S, third: T): Tripl3 = 77 | object : Tripl3/*, Pai2 by Pai2(f, s)*/ { 78 | override inline val first get() = first 79 | override inline val second get() = second 80 | override inline val third get() = third 81 | } 82 | 83 | inline operator fun invoke(p: Triple): Tripl3 = 84 | p.let { (f, s, t) -> Tripl3(f, s, t) } 85 | } 86 | } 87 | 88 | 89 | /** 90 | * means either/both of "tuple2" and disambiguation of pair =x "to" y 91 | */ 92 | 93 | /** 94 | * a pair with uniform types 95 | * 96 | * 97 | * homage to eclipse types 98 | */ 99 | typealias Tw1n = Pai2 100 | 101 | /** 102 | * a factory method 103 | */ 104 | fun Tw1n(first: T, second: T): Tw1n = first t2 second 105 | 106 | /** 107 | * int-type specific twin with primitive array backing store. 108 | */ 109 | 110 | @JvmInline 111 | value class Tw1nt(val ia: IntArray) : Tw1n { 112 | override inline val first: Int get() = ia[0] 113 | override inline val second: Int get() = ia[1] 114 | } 115 | 116 | /** 117 | * long-type specific twin with primitive array backing store. 118 | */ 119 | @JvmInline 120 | value class Twln(val ia: LongArray) : Tw1n { 121 | override inline val first: Long get() = ia[0] 122 | override inline val second: Long get() = ia[1] 123 | } 124 | 125 | @JvmName("twinint") 126 | fun Tw1n(first: T, second: T): Tw1nt = Tw1nt(_a[first, second]) 127 | 128 | @JvmName("twinlong") 129 | fun Tw1n(first: T, second: T): Twln = Twln(_a[first, second]) 130 | 131 | @JvmName("unaryMinusTw1n") 132 | operator fun Tw1n.unaryMinus(): Array = _a[first, second] 133 | 134 | @JvmName("unaryPlusI") 135 | inline operator fun Tw1n.unaryPlus(): IntRange = (-this).let { (a, b) -> a..b } 136 | 137 | @JvmName("aSInt") 138 | inline infix fun Tw1n.α(noinline f: (Int) -> R): Vect0r = (-this).α(f) 139 | 140 | @JvmName("aSDouble") 141 | inline infix fun Tw1n.α(noinline f: (Double) -> R): Vect0r = (-this).α(f) 142 | 143 | @JvmName("aSLong") 144 | inline infix fun Tw1n.α(noinline f: (Long) -> R): Vect0r = (-this).α(f) 145 | 146 | @JvmName("aSFloat") 147 | inline infix fun Tw1n.α(noinline f: (Float) -> R): Vect0r = (-this).α(f) 148 | 149 | @JvmName("aSByte") 150 | inline infix fun Tw1n.α(noinline f: (Byte) -> R): Vect0r = (-this).α(f) 151 | 152 | @JvmName("aSChar") 153 | inline infix fun Tw1n.α(noinline f: (Char) -> R): Vect0r = (-this).α(f) 154 | 155 | @JvmName("aSShort") 156 | inline infix fun Tw1n.α(noinline f: (Short) -> R): Vect0r = (-this).α(f) 157 | 158 | 159 | /** 160 | * create range of enums ordinals, inclusive +(enum1 t2 enum2) 161 | */ 162 | @JvmName("unaryPlusS") 163 | inline operator fun > Tw1n.unaryPlus(): IntRange = (-this).let { (a, b) -> a..b } 164 | 165 | /** 166 | * create range of enums ordinals, inclusive (enum1..enum2) 167 | */ 168 | inline infix operator fun > Enum.rangeTo(ub: Enum): IntRange = this.ordinal..ub.ordinal 169 | 170 | 171 | @JvmName("αS") 172 | inline infix fun Tw1n.α(noinline f: (S) -> R): Pai2 R> = (-this).α(f) 173 | infix fun F.t2(s: S): Pai2 = Pai2(this, s) 174 | inline infix fun Pai2.t3(t: T): Tripl3 = 175 | let { (f: F, s) -> Tripl3(f, s, t) } 176 | 177 | infix fun > P.t3(t: T): Tripl3 = let { (a, b) -> Tripl3(a, b, t) } 178 | 179 | //overlaps Vect0r reverse and isn't compatible 180 | inline fun Pair.reversed(): Pai2 = second t2 first 181 | inline infix fun Tripl3.t4(d: D): Qu4d = 182 | let { (a: A, b: B, c: C) -> Qu4d(a, b, c, d) } 183 | 184 | /**inheritable version of quad that also provides its first three as a triple. */ 185 | interface Qu4d { 186 | val first: F 187 | val second: S 188 | val third: T 189 | val fourth: Z 190 | 191 | data class Quad(val x: F, val y: S, val z: T, val w: Z) 192 | 193 | /** 194 | * for println and serializable usecases, offload that stuff using this method. 195 | */ 196 | val quad: Quad get() = Quad(first, second, third, fourth) 197 | 198 | operator fun component1(): F = first 199 | operator fun component2(): S = second 200 | operator fun component3(): T = third 201 | operator fun component4(): Z = fourth 202 | 203 | companion object { 204 | 205 | operator fun invoke( 206 | first: F, 207 | second: S, 208 | third: T, 209 | fourth: Z, 210 | ): Qu4d = object : Qu4d { 211 | override inline val first get() = first 212 | override inline val second get() = second 213 | override inline val third get() = third 214 | override inline val fourth get() = fourth 215 | } 216 | 217 | 218 | operator fun invoke(p: Quad): Qu4d = p.let { (f, s, t, z) -> 219 | Qu4d(f, s, t, z) 220 | } 221 | 222 | 223 | operator fun invoke(p: Array<*>): Qu4d = p.let { (f, s, t, z) -> 224 | @Suppress("UNCHECKED_CAST") (Qu4d(f as F, s as S, t as T, z as Z)) 225 | } 226 | 227 | 228 | operator fun invoke(p: List<*>): Qu4d = p.let { (f, s, t, z) -> 229 | @Suppress("UNCHECKED_CAST") (Qu4d(f as F, s as S, t as T, z as Z)) 230 | } 231 | }} 232 | -------------------------------------------------------------------------------- /src/test/java/cursors/DatabinanceKlineIsamTest.kt: -------------------------------------------------------------------------------- 1 | //package cursors 2 | // 3 | //import cursors.context.Scalar 4 | //import cursors.io.IOMemento 5 | //import cursors.io.RowVec 6 | //import kotlin.test.* 7 | //import java.util.EnumMap 8 | //import java.time.Instant // Not used in this specific test, but good for KLine context 9 | //import cursors.context.Scalar.Companion.Scalar // For factory 10 | //import cursors.context.`⟲` // For context provider 11 | //import cursors.io.colIdx 12 | //import cursors.io.scalars // Extension property for Cursor 13 | //import vec.macros.Vect02_.left 14 | // 15 | //// Define an inline class for KLine data for type safety and minimal overhead 16 | //@JvmInline 17 | //value class KLine(val data: Vect0r) { 18 | // // Accessors based on KLineColumn order 19 | // inline val openTime: Long get() = data[KLineColumn.OPEN_TIME.ordinal] as Long 20 | // inline val open: Double get() = data[KLineColumn.OPEN.ordinal] as Double 21 | // inline val high: Double get() = data[KLineColumn.HIGH.ordinal] as Double 22 | // inline val val low: Double get() = data[KLineColumn.LOW.ordinal] as Double 23 | // inline val close: Double get() = data[KLineColumn.CLOSE.ordinal] as Double 24 | // inline val volume: Double get() = data[KLineColumn.VOLUME.ordinal] as Double 25 | // inline val closeTime: Long get() = data[KLineColumn.CLOSE_TIME.ordinal] as Long 26 | // inline val quoteAssetVolume: Double get() = data[KLineColumn.QUOTE_ASSET_VOLUME.ordinal] as Double 27 | // inline val numberOfTrades: Int get() = data[KLineColumn.NUMBER_OF_TRADES.ordinal] as Int 28 | // inline val takerBuyBaseAssetVolume: Double get() = data[KLineColumn.TAKER_BUY_BASE_ASSET_VOLUME.ordinal] as Double 29 | // inline val takerBuyQuoteAssetVolume: Double get() = data[KLineColumn.TAKER_BUY_QUOTE_ASSET_VOLUME.ordinal] as Double 30 | // 31 | // val isBullish: Boolean get() = close > open 32 | // val isBearish: Boolean get() = close < open 33 | // val range: Double get() = high - low 34 | //} 35 | // 36 | //// Enum to model KLine columns for type-safe access and metadata 37 | //enum class KLineColumn(val colName: String, val type: IOMemento) { 38 | // OPEN_TIME("openTime", IOMemento.IoLong), 39 | // OPEN("open", IOMemento.IoDouble), 40 | // HIGH("high", IOMemento.IoDouble), 41 | // LOW("low", IOMemento.IoDouble), 42 | // CLOSE("close", IOMemento.IoDouble), 43 | // VOLUME("volume", IOMemento.IoDouble), 44 | // CLOSE_TIME("closeTime", IOMemento.IoLong), 45 | // QUOTE_ASSET_VOLUME("quoteAssetVolume", IOMemento.IoDouble), 46 | // NUMBER_OF_TRADES("numberOfTrades", IOMemento.IoInt), 47 | // TAKER_BUY_BASE_ASSET_VOLUME("takerBuyBaseAssetVolume", IOMemento.IoDouble), 48 | // TAKER_BUY_QUOTE_ASSET_VOLUME("takerBuyQuoteAssetVolume", IOMemento.IoDouble), 49 | // IGNORE("ignore", IOMemento.IoString); // Typically present in Binance data 50 | // 51 | // fun toScalar(): Scalar = Scalar(type, colName) 52 | //} 53 | // 54 | //// Simulate a columnar ISAM builder 55 | //class ColumnarKLineStore { 56 | // // scar: Changed to EnumMap for potentially better performance with enum keys. 57 | // private val columns = EnumMap>(KLineColumn::class.java) 58 | // 59 | // fun addKLine(parsedCsvRow: List) { 60 | // // Simplified parsing, assumes CSV maps directly to KLineColumn order 61 | // if (parsedCsvRow.size < KLineColumn.entries.size) return // Skip malformed rows 62 | // 63 | // KLineColumn.entries.forEachIndexed { index, kLineCol -> 64 | // val list = columns.getOrPut(kLineCol) { mutableListOf() } 65 | // val valueStr = parsedCsvRow[index] 66 | // // scar: Added IoInt case. 67 | // list.add(when (kLineCol.type) { 68 | // IOMemento.IoLong -> valueStr.toLong() 69 | // IOMemento.IoDouble -> valueStr.toDouble() 70 | // IOMemento.IoInt -> valueStr.toInt() 71 | // IOMemento.IoString -> valueStr 72 | // else -> valueStr // Fallback for other types, though KLineColumn defines types explicitly 73 | // }) 74 | // } 75 | // } 76 | // 77 | // // Creates a Cursor view over the columnar data 78 | // fun getCursor(): Cursor { 79 | // if (columns.isEmpty() || columns.values.any { it.isEmpty() }) { 80 | // return SimpleCursor(Vect0r(0) { Scalar(IOMemento.IoNothing, "empty_kline_store") }, Vect0r(0) { Vect0r(0){""} }) 81 | // } 82 | // 83 | // val numRows = columns.values.first().size 84 | // val scalars = Vect0r(KLineColumn.entries.size) { i -> KLineColumn.entries[i].toScalar() } 85 | // 86 | // // Create a Vect0r of RowVecs. Each RowVec represents a row, but internally 87 | // // it holds references to the columnar data and the column index. 88 | // val rows = Vect0r(numRows) { rowIndex -> 89 | // RowVec(Vect0r(KLineColumn.entries.size) { colIndex -> 90 | // // This is the core of the columnar view: accessing data by column then row 91 | // val klineCol = KLineColumn.entries[colIndex] 92 | // val columnData = columns[klineCol] ?: error("Column data not found for ${klineCol.colName}") 93 | // columnData[rowIndex] 94 | // }) 95 | // } 96 | // 97 | // return SimpleCursor(scalars, rows) 98 | // } 99 | //} 100 | // 101 | //// DSEL Extensions for Cursor related to KLines 102 | //fun Cursor.kline(index: Int): KLine = KLine(this.at(index).left) 103 | // 104 | //inline fun Cursor.klineColumn(column: KLineColumn): Vect0r { 105 | // val colIdx = this.scalars.indexOfFirst { it.name == column.colName } 106 | // if (colIdx == -1) throw IllegalArgumentException("Column ${column.colName} not found in cursor") 107 | // return Vect0r(this.size) { rowIndex -> 108 | // (this.at(rowIndex)[colIdx].first as T) 109 | // } 110 | //} 111 | // 112 | //val Cursor.openTimes: Vect0r get() = klineColumn(KLineColumn.OPEN_TIME) 113 | //val Cursor.opens: Vect0r get() = klineColumn(KLineColumn.OPEN) 114 | //val Cursor.highs: Vect0r get() = klineColumn(KLineColumn.HIGH) 115 | //val Cursor.lows: Vect0r get() = klineColumn(KLineColumn.LOW) 116 | //val Cursor.closes: Vect0r get() = klineColumn(KLineColumn.CLOSE) 117 | //val Cursor.volumes: Vect0r get() = klineColumn(KLineColumn.VOLUME) 118 | //val Cursor.numTrades: Vect0r get() = klineColumn(KLineColumn.NUMBER_OF_TRADES) 119 | // 120 | //fun Cursor.filterKLines(predicate: (KLine) -> Boolean): Cursor { 121 | // val matchingRowIndices = mutableListOf() 122 | // for (i in 0 until this.size) { 123 | // if (predicate(this.kline(i))) { 124 | // matchingRowIndices.add(i) 125 | // } 126 | // } 127 | // if (matchingRowIndices.isEmpty()) return SimpleCursor(this.scalars, Vect0r(0){Vect0r(0){""}}) 128 | // return this[matchingRowIndices.toIntArray()] 129 | //} 130 | // 131 | //class DatabinanceKlineIsamTest { 132 | // 133 | // @Test 134 | // fun `should process KLine CSV data into columnar store and access via cursor`() { 135 | // val csvData = listOf( 136 | // "1672531200000,0.1,0.12,0.09,0.11,1000.0,1672531259999,100.11,10,500.0,50.05,ignore", 137 | // "1672531260000,0.11,0.15,0.10,0.14,1500.0,1672531319999,150.14,15,700.0,70.14,ignore", 138 | // "1672531320000,0.14,0.14,0.12,0.13,1200.0,1672531379999,120.13,12,600.0,60.13,ignore" 139 | // ) 140 | // 141 | // val store = ColumnarKLineStore() 142 | // csvData.forEach { row -> 143 | // store.addKLine(row.split(',')) 144 | // } 145 | // 146 | // val cursor = store.getCursor() 147 | // 148 | // // Test basic cursor properties 149 | // assertEquals(expected = 3, actual = cursor.size, message = "Cursor should have 3 rows") 150 | // assertEquals(expected = KLineColumn.entries.size, actual = cursor.scalars.size, "Cursor should have ${KLineColumn.entries.size} columns") 151 | // 152 | // // Verify first row, close price 153 | // val firstRowVec = cursor.at(0) 154 | // val closePriceFirstRow = firstRowVec[KLineColumn.CLOSE.ordinal].first 155 | // assertEquals(expected = 0.11, actual = closePriceFirstRow as Double, message = "Close price of first KLine") 156 | // 157 | // // Verify second row, open time via KLine value class accessor 158 | // val klineDataSecondRow = KLine(cursor.at(1).left) // Wrap row data in KLine value class 159 | // val openTimeSecondRow = klineDataSecondRow.openTime 160 | // assertEquals(expected = 1672531260000L, actual = openTimeSecondRow, message = "Open time of second KLine") 161 | // assertTrue(klineDataSecondRow.isBullish, "Second KLine should be bullish") 162 | // 163 | // // Verify third row, volume 164 | // val volumeThirdRow = (cursor.at(2)[KLineColumn.VOLUME.ordinal].first as Double) 165 | // assertEquals(expected = 1200.0, actual = volumeThirdRow, message = "Volume of third KLine") 166 | // assertEquals(0.02, cursor.kline(2).range, 1e-9, "Range of third KLine") 167 | // 168 | // // Verify scalar metadata 169 | // // scar: Scalar is Pai2, .second is name, .first is type 170 | // assertEquals(expected = "volume", actual = cursor.scalars[KLineColumn.VOLUME.ordinal].second) 171 | // assertEquals(expected = IOMemento.IoDouble, actual = cursor.scalars[KLineColumn.VOLUME.ordinal].first) 172 | // assertEquals(expected = "numberOfTrades", actual = cursor.scalars[KLineColumn.NUMBER_OF_TRADES.ordinal].second) 173 | // assertEquals(expected = IOMemento.IoInt, actual = cursor.scalars[KLineColumn.NUMBER_OF_TRADES.ordinal].first) 174 | // 175 | // // Demonstrate DSEL extensions 176 | // assertEquals(0.1, cursor.opens[0], "DSEL: First open price") 177 | // assertEquals(0.14, cursor.closes[1], "DSEL: Second close price") 178 | // assertEquals(10, cursor.numTrades[0], "DSEL: Number of trades in first KLine") 179 | // assertEquals(1672531200000L, cursor.kline(0).openTime, "DSEL: Open time of first KLine object") 180 | // 181 | // val highVolumeKlines = cursor.filterKLines { it.volume > 1000.0 } 182 | // assertEquals(2, highVolumeKlines.size, "DSEL: Should be 2 KLines with volume > 1000") 183 | // assertEquals(0.14, highVolumeKlines.kline(0).close, "DSEL: Close of first high-volume KLine") 184 | // assertEquals(0.13, highVolumeKlines.kline(1).close, "DSEL: Close of second high-volume KLine") 185 | // } 186 | //} 187 | --------------------------------------------------------------------------------