├── .clj-kondo └── config.edn ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build-data ├── mathematica-gradients.edn └── parsed-gradients.edn ├── docs ├── 00-sequences-and-arrays.md ├── 01-operations.md ├── 02-base-concepts.md ├── 03-sparse.md ├── 04-tensor.md ├── buffered-image.md ├── cheatsheet.md ├── colorize.md ├── datetime.md ├── dimensions-bytecode-gen.md ├── images │ ├── morning-conus.png │ └── noon-conus.png └── walkthrough.md ├── example-bytecode ├── ASTExampleClass.asm ├── ASTExampleClass.java ├── ASTExampleClassRdr.asm └── ASTExampleClassRdr.java ├── java └── tech │ └── v2 │ ├── datatype │ ├── BinaryOperators.java │ ├── BinaryReader.java │ ├── BinaryWriter.java │ ├── BitmapMap.java │ ├── BooleanIter.java │ ├── BooleanMutable.java │ ├── BooleanOp.java │ ├── BooleanReader.java │ ├── BooleanReaderIter.java │ ├── BooleanWriter.java │ ├── ByteConversions.java │ ├── ByteIter.java │ ├── ByteMutable.java │ ├── ByteReader.java │ ├── ByteReaderIter.java │ ├── ByteWriter.java │ ├── Comparator.java │ ├── ConversionOp.java │ ├── Countable.java │ ├── Datatype.java │ ├── DateUtility.java │ ├── DirectMappedOps.java │ ├── DoubleIter.java │ ├── DoubleMutable.java │ ├── DoubleReader.java │ ├── DoubleReaderIter.java │ ├── DoubleWriter.java │ ├── FloatIter.java │ ├── FloatMutable.java │ ├── FloatReader.java │ ├── FloatReaderIter.java │ ├── FloatWriter.java │ ├── Functions.java │ ├── IOBase.java │ ├── IndexingSystem.java │ ├── IntIter.java │ ├── IntMutable.java │ ├── IntReader.java │ ├── IntReaderIter.java │ ├── IntWriter.java │ ├── IterHelpers.java │ ├── IteratorObjectIter.java │ ├── ListPersistentVector.java │ ├── LongBitmapIter.java │ ├── LongIter.java │ ├── LongMutable.java │ ├── LongReader.java │ ├── LongReaderIter.java │ ├── LongWriter.java │ ├── MutableRemove.java │ ├── ObjectIter.java │ ├── ObjectMutable.java │ ├── ObjectReader.java │ ├── ObjectReaderIter.java │ ├── ObjectWriter.java │ ├── PackedInstant.java │ ├── PackedLocalDate.java │ ├── PackedLocalDateTime.java │ ├── PackedLocalTime.java │ ├── RangeDoubleSpliterator.java │ ├── ShortIter.java │ ├── ShortMutable.java │ ├── ShortReader.java │ ├── ShortReaderIter.java │ ├── ShortWriter.java │ ├── SimpleLongSet.java │ └── UnaryOperators.java │ └── tensor │ ├── BooleanTensorReader.java │ ├── BooleanTensorWriter.java │ ├── ByteTensorReader.java │ ├── ByteTensorWriter.java │ ├── DoubleTensorReader.java │ ├── DoubleTensorWriter.java │ ├── FloatTensorReader.java │ ├── FloatTensorWriter.java │ ├── IntTensorReader.java │ ├── IntTensorWriter.java │ ├── LongTensorReader.java │ ├── LongTensorWriter.java │ ├── ObjectTensorReader.java │ ├── ObjectTensorWriter.java │ ├── ShortTensorReader.java │ └── ShortTensorWriter.java ├── neanderthal └── tech │ └── libs │ └── neanderthal_test.clj ├── project.clj ├── resources ├── gradients.edn └── gradients.png ├── src └── tech │ ├── compute.clj │ ├── compute │ ├── context.clj │ ├── cpu │ │ ├── driver.clj │ │ └── utils.clj │ ├── driver.clj │ ├── registry.clj │ ├── tensor.clj │ └── verify │ │ ├── driver.clj │ │ ├── tensor.clj │ │ └── utils.clj │ ├── libs │ ├── buffered_image.clj │ └── neanderthal.clj │ └── v2 │ ├── datatype.clj │ ├── datatype │ ├── argsort.clj │ ├── argtypes.clj │ ├── array.clj │ ├── base.cljc │ ├── base_macros.clj │ ├── base_macros.cljs │ ├── binary_impl_helper.clj │ ├── binary_op.clj │ ├── binary_reader.clj │ ├── binary_search.clj │ ├── binary_writer.clj │ ├── bitmap.clj │ ├── boolean_op.clj │ ├── builtin_op_providers.clj │ ├── casting.clj │ ├── clj_range.clj │ ├── comparator.clj │ ├── datetime.clj │ ├── datetime │ │ └── operations.clj │ ├── direct_mapped.clj │ ├── double_ops.clj │ ├── fast_copy.clj │ ├── functional.clj │ ├── functional │ │ ├── impl.clj │ │ └── interpreter.clj │ ├── index_algebra.clj │ ├── io.clj │ ├── iterable │ │ ├── concat.clj │ │ ├── const.clj │ │ └── masked.clj │ ├── iterator.clj │ ├── javacpp.clj │ ├── jna.clj │ ├── list.clj │ ├── mmap.clj │ ├── monotonic_range.clj │ ├── mutable.clj │ ├── mutable │ │ └── iterable_to_list.clj │ ├── nio_access.clj │ ├── nio_buffer.clj │ ├── object_datatypes.clj │ ├── operation_provider.clj │ ├── pprint.clj │ ├── primitive.clj │ ├── protocols.clj │ ├── reader.clj │ ├── readers │ │ ├── concat.clj │ │ ├── const.clj │ │ ├── indexed.clj │ │ ├── range.clj │ │ ├── reverse.clj │ │ └── update.clj │ ├── record.clj │ ├── reduce_op.clj │ ├── rolling.clj │ ├── shape.clj │ ├── sparse │ │ ├── protocols.clj │ │ ├── reader.clj │ │ ├── sparse_base.clj │ │ └── sparse_buffer.clj │ ├── statistics.clj │ ├── struct.clj │ ├── type_resolution.clj │ ├── typecast.clj │ ├── typed_buffer.clj │ ├── unary_op.clj │ ├── vector_of.clj │ ├── writer.clj │ └── writers │ │ ├── concat.clj │ │ ├── indexed.clj │ │ └── iterable_to_writer.clj │ ├── libs │ └── blas.clj │ ├── tensor.clj │ └── tensor │ ├── color_gradients.clj │ ├── color_gradients │ └── build_gradients.clj │ ├── dimensions.clj │ ├── dimensions │ ├── analytics.clj │ ├── global_to_local.clj │ ├── select.clj │ └── shape.clj │ ├── impl.clj │ ├── pprint.clj │ ├── protocols.clj │ ├── tensor_copy.clj │ ├── typecast.clj │ └── utils.clj └── test ├── data └── test.jpg └── tech ├── compute └── cpu │ ├── driver_test.clj │ └── tensor_test.clj ├── libs └── buffered_image_test.clj └── v2 ├── apl ├── game_of_life.clj └── sparse_game_of_life.clj ├── datatype ├── array_test.clj ├── bitmap_test.clj ├── byte_conversions_test.clj ├── cheatsheet_test.clj ├── datetime_test.clj ├── index_algebra_test.clj ├── javacpp_test.clj ├── jna_test.clj ├── mmap_test.clj ├── object_arrays_test.clj ├── reader_test.clj ├── simple_ops_perf_test.clj ├── sparse │ └── sparse_buffer_test.clj ├── streams_test.clj ├── typed_buffer_test.clj └── vector_of_test.clj ├── datatype_test.clj └── tensor ├── broadcast_test.clj ├── color_gradient_test.clj ├── copy_test.clj ├── dimensions └── global_to_local_test.clj ├── dimensions_test.clj ├── index_system_test.clj ├── integration_test.clj ├── mmul_test.clj └── select_test.clj /.clj-kondo/config.edn: -------------------------------------------------------------------------------- 1 | {:linters {:unresolved-symbol {:level :off}}} 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | .nrepl-port 10 | .hgignore 11 | .hg/ 12 | checkouts 13 | *.log 14 | .cpcache 15 | .clj-kondo/.cache 16 | gradient-demo 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: clojure 2 | lein: 2.9.3 3 | jdk: openjdk8 4 | before_install: 5 | - curl -O https://download.clojure.org/install/linux-install-1.10.0.403.sh && chmod +x linux-install-1.10.0.403.sh && sudo ./linux-install-1.10.0.403.sh 6 | script: 7 | lein with-profile travis test 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tech.datatype 2 | [![Clojars Project](https://clojars.org/techascent/tech.datatype/latest-version.svg)](https://clojars.org/techascent/tech.datatype) 3 | 4 | * This library has been superceded by [dtype-next](https://github.com/cnuernber/dtype-next/) in order to provide: 5 | * Smaller runtime footprint + lower require times. 6 | * Fewer dependencies. 7 | * Graal Native support. 8 | 9 | 10 | `tech.datatype` is a Clojure library for efficient N-dimensional numerics across a range of primitive 11 | datatypes and a range of container types. 12 | 13 | As examples of what you may do, here are some colorized tensors built from the US NOAA 14 | HRRR and GFS models. 15 | 16 | | Morning Temps | Noon Temps | 17 | |--------- |------- | 18 | |![morning](docs/images/morning-conus.png)|![noon](docs/images/noon-conus.png)| 19 | 20 | 21 | * For a quick feel of the library, please check out our [cheatsheet](docs/cheatsheet.md) 22 | * Those images were generated by [colorizing tensors](docs/colorize.md) 23 | * Check out our docs on [buffered images](docs/buffered-image.md) 24 | * Brand new - [datetime support](docs/datetime.md) 25 | * APL-style [game of life](test/tech/v2/apl/game_of_life.clj) 26 | 27 | 28 | A nice (slightly out of date) post explaining more is 29 | [here](http://techascent.com/blog/datatype-library.html). 30 | 31 | 32 | Generalized efficient manipulations of sequences of primitive datatype. 33 | Includes specializations for java arrays, array views (subsection of an array) 34 | and nio buffers. There are specializations to allow implementations to provide 35 | efficient full typed copy functions when the types can be ascertained. 36 | 37 | Generic operations include: 38 | 1. datatype of this sequence 39 | 2. Writing to, reading from 40 | 3. Construction 41 | 4. Efficient mutable copy into a container. 42 | 5. Sparse buffer support 43 | 6. n-dimensional tensor support 44 | 7. Functional math support 45 | 46 | 47 | 48 | Design documenation is [here](docs). 49 | 50 | 51 | ## Useful Unit Tests To See 52 | 53 | * [basic operations](test/tech/v2/datatype_test.clj) 54 | * [unsigned operations](test/tech/v2/datatype/typed_buffer_test.clj) 55 | * [game of life](test/tech/v2/apl/game_of_life.clj) 56 | * [sparse game of life](test/tech/v2/apl/sparse_game_of_life.clj) 57 | 58 | 59 | ## License 60 | 61 | Copyright © 2019 TechAscent, LLC. 62 | 63 | Distributed under the Eclipse Public License either version 1.0 or (at 64 | your option) any later version. 65 | -------------------------------------------------------------------------------- /build-data/mathematica-gradients.edn: -------------------------------------------------------------------------------- 1 | {:base-url "https://reference.wolfram.com/language/guide/"} 2 | -------------------------------------------------------------------------------- /build-data/parsed-gradients.edn: -------------------------------------------------------------------------------- 1 | {:neon-colors "Files/ColorSchemes.en/6.png", :aquamarine "Files/ColorSchemes.en/44.png", :coffee-tones "Files/ColorSchemes.en/31.png", :deep-sea-colors "Files/ColorSchemes.en/17.png", :fuchsia-tones "Files/ColorSchemes.en/33.png", :red-blue-tones "Files/ColorSchemes.en/30.png", :brass-tones "Files/ColorSchemes.en/25.png", :army-colors "Files/ColorSchemes.en/3.png", :green-pink-tones "Files/ColorSchemes.en/26.png", :green-brown-terrain "Files/ColorSchemes.en/40.png", :thermometer-colors "Files/ColorSchemes.en/22.png", :cherry-tones "Files/ColorSchemes.en/29.png", :atlantic-colors "Files/ColorSchemes.en/5.png", :dark-terrain "Files/ColorSchemes.en/38.png", :fall-colors "Files/ColorSchemes.en/19.png", :rainbow "Files/ColorSchemes.en/47.png", :fruit-punch-colors "Files/ColorSchemes.en/21.png", :solar-colors "Files/ColorSchemes.en/14.png", :dark-bands "Files/ColorSchemes.en/43.png", :avocado-colors "Files/ColorSchemes.en/9.png", :light-temperature-map "Files/ColorSchemes.en/50.png", :sunset-colors "Files/ColorSchemes.en/20.png", :candy-colors "Files/ColorSchemes.en/13.png", :watermelon-colors "Files/ColorSchemes.en/24.png", :beach-colors "Files/ColorSchemes.en/11.png", :island-colors "Files/ColorSchemes.en/23.png", :pastel "Files/ColorSchemes.en/45.png", :gray-yellow-tones "Files/ColorSchemes.en/37.png", :temperature-map "Files/ColorSchemes.en/49.png", :blue-green-yellow "Files/ColorSchemes.en/46.png", :bright-bands "Files/ColorSchemes.en/42.png", :sandy-terrain "Files/ColorSchemes.en/41.png", :gray-tones "Files/ColorSchemes.en/35.png", :rust-tones "Files/ColorSchemes.en/32.png", :pearl-colors "Files/ColorSchemes.en/8.png", :dark-rainbow "Files/ColorSchemes.en/48.png", :valentine-tones "Files/ColorSchemes.en/36.png", :cmyk-colors "Files/ColorSchemes.en/15.png", :mint-colors "Files/ColorSchemes.en/4.png", :lake-colors "Files/ColorSchemes.en/2.png", :pigeon-tones "Files/ColorSchemes.en/28.png", :aurora-colors "Files/ColorSchemes.en/7.png", :plum-colors "Files/ColorSchemes.en/10.png", :alpine-colors "Files/ColorSchemes.en/1.png", :light-terrain "Files/ColorSchemes.en/39.png", :southwest-colors "Files/ColorSchemes.en/16.png", :sienna-tones "Files/ColorSchemes.en/34.png", :brown-cyan-tones "Files/ColorSchemes.en/27.png", :rose-colors "Files/ColorSchemes.en/12.png", :starry-night-colors "Files/ColorSchemes.en/18.png"} -------------------------------------------------------------------------------- /docs/colorize.md: -------------------------------------------------------------------------------- 1 | # Colorizing 2 | 3 | 4 | The images displayed of US surface temperatures where generated via colorizing 5 | a tensor: 6 | 7 | ```clojure 8 | (defn color-lerp 9 | ^BufferedImage [api-varname variable-data tile-size] 10 | (if-let [color-data (get default-api-variable-color-map api-varname)] 11 | (let [{:keys [min-value max-value gradient-name]} color-data] 12 | (when-not (= (dtype/ecount variable-data) 13 | (* tile-size tile-size)) 14 | (throw (Exception. "Variable data is wrong size."))) 15 | (-> (dtt/reshape variable-data [tile-size tile-size]) 16 | (dtt-gradients/colorize gradient-name 17 | :data-min min-value 18 | :data-max max-value))) 19 | (throw (Exception. (format "Failed to find color data for var: %s" api-varname))))) 20 | ``` 21 | 22 | * [Gradient Demo](https://github.com/techascent/tech.datatype/blob/640733fa994f96dd3058e9ec83049d2efb3802c4/src/tech/v2/tensor/color_gradients.clj#L177) 23 | * [Documentation for colorizing](https://github.com/techascent/tech.datatype/blob/640733fa994f96dd3058e9ec83049d2efb3802c4/src/tech/v2/tensor/color_gradients.clj#L134) 24 | -------------------------------------------------------------------------------- /docs/images/morning-conus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techascent/tech.datatype/8cc83d771d9621d580fd5d4d0625005bd7ab0e0c/docs/images/morning-conus.png -------------------------------------------------------------------------------- /docs/images/noon-conus.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techascent/tech.datatype/8cc83d771d9621d580fd5d4d0625005bd7ab0e0c/docs/images/noon-conus.png -------------------------------------------------------------------------------- /example-bytecode/ASTExampleClass.asm: -------------------------------------------------------------------------------- 1 | Compiled from "ASTExampleClass.java" 2 | public class tech.v2.tensor.dimensions.ASTExampleClass { 3 | public final long maxShapeStride0; 4 | 5 | public final long stride0; 6 | 7 | public final long shape1; 8 | 9 | public final long nElems; 10 | 11 | public tech.v2.tensor.dimensions.ASTExampleClass(java.lang.Object[], long[], long[], long[], long[]); 12 | Code: 13 | 0: aload_0 14 | 1: invokespecial #1 // Method java/lang/Object."":()V 15 | 4: aload_0 16 | 5: aload 5 17 | 7: iconst_0 18 | 8: laload 19 | 9: putfield #2 // Field maxShapeStride0:J 20 | 12: aload_0 21 | 13: aload_2 22 | 14: iconst_0 23 | 15: laload 24 | 16: putfield #3 // Field stride0:J 25 | 19: aload_0 26 | 20: aload_1 27 | 21: iconst_1 28 | 22: aaload 29 | 23: checkcast #4 // class java/lang/Long 30 | 26: invokevirtual #5 // Method java/lang/Long.longValue:()J 31 | 29: putfield #6 // Field shape1:J 32 | 32: aload_0 33 | 33: aload 4 34 | 35: iconst_0 35 | 36: laload 36 | 37: aload 5 37 | 39: iconst_0 38 | 40: laload 39 | 41: lmul 40 | 42: putfield #7 // Field nElems:J 41 | 45: return 42 | 43 | public long lsize(); 44 | Code: 45 | 0: aload_0 46 | 1: getfield #7 // Field nElems:J 47 | 4: lreturn 48 | 49 | public long read(long); 50 | Code: 51 | 0: lload_1 52 | 1: aload_0 53 | 2: getfield #2 // Field maxShapeStride0:J 54 | 5: ldiv 55 | 6: aload_0 56 | 7: getfield #3 // Field stride0:J 57 | 10: lmul 58 | 11: lload_1 59 | 12: aload_0 60 | 13: getfield #6 // Field shape1:J 61 | 16: lrem 62 | 17: ladd 63 | 18: lreturn 64 | } 65 | -------------------------------------------------------------------------------- /example-bytecode/ASTExampleClass.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor.dimensions; 2 | 3 | import tech.v2.datatype.LongReader; 4 | 5 | //(+ (* (quot idx max-shape-stride-0) stride-0) (rem idx shape-1)) 6 | public class ASTExampleClass 7 | { 8 | public final long maxShapeStride0; 9 | public final long stride0; 10 | public final long shape1; 11 | public final long nElems; 12 | public ASTExampleClass( Object[] shape, long[] strides, long[] offsets, 13 | long[] maxShape, long[] maxShapeStride ) { 14 | maxShapeStride0 = maxShapeStride[0]; 15 | stride0 = strides[0]; 16 | shape1 = (long) shape[1]; 17 | nElems = maxShape[0] * maxShapeStride[0]; 18 | } 19 | public long lsize() { return nElems; } 20 | public long read(long idx) { 21 | return ((idx / maxShapeStride0) * stride0) 22 | + (idx % shape1); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /example-bytecode/ASTExampleClassRdr.asm: -------------------------------------------------------------------------------- 1 | Compiled from "ASTExampleClassRdr.java" 2 | public class tech.v2.tensor.dimensions.ASTExampleClassRdr { 3 | public final long maxShapeStride0; 4 | 5 | public final long stride0; 6 | 7 | public final tech.v2.datatype.LongReader shape1; 8 | 9 | public final long nElems; 10 | 11 | public tech.v2.tensor.dimensions.ASTExampleClassRdr(java.lang.Object[], long[], long[], long[], long[]); 12 | Code: 13 | 0: aload_0 14 | 1: invokespecial #1 // Method java/lang/Object."":()V 15 | 4: aload_0 16 | 5: aload 5 17 | 7: iconst_0 18 | 8: laload 19 | 9: putfield #2 // Field maxShapeStride0:J 20 | 12: aload_0 21 | 13: aload_2 22 | 14: iconst_0 23 | 15: laload 24 | 16: putfield #3 // Field stride0:J 25 | 19: aload_0 26 | 20: aload_1 27 | 21: iconst_1 28 | 22: aaload 29 | 23: checkcast #4 // class tech/v2/datatype/LongReader 30 | 26: putfield #5 // Field shape1:Ltech/v2/datatype/LongReader; 31 | 29: aload_0 32 | 30: aload 4 33 | 32: iconst_0 34 | 33: laload 35 | 34: aload 5 36 | 36: iconst_0 37 | 37: laload 38 | 38: lmul 39 | 39: putfield #6 // Field nElems:J 40 | 42: return 41 | 42 | public long lsize(); 43 | Code: 44 | 0: aload_0 45 | 1: getfield #6 // Field nElems:J 46 | 4: lreturn 47 | 48 | public long read(long); 49 | Code: 50 | 0: lload_1 51 | 1: aload_0 52 | 2: getfield #2 // Field maxShapeStride0:J 53 | 5: ldiv 54 | 6: aload_0 55 | 7: getfield #3 // Field stride0:J 56 | 10: lmul 57 | 11: aload_0 58 | 12: getfield #5 // Field shape1:Ltech/v2/datatype/LongReader; 59 | 15: lload_1 60 | 16: aload_0 61 | 17: getfield #5 // Field shape1:Ltech/v2/datatype/LongReader; 62 | 20: invokeinterface #7, 1 // InterfaceMethod tech/v2/datatype/LongReader.lsize:()J 63 | 25: lrem 64 | 26: invokeinterface #8, 3 // InterfaceMethod tech/v2/datatype/LongReader.read:(J)J 65 | 31: ladd 66 | 32: lreturn 67 | } 68 | -------------------------------------------------------------------------------- /example-bytecode/ASTExampleClassRdr.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor.dimensions; 2 | 3 | import tech.v2.datatype.LongReader; 4 | 5 | //(+ (* (quot idx max-shape-stride-0) stride-0) (rem idx shape-1)) 6 | public class ASTExampleClassRdr 7 | { 8 | public final long maxShapeStride0; 9 | public final long stride0; 10 | public final LongReader shape1; 11 | public final long nElems; 12 | public ASTExampleClassRdr( Object[] shape, long[] strides, long[] offsets, 13 | long[] maxShape, long[] maxShapeStride ) { 14 | maxShapeStride0 = maxShapeStride[0]; 15 | stride0 = strides[0]; 16 | shape1 = (LongReader) shape[1]; 17 | nElems = maxShape[0] * maxShapeStride[0]; 18 | } 19 | public long lsize() { return nElems; } 20 | public long read(long idx) { 21 | return 22 | ((idx / maxShapeStride0) * stride0) 23 | + shape1.read(idx % shape1.lsize()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BinaryOperators.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | 5 | public class BinaryOperators 6 | { 7 | public interface ByteBinary extends Datatype, IFn 8 | { 9 | byte op(byte lhs, byte rhs); 10 | default byte finalize(byte accum, long num_elems) { 11 | return accum; 12 | } 13 | default Object invoke(Object lhs, Object rhs) { 14 | return op((byte) lhs, (byte) rhs); 15 | } 16 | }; 17 | public interface ShortBinary extends Datatype, IFn 18 | { 19 | short op(short lhs, short rhs); 20 | default short finalize(short accum, long num_elems) { 21 | return accum; 22 | } 23 | default Object invoke(Object lhs, Object rhs) { 24 | return op((short) lhs, (short) rhs); 25 | } 26 | }; 27 | public interface IntBinary extends Datatype, IFn 28 | { 29 | int op(int lhs, int rhs); 30 | default int finalize(int accum, long num_elems) { 31 | return accum; 32 | } 33 | default Object invoke(Object lhs, Object rhs) 34 | { 35 | return op((int) lhs, (int) rhs); 36 | } 37 | }; 38 | public interface LongBinary extends Datatype, IFn 39 | { 40 | long op(long lhs, long rhs); 41 | default long finalize(long accum, long num_elems) { 42 | return accum; 43 | } 44 | default Object invoke(Object lhs, Object rhs) { 45 | return op((long) lhs, (long) rhs); 46 | } 47 | }; 48 | public interface FloatBinary extends Datatype, IFn 49 | { 50 | float op(float lhs, float rhs); 51 | default float finalize(float accum, long num_elems) { 52 | return accum; 53 | } 54 | default Object invoke(Object lhs, Object rhs) { 55 | return op((float) lhs, (float) rhs); 56 | } 57 | }; 58 | public interface DoubleBinary extends Datatype, IFn 59 | { 60 | double op(double lhs, double rhs); 61 | default double finalize(double accum, long num_elems) { 62 | return accum; 63 | } 64 | default Object invoke(Object lhs, Object rhs) { 65 | return op((double) lhs, (double) rhs); 66 | } 67 | }; 68 | public interface BooleanBinary extends Datatype, IFn 69 | { 70 | boolean op(boolean lhs, boolean rhs); 71 | default boolean finalize(boolean accum, long num_elems) { 72 | return accum; 73 | } 74 | default Object invoke(Object lhs, Object rhs) 75 | { 76 | return op((boolean) lhs, (boolean) rhs); 77 | } 78 | }; 79 | public interface ObjectBinary extends Datatype, IFn 80 | { 81 | Object op(Object lhs, Object rhs); 82 | default Object finalize(Object accum, long num_elems) { 83 | return accum; 84 | } 85 | default Object invoke(Object lhs, Object rhs) 86 | { 87 | return op(lhs, rhs); 88 | } 89 | }; 90 | } 91 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BinaryReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public interface BinaryReader 5 | { 6 | boolean readBoolean(long offset); 7 | byte readByte(long offset); 8 | short readShort(long offset); 9 | int readInt(long offset); 10 | long readLong(long offset); 11 | float readFloat(long offset); 12 | double readDouble(long offset); 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BinaryWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public interface BinaryWriter 5 | { 6 | void writeBoolean(boolean data, long offset); 7 | void writeByte(byte data, long offset); 8 | void writeShort(short data, long offset); 9 | void writeInt(short data, long offset); 10 | void writeLong(long data, long offset); 11 | void writeFloat(float data, long offset); 12 | void writeDouble(double data, long offset); 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.booleans.BooleanIterator; 4 | 5 | 6 | public interface BooleanIter extends Datatype, BooleanIterator 7 | { 8 | boolean current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface BooleanMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "boolean"); } 8 | void insert(long idx, boolean value); 9 | default void append(boolean value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanOp.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public class BooleanOp 5 | { 6 | public interface ByteUnary 7 | { 8 | boolean op(byte value); 9 | } 10 | public interface ShortUnary 11 | { 12 | boolean op(short value); 13 | } 14 | public interface IntUnary 15 | { 16 | boolean op(int value); 17 | } 18 | public interface LongUnary 19 | { 20 | boolean op(long value); 21 | } 22 | public interface FloatUnary 23 | { 24 | boolean op(float value); 25 | } 26 | public interface DoubleUnary 27 | { 28 | boolean op(double value); 29 | } 30 | public interface ObjectUnary 31 | { 32 | boolean op(Object value); 33 | } 34 | 35 | 36 | public interface ByteBinary 37 | { 38 | boolean op(byte lhs, byte rhs); 39 | } 40 | public interface ShortBinary 41 | { 42 | boolean op(short lhs, short rhs); 43 | } 44 | public interface IntBinary 45 | { 46 | boolean op(int lhs, int rhs); 47 | } 48 | public interface LongBinary 49 | { 50 | boolean op(long lhs, long rhs); 51 | } 52 | public interface FloatBinary 53 | { 54 | boolean op(float lhs, float rhs); 55 | } 56 | public interface DoubleBinary 57 | { 58 | boolean op(double lhs, double rhs); 59 | } 60 | public interface ObjectBinary 61 | { 62 | boolean op(Object lhs, Object rhs); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.IntStream; 11 | import clojure.lang.RT; 12 | import clojure.lang.ISeq; 13 | 14 | 15 | public interface BooleanReader extends IOBase, Iterable, IFn, 16 | List, RandomAccess, Sequential, 17 | Indexed 18 | { 19 | boolean read(long idx); 20 | default Object getDatatype () { return Keyword.intern(null, "boolean"); } 21 | default int size() { return RT.intCast(lsize()); } 22 | default Object get(int idx) { return read(idx); } 23 | default boolean isEmpty() { return lsize() == 0; } 24 | default Object[] toArray() { 25 | int nElems = size(); 26 | Object[] data = new Object[nElems]; 27 | 28 | for(int idx=0; idx < nElems; ++idx) { 29 | data[idx] = read(idx); 30 | } 31 | return data; 32 | } 33 | default Iterator iterator() { 34 | return new BooleanReaderIter(this); 35 | } 36 | default Object invoke(Object arg) { 37 | return read(RT.uncheckedLongCast(arg)); 38 | } 39 | default Object applyTo(ISeq items) { 40 | if (1 == items.count()) { 41 | return invoke(items.first()); 42 | } else { 43 | //Abstract method error 44 | return invoke(items.first(), items.next()); 45 | } 46 | } 47 | default IntStream typedStream() { 48 | return IntStream.range(0, size()).map(i -> read(i) ? 1 : 0 ); 49 | } 50 | default int count() { return size(); } 51 | default Object nth(int idx) { return read(idx); } 52 | default Object nth(int idx, Object notFound) { 53 | if (idx >= 0 && idx <= size()) { 54 | return read(idx); 55 | } else { 56 | return notFound; 57 | } 58 | } 59 | }; 60 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class BooleanReaderIter implements IOBase, BooleanIter 7 | { 8 | long idx; 9 | long num_elems; 10 | BooleanReader reader; 11 | public BooleanReaderIter(BooleanReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public boolean nextBoolean() { 21 | boolean retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public boolean current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/BooleanWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface BooleanWriter extends IOBase, IFn 9 | { 10 | void write(long idx, boolean value); 11 | default Object getDatatype () { return Keyword.intern(null, "boolean"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write( RT.longCast(idx), RT.booleanCast(value)); 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ByteIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.bytes.ByteIterator; 4 | 5 | 6 | public interface ByteIter extends Datatype, ByteIterator 7 | { 8 | byte current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ByteMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface ByteMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "int8"); } 8 | void insert(long idx, byte value); 9 | default void append(byte value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ByteReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import clojure.lang.RT; 11 | import clojure.lang.ISeq; 12 | import java.util.stream.IntStream; 13 | import java.util.stream.Stream; 14 | 15 | 16 | public interface ByteReader extends IOBase, Iterable, IFn, 17 | List, RandomAccess, Sequential, 18 | Indexed 19 | { 20 | byte read(long idx); 21 | default int size() { return RT.intCast(lsize()); } 22 | default Object get(int idx) { return read(idx); } 23 | default boolean isEmpty() { return lsize() == 0; } 24 | default Object getDatatype () { return Keyword.intern(null, "int8"); } 25 | default Object[] toArray() { 26 | int nElems = size(); 27 | Object[] data = new Object[nElems]; 28 | 29 | for(int idx=0; idx < nElems; ++idx) { 30 | data[idx] = read(idx); 31 | } 32 | return data; 33 | } 34 | default Iterator iterator() { 35 | return new ByteReaderIter(this); 36 | } 37 | default Object invoke(Object arg) { 38 | return read( RT.uncheckedLongCast( arg )); 39 | } 40 | default Object applyTo(ISeq items) { 41 | if (1 == items.count()) { 42 | return invoke(items.first()); 43 | } else { 44 | //Abstract method error 45 | return invoke(items.first(), items.next()); 46 | } 47 | } 48 | default IntStream typedStream() { 49 | return IntStream.range(0, size()).map(i -> read(i)); 50 | } 51 | default int count() { return size(); } 52 | default Object nth(int idx) { return read(idx); } 53 | default Object nth(int idx, Object notFound) { 54 | if (idx >= 0 && idx <= size()) { 55 | return read(idx); 56 | } else { 57 | return notFound; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ByteReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class ByteReaderIter implements IOBase, ByteIter 7 | { 8 | long idx; 9 | long num_elems; 10 | ByteReader reader; 11 | public ByteReaderIter(ByteReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public byte nextByte() { 21 | byte retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public byte current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ByteWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ByteWriter extends IOBase, IFn 9 | { 10 | void write(long idx, byte value); 11 | default Object getDatatype () { return Keyword.intern(null, "int8"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write( RT.longCast( idx ), RT.byteCast( value) ); 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/Comparator.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.bytes.ByteComparator; 4 | import it.unimi.dsi.fastutil.shorts.ShortComparator; 5 | import it.unimi.dsi.fastutil.ints.IntComparator; 6 | import it.unimi.dsi.fastutil.longs.LongComparator; 7 | import it.unimi.dsi.fastutil.floats.FloatComparator; 8 | import it.unimi.dsi.fastutil.doubles.DoubleComparator; 9 | 10 | 11 | //Helper class to make building typed comparators easier. 12 | public class Comparator 13 | { 14 | public interface ByteComp extends ByteComparator 15 | { 16 | default int compare(byte lhs, byte rhs) 17 | { 18 | return compareBytes(lhs, rhs); 19 | } 20 | int compareBytes(byte lhs, byte rhs); 21 | } 22 | public interface ShortComp extends ShortComparator 23 | { 24 | default int compare(short lhs, short rhs) 25 | { 26 | return compareShorts(lhs, rhs); 27 | } 28 | int compareShorts(short lhs, short rhs); 29 | } 30 | public interface IntComp extends IntComparator 31 | { 32 | default int compare(int lhs, int rhs) 33 | { 34 | return compareInts(lhs, rhs); 35 | } 36 | public int compareInts(int lhs, int rhs); 37 | } 38 | public interface LongComp extends LongComparator 39 | { 40 | default int compare(long lhs, long rhs) 41 | { 42 | return compareLongs(lhs, rhs); 43 | } 44 | public int compareLongs(long lhs, long rhs); 45 | } 46 | public interface FloatComp extends FloatComparator 47 | { 48 | default int compare(float lhs, float rhs) 49 | { 50 | return compareFloats(lhs, rhs); 51 | } 52 | public int compareFloats(float lhs, float rhs); 53 | } 54 | public interface DoubleComp extends DoubleComparator 55 | { 56 | default int compare(double lhs, double rhs) 57 | { 58 | return compareDoubles(lhs, rhs); 59 | } 60 | public int compareDoubles(double lhs, double rhs); 61 | } 62 | }; 63 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ConversionOp.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public class ConversionOp 6 | { 7 | public interface LongObj 8 | { 9 | public Object longToObj(long item); 10 | }; 11 | public interface ObjLong 12 | { 13 | public long longToObj(Object item); 14 | }; 15 | public interface IntObj 16 | { 17 | public Object intToObj(int item); 18 | }; 19 | public interface ObjInt 20 | { 21 | public int objToInt(Object item); 22 | }; 23 | }; 24 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/Countable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public interface Countable 5 | { 6 | //long size 7 | long lsize(); 8 | }; 9 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/Datatype.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface Datatype 6 | { 7 | Object getDatatype(); 8 | } 9 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DirectMappedOps.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import com.sun.jna.Pointer; 4 | 5 | public class DirectMappedOps 6 | { 7 | public static native Pointer memset(Pointer data, int val, int numBytes); 8 | public static native Pointer memcpy(Pointer dst, Pointer src, int numBytes); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DoubleIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.doubles.DoubleIterator; 4 | 5 | 6 | public interface DoubleIter extends Datatype, DoubleIterator 7 | { 8 | double current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DoubleMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public interface DoubleMutable extends MutableRemove 7 | { 8 | default Object getDatatype () { return Keyword.intern(null, "float64"); } 9 | void insert(long idx, double value); 10 | default void append(double value) 11 | { 12 | insert(lsize(), value); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DoubleReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.DoubleStream; 11 | import java.util.stream.StreamSupport; 12 | import clojure.lang.RT; 13 | import clojure.lang.ISeq; 14 | 15 | 16 | public interface DoubleReader extends IOBase, Iterable, IFn, 17 | List, RandomAccess, Sequential, 18 | Indexed 19 | { 20 | double read(long idx); 21 | default Object getDatatype () { return Keyword.intern(null, "float64"); } 22 | default int size() { return RT.intCast(lsize()); } 23 | default Object get(int idx) { return read(idx); } 24 | default boolean isEmpty() { return lsize() == 0; } 25 | default Object[] toArray() { 26 | int nElems = size(); 27 | Object[] data = new Object[nElems]; 28 | 29 | for(int idx=0; idx < nElems; ++idx) { 30 | data[idx] = read(idx); 31 | } 32 | return data; 33 | } 34 | default Iterator iterator() { 35 | return new DoubleReaderIter(this); 36 | } 37 | default Object invoke(Object arg) { 38 | return read(RT.uncheckedLongCast(arg)); 39 | } 40 | default Object applyTo(ISeq items) { 41 | if (1 == items.count()) { 42 | return invoke(items.first()); 43 | } else { 44 | //Abstract method error 45 | return invoke(items.first(), items.next()); 46 | } 47 | } 48 | default DoubleStream typedStream() { 49 | return StreamSupport.doubleStream(new RangeDoubleSpliterator(0, size(), 50 | new RangeDoubleSpliterator.LongDoubleConverter() { public double longToDouble(long arg) { return read(arg); } }, 51 | false), 52 | false); 53 | } 54 | default int count() { return size(); } 55 | default Object nth(int idx) { return read(idx); } 56 | default Object nth(int idx, Object notFound) { 57 | if (idx >= 0 && idx <= size()) { 58 | return read(idx); 59 | } else { 60 | return notFound; 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DoubleReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class DoubleReaderIter implements IOBase, DoubleIter 7 | { 8 | long idx; 9 | long num_elems; 10 | DoubleReader reader; 11 | public DoubleReaderIter(DoubleReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public double nextDouble() { 21 | double retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public double current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/DoubleWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface DoubleWriter extends IOBase, IFn 9 | { 10 | void write(long idx, double value); 11 | default Object getDatatype () { return Keyword.intern(null, "float64"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write( RT.longCast(idx), RT.doubleCast(value)); 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/FloatIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.floats.FloatIterator; 4 | 5 | 6 | public interface FloatIter extends Datatype, FloatIterator 7 | { 8 | float current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/FloatMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface FloatMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "float32"); } 8 | void insert(long idx, float value); 9 | default void append(float value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/FloatReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.DoubleStream; 11 | import java.util.stream.StreamSupport; 12 | import clojure.lang.RT; 13 | import clojure.lang.ISeq; 14 | 15 | 16 | public interface FloatReader extends IOBase, Iterable, IFn, 17 | List, RandomAccess, Sequential, 18 | Indexed 19 | { 20 | float read(long idx); 21 | default Object getDatatype () { return Keyword.intern(null, "float32"); } 22 | default int size() { return RT.intCast(lsize()); } 23 | default Object get(int idx) { return read(idx); } 24 | default boolean isEmpty() { return lsize() == 0; } 25 | default Object[] toArray() { 26 | int nElems = size(); 27 | Object[] data = new Object[nElems]; 28 | 29 | for(int idx=0; idx < nElems; ++idx) { 30 | data[idx] = read(idx); 31 | } 32 | return data; 33 | } 34 | default Iterator iterator() { 35 | return new FloatReaderIter(this); 36 | } 37 | default Object invoke(Object arg) { 38 | return read(RT.uncheckedLongCast(arg)); 39 | } 40 | default Object applyTo(ISeq items) { 41 | if (1 == items.count()) { 42 | return invoke(items.first()); 43 | } else { 44 | //Abstract method error 45 | return invoke(items.first(), items.next()); 46 | } 47 | } 48 | default DoubleStream typedStream() { 49 | return StreamSupport.doubleStream(new RangeDoubleSpliterator(0, size(), 50 | new RangeDoubleSpliterator.LongDoubleConverter() { public double longToDouble(long arg) { return read(arg); } }, 51 | false), 52 | false); 53 | } 54 | default int count() { return size(); } 55 | default Object nth(int idx) { return read(idx); } 56 | default Object nth(int idx, Object notFound) { 57 | if (idx >= 0 && idx <= size()) { 58 | return read(idx); 59 | } else { 60 | return notFound; 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/FloatReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class FloatReaderIter implements IOBase, FloatIter 7 | { 8 | long idx; 9 | long num_elems; 10 | FloatReader reader; 11 | public FloatReaderIter(FloatReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public float nextFloat() { 21 | float retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public float current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/FloatWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface FloatWriter extends IOBase, IFn 9 | { 10 | void write(long idx, float value); 11 | default Object getDatatype () { return Keyword.intern(null, "float32"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write(RT.longCast(idx), RT.floatCast(value)); 15 | return null; 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/Functions.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | /** 3 | * In order to use java.util.Map classes well you often need to use Functions 4 | * and BiFunctions. The problem is that you cannot pass more context into these objects 5 | * so a simple extension to these objects is to provide more context. 6 | * In all cases, context is the first argument to supplied function. 7 | */ 8 | 9 | import clojure.lang.IFn; 10 | import java.util.function.Function; 11 | import java.util.function.BiFunction; 12 | 13 | 14 | public class Functions 15 | { 16 | public class CtxFunction implements Function 17 | { 18 | public final IFn fn; 19 | public Object context; 20 | public CtxFunction(IFn _fn) 21 | { 22 | this.fn = _fn; 23 | this.context = null; 24 | } 25 | public void setContext(Object ctx) 26 | { 27 | this.context = ctx; 28 | } 29 | public Object apply( Object arg) 30 | { 31 | return fn.invoke(context, arg); 32 | } 33 | } 34 | 35 | public class CtxBiFunction implements BiFunction 36 | { 37 | public final IFn fn; 38 | public Object context; 39 | public CtxBiFunction(IFn _fn) 40 | { 41 | this.fn = _fn; 42 | this.context = null; 43 | } 44 | public void setContext(Object ctx) 45 | { 46 | this.context = ctx; 47 | } 48 | public Object apply(Object arg1, Object arg2) 49 | { 50 | return fn.invoke(context, arg1, arg2); 51 | } 52 | } 53 | 54 | public interface LongBiFunction 55 | { 56 | public Object apply(long arg1, Object arg2); 57 | } 58 | 59 | public static class LongCtxFunction implements Function 60 | { 61 | public final LongBiFunction fn; 62 | public long context; 63 | public LongCtxFunction(LongBiFunction _fn) 64 | { 65 | this.fn = _fn; 66 | } 67 | public void setContext(long ctx) 68 | { 69 | this.context = ctx; 70 | } 71 | public Object apply(Object arg1) 72 | { 73 | return fn.apply(context, arg1); 74 | } 75 | } 76 | 77 | public interface LongTriFunction 78 | { 79 | public Object apply(long arg1, Object arg2, Object arg3); 80 | } 81 | 82 | public static class LongCtxBiFunction implements BiFunction 83 | { 84 | public final LongTriFunction fn; 85 | public long context; 86 | public LongCtxBiFunction(LongTriFunction _fn) 87 | { 88 | this.fn = _fn; 89 | } 90 | public void setContext(long ctx) 91 | { 92 | this.context = ctx; 93 | } 94 | public Object apply(Object arg1, Object arg2) 95 | { 96 | return fn.apply(context, arg1, arg2); 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IOBase.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public interface IOBase extends Datatype, Countable 5 | { 6 | } 7 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IndexingSystem.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public class IndexingSystem 5 | { 6 | public interface Backward 7 | { 8 | //Returns java Long or an Iterable 9 | Object localToGlobal(long local_idx); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IntIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.ints.IntIterator; 4 | 5 | 6 | public interface IntIter extends Datatype, IntIterator 7 | { 8 | int current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IntMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface IntMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "int32"); } 8 | void insert(long idx, int value); 9 | default void append(int value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IntReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.IntStream; 11 | import clojure.lang.RT; 12 | import clojure.lang.ISeq; 13 | 14 | 15 | public interface IntReader extends IOBase, Iterable, IFn, 16 | List, RandomAccess, Sequential, 17 | Indexed 18 | { 19 | int read(long idx); 20 | default Object getDatatype () { return Keyword.intern(null, "int32"); } 21 | default int size() { return RT.intCast(lsize()); } 22 | default Object get(int idx) { return read(idx); } 23 | default boolean isEmpty() { return lsize() == 0; } 24 | default Object[] toArray() { 25 | int nElems = size(); 26 | Object[] data = new Object[nElems]; 27 | 28 | for(int idx=0; idx < nElems; ++idx) { 29 | data[idx] = read(idx); 30 | } 31 | return data; 32 | } 33 | default Iterator iterator() { 34 | return new IntReaderIter(this); 35 | } 36 | default Object invoke(Object arg) { 37 | return read(RT.uncheckedLongCast(arg)); 38 | } 39 | default Object applyTo(ISeq items) { 40 | if (1 == items.count()) { 41 | return invoke(items.first()); 42 | } else { 43 | //Abstract method error 44 | return invoke(items.first(), items.next()); 45 | } 46 | } 47 | default IntStream typedStream() { 48 | return IntStream.range(0, size()).map(i -> read(i)); 49 | } 50 | default int count() { return size(); } 51 | default Object nth(int idx) { return read(idx); } 52 | default Object nth(int idx, Object notFound) { 53 | if (idx >= 0 && idx <= size()) { 54 | return read(idx); 55 | } else { 56 | return notFound; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IntReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class IntReaderIter implements IOBase, IntIter 7 | { 8 | long idx; 9 | long num_elems; 10 | IntReader reader; 11 | public IntReaderIter(IntReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public int nextInt() { 21 | int retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public int current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IntWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface IntWriter extends IOBase, IFn 9 | { 10 | void write(long idx, int value); 11 | default Object getDatatype () { return Keyword.intern(null, "int32"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write(RT.longCast(idx), RT.intCast(value)); 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/IteratorObjectIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | import java.util.Iterator; 5 | import clojure.lang.Keyword; 6 | 7 | 8 | public class IteratorObjectIter implements ObjectIter 9 | { 10 | Iterator iter; 11 | boolean has_next; 12 | Object current_object; 13 | Keyword dtype; 14 | 15 | public IteratorObjectIter( Iterator _iter, Keyword _dtype ) 16 | { 17 | iter = _iter; 18 | dtype = _dtype; 19 | has_next = iter.hasNext(); 20 | if( has_next ) 21 | current_object = iter.next(); 22 | } 23 | public Object getDatatype() { return dtype; } 24 | public boolean hasNext() { return has_next; } 25 | public Object next() { 26 | //Force exception from base iterator. 27 | if (!has_next) 28 | iter.next(); 29 | 30 | Object retval = current_object; 31 | has_next = iter.hasNext(); 32 | if (has_next) { 33 | current_object = iter.next(); 34 | } 35 | else { 36 | current_object = null; 37 | } 38 | return retval; 39 | } 40 | public Object current() { return current_object; } 41 | } 42 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ListPersistentVector.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | import clojure.lang.APersistentVector; 5 | import clojure.lang.IPersistentVector; 6 | import clojure.lang.IPersistentCollection; 7 | import clojure.lang.IPersistentStack; 8 | import clojure.lang.Indexed; 9 | import java.util.List; 10 | import java.util.ArrayList; 11 | import java.util.Collections; 12 | 13 | 14 | public class ListPersistentVector extends APersistentVector 15 | { 16 | public final List data; 17 | public ListPersistentVector (List _data) { 18 | data = _data; 19 | } 20 | public IPersistentVector cons(Object val) { 21 | ArrayList newData = new ArrayList(data); 22 | newData.add(val); 23 | return new ListPersistentVector(Collections.unmodifiableList(newData)); 24 | } 25 | public IPersistentVector assocN(int idx, Object val) { 26 | ArrayList newData = new ArrayList(data); 27 | newData.set(idx,val); 28 | return new ListPersistentVector(Collections.unmodifiableList(newData)); 29 | } 30 | public IPersistentCollection empty() { 31 | return new ListPersistentVector(Collections.unmodifiableList(new ArrayList())); 32 | } 33 | public IPersistentStack pop() { 34 | if (data.size() != 0) 35 | return new ListPersistentVector(data.subList(0, data.size()-1)); 36 | return this; 37 | } 38 | public int count() { 39 | return data.size(); 40 | } 41 | public Object nth(int idx) { 42 | return data.get(idx); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongBitmapIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | import clojure.lang.Keyword; 5 | import org.roaringbitmap.IntIterator; 6 | 7 | 8 | public class LongBitmapIter implements LongIter 9 | { 10 | IntIterator iter; 11 | long currentValue; 12 | public LongBitmapIter(IntIterator _iter) 13 | { 14 | iter = _iter; 15 | } 16 | public Object getDatatype() { return Keyword.intern(null, "uint32"); } 17 | public boolean hasNext() { return iter.hasNext(); } 18 | public long nextLong() { 19 | currentValue = Integer.toUnsignedLong( iter.next() ); 20 | return currentValue; 21 | } 22 | public long current() { 23 | return currentValue; 24 | } 25 | }; 26 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.longs.LongIterator; 4 | 5 | 6 | public interface LongIter extends Datatype, LongIterator 7 | { 8 | long current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface LongMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "int64"); } 8 | void insert(long idx, long value); 9 | default void append(long value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.LongStream; 11 | import clojure.lang.RT; 12 | import clojure.lang.ISeq; 13 | 14 | 15 | public interface LongReader extends IOBase, Iterable, IFn, 16 | List, RandomAccess, Sequential, 17 | Indexed 18 | { 19 | long read(long idx); 20 | default Object getDatatype () { return Keyword.intern(null, "int64"); } 21 | default int size() { return RT.intCast(lsize()); } 22 | default Object get(int idx) { return read(idx); } 23 | default boolean isEmpty() { return lsize() == 0; } 24 | default Object[] toArray() { 25 | int nElems = size(); 26 | Object[] data = new Object[nElems]; 27 | 28 | for(int idx=0; idx < nElems; ++idx) { 29 | data[idx] = read(idx); 30 | } 31 | return data; 32 | } 33 | default Iterator iterator() { 34 | return new LongReaderIter(this); 35 | } 36 | default Object invoke(Object arg) { 37 | return read(RT.uncheckedLongCast(arg)); 38 | } 39 | default Object applyTo(ISeq items) { 40 | if (1 == items.count()) { 41 | return invoke(items.first()); 42 | } else { 43 | //Abstract method error 44 | return invoke(items.first(), items.next()); 45 | } 46 | } 47 | default LongStream typedStream() { 48 | return LongStream.range(0, size()).map(i -> read(i)); 49 | } 50 | default int count() { return size(); } 51 | default Object nth(int idx) { return read(idx); } 52 | default Object nth(int idx, Object notFound) { 53 | if (idx >= 0 && idx <= size()) { 54 | return read(idx); 55 | } else { 56 | return notFound; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class LongReaderIter implements IOBase, LongIter 7 | { 8 | long idx; 9 | long num_elems; 10 | LongReader reader; 11 | public LongReaderIter(LongReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public long nextLong() { 21 | long retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public long current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/LongWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface LongWriter extends IOBase, IFn 9 | { 10 | void write(long idx, long value); 11 | default Object getDatatype () { return Keyword.intern(null, "int64"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write(RT.longCast(idx), RT.longCast(value)); 15 | return null; 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/MutableRemove.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | 4 | public interface MutableRemove extends IOBase 5 | { 6 | void mremove(long idx); 7 | } 8 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ObjectIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import java.util.Iterator; 4 | 5 | 6 | public interface ObjectIter extends Datatype, Iterator 7 | { 8 | Object current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ObjectMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface ObjectMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "object"); } 8 | void insert(long idx, Object value); 9 | default void append(Object value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ObjectReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.RT; 7 | import clojure.lang.ISeq; 8 | import clojure.lang.Indexed; 9 | import java.util.Iterator; 10 | import java.util.List; 11 | import java.util.RandomAccess; 12 | import java.util.stream.Stream; 13 | 14 | 15 | public interface ObjectReader extends IOBase, Iterable, IFn, 16 | List, RandomAccess, Sequential, 17 | Indexed 18 | { 19 | Object read(long idx); 20 | default Object getDatatype () { return Keyword.intern(null, "object"); } 21 | default int size() { return RT.intCast(lsize()); } 22 | default Object get(int idx) { return read(idx); } 23 | default boolean isEmpty() { return lsize() == 0; } 24 | default Object[] toArray() { 25 | int nElems = size(); 26 | Object[] data = new Object[nElems]; 27 | 28 | for(int idx=0; idx < nElems; ++idx) { 29 | data[idx] = read(idx); 30 | } 31 | return data; 32 | } 33 | default Iterator iterator() { 34 | return new ObjectReaderIter(this); 35 | } 36 | default Object invoke(Object arg) { 37 | return read(RT.uncheckedLongCast(arg)); 38 | } 39 | default Object applyTo(ISeq items) { 40 | if (1 == items.count()) { 41 | return invoke(items.first()); 42 | } else { 43 | //Abstract method error 44 | return invoke(items.first(), items.next()); 45 | } 46 | } 47 | default Stream typedStream() { 48 | return stream(); 49 | } 50 | default int count() { return size(); } 51 | default Object nth(int idx) { return read(idx); } 52 | default Object nth(int idx, Object notFound) { 53 | if (idx >= 0 && idx <= size()) { 54 | return read(idx); 55 | } else { 56 | return notFound; 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ObjectReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class ObjectReaderIter implements IOBase, ObjectIter 7 | { 8 | long idx; 9 | long num_elems; 10 | ObjectReader reader; 11 | public ObjectReaderIter(ObjectReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public Object next() { 21 | Object retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public Object current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ObjectWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ObjectWriter extends IOBase, IFn 9 | { 10 | void write(long idx, Object value); 11 | default Object getDatatype () { return Keyword.intern(null, "object"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write( RT.longCast(idx), value); 15 | return null; 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/RangeDoubleSpliterator.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import java.util.Comparator; 4 | import java.util.Objects; 5 | import java.util.Spliterator; 6 | import java.util.function.Consumer; 7 | import java.util.function.DoubleConsumer; 8 | import java.util.function.IntConsumer; 9 | import java.util.function.LongConsumer; 10 | 11 | 12 | public class RangeDoubleSpliterator implements Spliterator.OfDouble { 13 | // Can never be greater that upTo, this avoids overflow if upper bound 14 | // is Long.MAX_VALUE 15 | // All elements are traversed if from == upTo & last == 0 16 | private long from; 17 | public final long upTo; 18 | // 1 if the range is closed and the last element has not been traversed 19 | // Otherwise, 0 if the range is open, or is a closed range and all 20 | // elements have been traversed 21 | private int last; 22 | public interface LongDoubleConverter 23 | { 24 | double longToDouble(long arg); 25 | }; 26 | 27 | public final LongDoubleConverter converter; 28 | 29 | public RangeDoubleSpliterator(long from, long upTo, LongDoubleConverter converter, 30 | boolean closed) { 31 | this(from, upTo, closed ? 1 : 0, converter); 32 | } 33 | 34 | private RangeDoubleSpliterator(long from, long upTo, int last, 35 | LongDoubleConverter converter) { 36 | Objects.requireNonNull(converter); 37 | assert upTo - from + last > 0; 38 | this.from = from; 39 | this.upTo = upTo; 40 | this.last = last; 41 | this.converter = converter; 42 | } 43 | 44 | @Override 45 | public boolean tryAdvance(DoubleConsumer consumer) { 46 | Objects.requireNonNull(consumer); 47 | 48 | final long i = from; 49 | if (i < upTo) { 50 | from++; 51 | consumer.accept(converter.longToDouble(i)); 52 | return true; 53 | } 54 | else if (last > 0) { 55 | last = 0; 56 | consumer.accept(converter.longToDouble(i)); 57 | return true; 58 | } 59 | return false; 60 | } 61 | 62 | @Override 63 | public void forEachRemaining(DoubleConsumer consumer) { 64 | Objects.requireNonNull(consumer); 65 | 66 | long i = from; 67 | final long hUpTo = upTo; 68 | int hLast = last; 69 | from = upTo; 70 | last = 0; 71 | while (i < hUpTo) { 72 | consumer.accept(converter.longToDouble(i++)); 73 | } 74 | if (hLast > 0) { 75 | // Last element of closed range 76 | consumer.accept(converter.longToDouble(i)); 77 | } 78 | } 79 | 80 | @Override 81 | public long estimateSize() { 82 | return upTo - from + last; 83 | } 84 | 85 | @Override 86 | public int characteristics() { 87 | return Spliterator.ORDERED | Spliterator.SIZED | Spliterator.SUBSIZED | 88 | Spliterator.IMMUTABLE | Spliterator.NONNULL; 89 | } 90 | 91 | @Override 92 | public Comparator getComparator() { 93 | return null; 94 | } 95 | 96 | @Override 97 | public Spliterator.OfDouble trySplit() { 98 | long size = estimateSize(); 99 | return size <= 1 100 | ? null 101 | // Left split always has a half-open range 102 | : new RangeDoubleSpliterator(from, from = from + splitPoint(size), 0, converter); 103 | } 104 | 105 | /** 106 | * The spliterator size below which the spliterator will be split 107 | * at the mid-point to produce balanced splits. Above this size the 108 | * spliterator will be split at a ratio of 109 | * 1:(RIGHT_BALANCED_SPLIT_RATIO - 1) 110 | * to produce right-balanced splits. 111 | * 112 | *

Such splitting ensures that for very large ranges that the left 113 | * side of the range will more likely be processed at a lower-depth 114 | * than a balanced tree at the expense of a higher-depth for the right 115 | * side of the range. 116 | * 117 | *

This is optimized for cases such as LongStream.longs() that is 118 | * implemented as range of 0 to Long.MAX_VALUE but is likely to be 119 | * augmented with a limit operation that limits the number of elements 120 | * to a count lower than this threshold. 121 | */ 122 | private static final long BALANCED_SPLIT_THRESHOLD = 1 << 24; 123 | 124 | /** 125 | * The split ratio of the left and right split when the spliterator 126 | * size is above BALANCED_SPLIT_THRESHOLD. 127 | */ 128 | private static final long RIGHT_BALANCED_SPLIT_RATIO = 1 << 3; 129 | 130 | private long splitPoint(long size) { 131 | long d = (size < BALANCED_SPLIT_THRESHOLD) ? 2 : RIGHT_BALANCED_SPLIT_RATIO; 132 | // 2 <= size <= Long.MAX_VALUE 133 | return size / d; 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ShortIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.shorts.ShortIterator; 4 | 5 | 6 | public interface ShortIter extends Datatype, ShortIterator 7 | { 8 | short current(); 9 | } 10 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ShortMutable.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | public interface ShortMutable extends MutableRemove 6 | { 7 | default Object getDatatype () { return Keyword.intern(null, "int16"); } 8 | void insert(long idx, short value); 9 | default void append(short value) 10 | { 11 | insert(lsize(), value); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ShortReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.Sequential; 6 | import clojure.lang.Indexed; 7 | import java.util.Iterator; 8 | import java.util.List; 9 | import java.util.RandomAccess; 10 | import java.util.stream.IntStream; 11 | import clojure.lang.RT; 12 | import clojure.lang.ISeq; 13 | 14 | 15 | public interface ShortReader extends IOBase, Iterable, IFn, 16 | List, RandomAccess, Sequential, 17 | Indexed 18 | 19 | { 20 | short read(long idx); 21 | default Object getDatatype () { return Keyword.intern(null, "int16"); } 22 | default int size() { return RT.intCast(lsize()); } 23 | default Object get(int idx) { return read(idx); } 24 | default boolean isEmpty() { return lsize() == 0; } 25 | default Object[] toArray() { 26 | int nElems = size(); 27 | Object[] data = new Object[nElems]; 28 | 29 | for(int idx=0; idx < nElems; ++idx) { 30 | data[idx] = read(idx); 31 | } 32 | return data; 33 | } 34 | default Iterator iterator() { 35 | return new ShortReaderIter(this); 36 | } 37 | default Object invoke(Object arg) { 38 | return read(RT.uncheckedLongCast(arg)); 39 | } 40 | default Object applyTo(ISeq items) { 41 | if (1 == items.count()) { 42 | return invoke(items.first()); 43 | } else { 44 | //Abstract method error 45 | return invoke(items.first(), items.next()); 46 | } 47 | } 48 | default IntStream typedStream() { 49 | return IntStream.range(0, size()).map(i -> read(i)); 50 | } 51 | default int count() { return size(); } 52 | default Object nth(int idx) { return read(idx); } 53 | default Object nth(int idx, Object notFound) { 54 | if (idx >= 0 && idx <= size()) { 55 | return read(idx); 56 | } else { 57 | return notFound; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ShortReaderIter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.Keyword; 4 | 5 | 6 | public class ShortReaderIter implements IOBase, ShortIter 7 | { 8 | long idx; 9 | long num_elems; 10 | ShortReader reader; 11 | public ShortReaderIter(ShortReader _reader) 12 | { 13 | idx = 0; 14 | num_elems = _reader.lsize(); 15 | reader = _reader; 16 | } 17 | public Object getDatatype() { return reader.getDatatype(); } 18 | public long lsize() { return num_elems - idx; } 19 | public boolean hasNext() { return idx < num_elems; } 20 | public short nextShort() { 21 | short retval = reader.read(idx); 22 | ++idx; 23 | return retval; 24 | } 25 | public short current() { 26 | return reader.read(idx); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/ShortWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | import clojure.lang.Keyword; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ShortWriter extends IOBase, IFn 9 | { 10 | void write(long idx, short value); 11 | default Object getDatatype () { return Keyword.intern(null, "int16"); } 12 | default Object invoke(Object idx, Object value) 13 | { 14 | write(RT.longCast(idx), RT.shortCast(value)); 15 | return null; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/SimpleLongSet.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import it.unimi.dsi.fastutil.longs.LongSet; 4 | import clojure.lang.RT; 5 | import clojure.lang.Keyword; 6 | 7 | 8 | public interface SimpleLongSet extends IOBase, LongSet 9 | { 10 | default Object getDatatype () { return Keyword.intern(null, "int64"); } 11 | default int size() { return RT.intCast(lsize()); } 12 | default boolean contains(Long obj) { return lcontains(RT.longCast(obj)); } 13 | default boolean contains(long obj) { return lcontains(obj); } 14 | default boolean add(long obj) { return ladd(obj); } 15 | default boolean remove(long obj) { return lremove(obj); } 16 | boolean ladd(long obj); 17 | boolean lcontains(long obj); 18 | boolean lremove(long obj); 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/datatype/UnaryOperators.java: -------------------------------------------------------------------------------- 1 | package tech.v2.datatype; 2 | 3 | import clojure.lang.IFn; 4 | 5 | public class UnaryOperators 6 | { 7 | public interface ByteUnary extends Datatype, IFn 8 | { 9 | byte op(byte arg); 10 | default Object invoke(Object arg) 11 | { 12 | return op((byte) arg); 13 | } 14 | }; 15 | public interface ShortUnary extends Datatype, IFn 16 | { 17 | short op(short arg); 18 | default Object invoke(Object arg) 19 | { 20 | return op((short) arg); 21 | } 22 | }; 23 | public interface IntUnary extends Datatype, IFn 24 | { 25 | int op(int arg); 26 | default Object invoke(Object arg) 27 | { 28 | return op((int) arg); 29 | } 30 | }; 31 | public interface LongUnary extends Datatype, IFn 32 | { 33 | long op(long arg); 34 | default Object invoke(Object arg) 35 | { 36 | return op((long) arg); 37 | } 38 | }; 39 | public interface FloatUnary extends Datatype, IFn 40 | { 41 | float op(float arg); 42 | default Object invoke(Object arg) 43 | { 44 | return op((float) arg); 45 | } 46 | }; 47 | public interface DoubleUnary extends Datatype, IFn 48 | { 49 | double op(double arg); 50 | default Object invoke(Object arg) 51 | { 52 | return op((double) arg); 53 | } 54 | }; 55 | public interface BooleanUnary extends Datatype, IFn 56 | { 57 | boolean op(boolean arg); 58 | default Object invoke(Object arg) 59 | { 60 | return op((boolean) arg); 61 | } 62 | }; 63 | public interface ObjectUnary extends Datatype, IFn 64 | { 65 | Object op(Object arg); 66 | default Object invoke(Object arg) 67 | { 68 | return op(arg); 69 | } 70 | }; 71 | } 72 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/BooleanTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.BooleanReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface BooleanTensorReader extends BooleanReader 9 | { 10 | boolean read2d(long row, long col); 11 | boolean read3d(long height, long width, long chan); 12 | boolean tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/BooleanTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.BooleanWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface BooleanTensorWriter extends BooleanWriter 9 | { 10 | boolean write2d(long row, long col, boolean value); 11 | boolean write3d(long height, long width, long chan, boolean value); 12 | boolean tensorWrite(Iterable dims, boolean value); 13 | default Object invoke(Object row, Object col, Object value) { 14 | return write2d(RT.longCast(row), RT.longCast(col), RT.booleanCast(value)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan, Object value) { 17 | return write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.booleanCast(value)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ByteTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.ByteReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ByteTensorReader extends ByteReader 9 | { 10 | byte read2d(long row, long col); 11 | byte read3d(long height, long width, long chan); 12 | byte tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ByteTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.ByteWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ByteTensorWriter extends ByteWriter 9 | { 10 | void write2d(long row, long col, byte val); 11 | void write3d(long height, long width, long chan, byte val); 12 | void tensorWrite(Iterable dims, byte val); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), RT.byteCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.byteCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/DoubleTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.DoubleReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface DoubleTensorReader extends DoubleReader 9 | { 10 | double read2d(long row, long col); 11 | double read3d(long height, long width, long chan); 12 | double tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/DoubleTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.DoubleWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface DoubleTensorWriter extends DoubleWriter 9 | { 10 | void write2d(long row, long col, double val); 11 | void write3d(long height, long width, long chan, double val); 12 | void tensorWrite(Iterable dims, double val); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), RT.doubleCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.doubleCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/FloatTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.FloatReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface FloatTensorReader extends FloatReader 9 | { 10 | float read2d(long row, long col); 11 | float read3d(long height, long width, long chan); 12 | float tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/FloatTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.FloatWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface FloatTensorWriter extends FloatWriter 9 | { 10 | void write2d(long row, long col, float val); 11 | void write3d(long height, long width, long chan, float val); 12 | void tensorWrite(Iterable dims, float val); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), RT.floatCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.floatCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/IntTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.IntReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface IntTensorReader extends IntReader 9 | { 10 | int read2d(long row, long col); 11 | int read3d(long height, long width, long chan); 12 | int tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/IntTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.IntWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface IntTensorWriter extends IntWriter 9 | { 10 | void write2d(long row, long col, int val); 11 | void write3d(long height, long width, long chan, int val); 12 | void tensorWrite(Iterable dims, int val); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.intCast(row), RT.intCast(col), RT.intCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.intCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/LongTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.LongReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface LongTensorReader extends LongReader 9 | { 10 | long read2d(long row, long col); 11 | long read3d(long height, long width, long chan); 12 | long tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/LongTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.LongWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface LongTensorWriter extends LongWriter 9 | { 10 | long write2d(long row, long col, long val); 11 | long write3d(long height, long width, long chan, long val); 12 | long tensorWrite(Iterable dims, long val); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), RT.longCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.longCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ObjectTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.ObjectReader; 4 | import clojure.lang.RT; 5 | 6 | 7 | public interface ObjectTensorReader extends ObjectReader 8 | { 9 | Object read2d(long row, long col); 10 | Object read3d(long height, long width, long chan); 11 | Object tensorRead(Iterable dims); 12 | default Object invoke(Object row, Object col) { 13 | return read2d(RT.longCast(row), RT.longCast(col)); 14 | } 15 | default Object invoke(Object row, Object col, Object chan) { 16 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ObjectTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.ObjectWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ObjectTensorWriter extends ObjectWriter 9 | { 10 | void write2d(long row, long col, Object value); 11 | void write3d(long height, long width, long chan, Object value); 12 | void tensorWrite(Iterable dims, Object value); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), val); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), val); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ShortTensorReader.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.ShortReader; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ShortTensorReader extends ShortReader 9 | { 10 | short read2d(long row, long col); 11 | short read3d(long height, long width, long chan); 12 | short tensorRead(Iterable dims); 13 | default Object invoke(Object row, Object col) { 14 | return read2d(RT.longCast(row), RT.longCast(col)); 15 | } 16 | default Object invoke(Object row, Object col, Object chan) { 17 | return read3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /java/tech/v2/tensor/ShortTensorWriter.java: -------------------------------------------------------------------------------- 1 | package tech.v2.tensor; 2 | 3 | import tech.v2.datatype.IntIter; 4 | import tech.v2.datatype.ShortWriter; 5 | import clojure.lang.RT; 6 | 7 | 8 | public interface ShortTensorWriter extends ShortWriter 9 | { 10 | void write2d(long row, long col, short value); 11 | void write3d(long height, long width, long chan, short value); 12 | void tensorWrite(Iterable dims, short value); 13 | default Object invoke(Object row, Object col, Object val) { 14 | write2d(RT.longCast(row), RT.longCast(col), RT.shortCast(val)); 15 | return null; 16 | } 17 | default Object invoke(Object row, Object col, Object chan, Object val) { 18 | write3d(RT.longCast(row), RT.longCast(col), RT.longCast(chan), RT.shortCast(val)); 19 | return null; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /neanderthal/tech/libs/neanderthal_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.libs.neanderthal-test 2 | (:require [uncomplicate.neanderthal.core :as n-core] 3 | [uncomplicate.neanderthal.native :as n-native] 4 | [tech.v2.tensor :as dtt] 5 | [tech.v2.datatype.functional :as dfn] 6 | [clojure.test :refer [deftest is]] 7 | [tech.libs.neanderthal] 8 | [tech.v2.datatype])) 9 | 10 | 11 | (deftest basic-neanderthal-test 12 | (let [a (n-native/dge 3 3 (range 9))] 13 | (is (dfn/equals (dtt/ensure-tensor a) 14 | (-> (dtt/->tensor (partition 3 (range 9))) 15 | (dtt/transpose [1 0])))) 16 | (let [second-row (second (n-core/rows a))] 17 | (is (dfn/equals (dtt/ensure-tensor second-row) 18 | [1 4 7]))))) 19 | 20 | 21 | (deftest basic-neanderthal-test-row-major 22 | (let [b (n-native/dge 3 3 (range 9) {:layout :row})] 23 | (is (dfn/equals (dtt/ensure-tensor b) 24 | (dtt/->tensor (partition 3 (range 9))))) 25 | (let [second-row (second (n-core/rows b))] 26 | (is (dfn/equals (dtt/ensure-tensor second-row) 27 | [3 4 5]))))) 28 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject techascent/tech.datatype "5.19-SNAPSHOT" 2 | :description "Library for efficient manipulation of contiguous mutable containers of primitive datatypes." 3 | :url "http://github.com/techascent/tech.datatype" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.10.1"] 7 | [techascent/tech.jna "3.24"] 8 | [techascent/tech.parallel "2.11"] 9 | [insn "0.4.0" 10 | :exclusions [org.ow2.asm/asm]] 11 | [org.ow2.asm/asm "7.1"] 12 | [it.unimi.dsi/fastutil "8.2.1"] 13 | [kixi/stats "0.5.4" 14 | :exclusions [org.clojure/test.check]] 15 | [org.clojure/math.combinatorics "0.1.6"] 16 | [org.clojure/math.numeric-tower "0.0.4"] 17 | [org.clojure/test.check "1.1.0"] 18 | [org.apache.commons/commons-math3 "3.6.1"] 19 | [primitive-math "0.1.6"] 20 | [camel-snake-kebab "0.4.0"] 21 | [org.roaringbitmap/RoaringBitmap "0.8.13"] 22 | [org.xerial.larray/larray-mmap "0.4.1"]] 23 | :profiles {:dev {:dependencies [[org.bytedeco.javacpp-presets/opencv-platform 24 | "4.0.1-1.4.4"] 25 | [criterium "0.4.5"] 26 | [ch.qos.logback/logback-classic "1.1.3"] 27 | [com.clojure-goes-fast/clj-memory-meter "0.1.0"] 28 | [uncomplicate/neanderthal "0.35.0"]] 29 | ;;Separating out these paths for travis 30 | :test-paths ["neanderthal" "test"]} 31 | :travis {:dependencies [[org.bytedeco.javacpp-presets/opencv-platform 32 | "4.0.1-1.4.4"] 33 | [criterium "0.4.5"] 34 | [ch.qos.logback/logback-classic "1.1.3"] 35 | [com.clojure-goes-fast/clj-memory-meter "0.1.0"]]} 36 | :uberjar {:aot [tech.v2.datatype] 37 | :uberjar-name "datatype.jar"}} 38 | :java-source-paths ["java"]) 39 | -------------------------------------------------------------------------------- /resources/gradients.edn: -------------------------------------------------------------------------------- 1 | {:neon-colors {:tensor-index 0, :gradient-shape [260 3]}, :aquamarine {:tensor-index 1, :gradient-shape [260 3]}, :coffee-tones {:tensor-index 2, :gradient-shape [260 3]}, :deep-sea-colors {:tensor-index 3, :gradient-shape [260 3]}, :fuchsia-tones {:tensor-index 4, :gradient-shape [260 3]}, :green-red {:tensor-index 50, :gradient-shape [260 3]}, :red-blue-tones {:tensor-index 5, :gradient-shape [260 3]}, :brass-tones {:tensor-index 6, :gradient-shape [260 3]}, :army-colors {:tensor-index 7, :gradient-shape [260 3]}, :green-pink-tones {:tensor-index 8, :gradient-shape [260 3]}, :green-brown-terrain {:tensor-index 9, :gradient-shape [260 3]}, :thermometer-colors {:tensor-index 10, :gradient-shape [260 3]}, :cherry-tones {:tensor-index 11, :gradient-shape [260 3]}, :atlantic-colors {:tensor-index 12, :gradient-shape [260 3]}, :dark-terrain {:tensor-index 13, :gradient-shape [260 3]}, :fall-colors {:tensor-index 14, :gradient-shape [260 3]}, :rainbow {:tensor-index 15, :gradient-shape [260 3]}, :fruit-punch-colors {:tensor-index 16, :gradient-shape [260 3]}, :solar-colors {:tensor-index 17, :gradient-shape [260 3]}, :dark-bands {:tensor-index 18, :gradient-shape [260 3]}, :avocado-colors {:tensor-index 19, :gradient-shape [260 3]}, :light-temperature-map {:tensor-index 20, :gradient-shape [260 3]}, :sunset-colors {:tensor-index 21, :gradient-shape [260 3]}, :candy-colors {:tensor-index 22, :gradient-shape [260 3]}, :watermelon-colors {:tensor-index 23, :gradient-shape [260 3]}, :beach-colors {:tensor-index 24, :gradient-shape [260 3]}, :island-colors {:tensor-index 25, :gradient-shape [260 3]}, :pastel {:tensor-index 26, :gradient-shape [260 3]}, :gray-yellow-tones {:tensor-index 27, :gradient-shape [260 3]}, :temperature-map {:tensor-index 28, :gradient-shape [260 3]}, :blue-green-yellow {:tensor-index 29, :gradient-shape [260 3]}, :bright-bands {:tensor-index 30, :gradient-shape [260 3]}, :sandy-terrain {:tensor-index 31, :gradient-shape [260 3]}, :gray-tones {:tensor-index 32, :gradient-shape [260 3]}, :rust-tones {:tensor-index 33, :gradient-shape [260 3]}, :pearl-colors {:tensor-index 34, :gradient-shape [260 3]}, :dark-rainbow {:tensor-index 35, :gradient-shape [260 3]}, :valentine-tones {:tensor-index 36, :gradient-shape [260 3]}, :cmyk-colors {:tensor-index 37, :gradient-shape [260 3]}, :mint-colors {:tensor-index 38, :gradient-shape [260 3]}, :lake-colors {:tensor-index 39, :gradient-shape [260 3]}, :pigeon-tones {:tensor-index 40, :gradient-shape [260 3]}, :aurora-colors {:tensor-index 41, :gradient-shape [260 3]}, :plum-colors {:tensor-index 42, :gradient-shape [260 3]}, :alpine-colors {:tensor-index 43, :gradient-shape [260 3]}, :light-terrain {:tensor-index 44, :gradient-shape [260 3]}, :southwest-colors {:tensor-index 45, :gradient-shape [260 3]}, :sienna-tones {:tensor-index 46, :gradient-shape [260 3]}, :brown-cyan-tones {:tensor-index 47, :gradient-shape [260 3]}, :rose-colors {:tensor-index 48, :gradient-shape [260 3]}, :starry-night-colors {:tensor-index 49, :gradient-shape [260 3]}} -------------------------------------------------------------------------------- /resources/gradients.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techascent/tech.datatype/8cc83d771d9621d580fd5d4d0625005bd7ab0e0c/resources/gradients.png -------------------------------------------------------------------------------- /src/tech/compute/context.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.context 2 | "Compute context allows someone to setup the default driver, device, and stream to 3 | use in function calls." 4 | (:require [tech.compute.registry :as registry] 5 | [tech.compute.driver :as drv])) 6 | 7 | 8 | (def ^:dynamic *context {}) 9 | 10 | 11 | (defn default-driver 12 | [] 13 | (or (:driver *context) 14 | (registry/driver @registry/*cpu-driver-name*))) 15 | 16 | 17 | (defn default-device 18 | [] 19 | (let [retval 20 | (or (:device *context) 21 | (first (drv/get-devices (default-driver))))] 22 | (when-not retval 23 | (throw (Exception. 24 | (format "%s: No devices found" 25 | (drv/driver-name (default-driver)))))) 26 | retval)) 27 | 28 | 29 | (defn default-stream 30 | [] 31 | (or (:stream *context) 32 | (drv/default-stream (default-device)))) 33 | 34 | 35 | (defmacro with-context 36 | [context & body] 37 | `(with-bindings {#'*context ~context} 38 | ~@body)) 39 | 40 | 41 | (defmacro with-merged-context 42 | [context & body] 43 | `(with-context 44 | (merge *context ~context) 45 | ~@body)) 46 | 47 | 48 | (defn default-context 49 | [] 50 | {:driver (default-driver) 51 | :device (default-device) 52 | :stream (default-stream)}) 53 | 54 | 55 | (defn options->context 56 | "Given an options map, return a augmented map that always includes 57 | device, driver, and stream." 58 | [opt-map] 59 | (merge (default-context) opt-map)) 60 | -------------------------------------------------------------------------------- /src/tech/compute/cpu/utils.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.cpu.utils) 2 | 3 | 4 | (set! *warn-on-reflection* true) 5 | (set! *unchecked-math* true) 6 | 7 | 8 | (defn in-range? 9 | [^long lhs-off ^long lhs-len ^long rhs-off ^long rhs-len] 10 | (or (and (>= rhs-off lhs-off) 11 | (< rhs-off (+ lhs-off lhs-len))) 12 | (and (>= lhs-off rhs-off) 13 | (< lhs-off (+ rhs-off rhs-len))))) 14 | -------------------------------------------------------------------------------- /src/tech/compute/driver.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.driver 2 | "Base set of protocols required to move information from the host to the device as well as 3 | enable some form of computation on a given device. There is a cpu implementation provided for 4 | reference. 5 | 6 | Base datatypes are defined: 7 | * Driver: Enables enumeration of devices and creation of host buffers. 8 | * Device: Creates streams and device buffers. 9 | * Stream: Stream of execution occuring on the device. 10 | * Event: A synchronization primitive emitted in a stream to notify other 11 | streams that might be blocking.") 12 | 13 | 14 | (defprotocol PDriverProvider 15 | "Get a driver from an object" 16 | (get-driver [impl])) 17 | 18 | 19 | (defprotocol PDeviceProvider 20 | "Get a device from an object." 21 | (get-device [impl])) 22 | 23 | 24 | (defprotocol PStreamProvider 25 | "Get a stream from an object" 26 | (get-stream [impl])) 27 | 28 | 29 | (defprotocol PDriver 30 | "A driver is a generic compute abstraction. Could be a group of threads, could be a 31 | machine on a network or it could be a CUDA or OpenCL driver. A stream is a stream of 32 | execution (analogous to a thread) where subsequent calls are serialized. All buffers 33 | implement a few of the datatype interfaces, at least get-datatype and ecount. Host 34 | buffers are expected to implement enough of the datatype interfaces to allow a copy 35 | operation from generic datatypes into them. This means at least PAccess." 36 | (driver-name [driver] 37 | "A system-unique name for this driver. The keyworded namespace it is implemented in 38 | is ideal.") 39 | (get-devices [driver] 40 | "Get a list of devices accessible to the system.") 41 | (allocate-host-buffer [driver elem-count elem-type options] 42 | "Allocate a host buffer. Transfer from host to device requires data first copied 43 | into a host buffer and then uploaded to a device buffer. 44 | options: 45 | :usage-type 46 | usage-type: #{:one-time :reusable} 47 | Hint to allow implementations to allocate different types of host buffers each 48 | optimized for the desired use case. Default is one-time.") 49 | (acceptable-host-buffer? [driver buffer] 50 | "Will this buffer work as a host buffer?")) 51 | 52 | (defonce host-buffer-usage-types #{:one-time :reusable}) 53 | 54 | 55 | (defprotocol PDevice 56 | (supports-create-stream? [device] 57 | "Does this device support create-stream?") 58 | (default-stream [device] 59 | "All devices must have a default stream whether they support create or not.") 60 | (create-stream [device] 61 | "Create a stream of execution. Streams are indepenent threads of execution. They 62 | can be synchronized with each other and the main thread using events.") 63 | (allocate-device-buffer [device elem-count elem-type options] 64 | "Allocate a device buffer. This is the generic unit of data storage used for 65 | computation. No options at this time.") 66 | (device->device-copy-compatible? [src-device dst-device] 67 | "When two devices differ, it may be possible to copy from src to dest device.") 68 | (acceptable-device-buffer? [device item] 69 | "Do whatever checks necessary to ensure that this item can be used as a device 70 | buffer for this device.")) 71 | 72 | 73 | (defprotocol PStream 74 | "Basic functionality expected of streams. Streams are an abstraction of a stream of 75 | execution and can be synchonized with the host or with each other using events. CPU's 76 | are considered devices." 77 | (copy-device->device [stream dev-a dev-a-off dev-b dev-b-off elem-count] 78 | "copy from one device to another. Used to initiate host->device, device->host, 79 | host->host and device->device copies with memcpy semantics.") 80 | (sync-with-host [stream] 81 | "Block host until stream's queue is finished executing") 82 | (sync-with-stream [src-stream dst-stream] 83 | "Create an event in src-stream's execution queue, then have dst stream wait on 84 | that event. This allows dst-stream to ensure src-stream has reached a certain point 85 | of execution before continuing. Both streams must be of the same driver.")) 86 | -------------------------------------------------------------------------------- /src/tech/compute/registry.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.registry 2 | "Place to store global information about the drivers available to the compute 3 | subystem." 4 | (:require [tech.compute.driver :as drv])) 5 | 6 | 7 | (def ^:dynamic *registered-drivers* (atom {})) 8 | (def ^:dynamic *cpu-driver-name* (atom nil)) 9 | 10 | 11 | (defn- find-driver 12 | [driver-name] 13 | (get @*registered-drivers* driver-name)) 14 | 15 | 16 | (defn driver 17 | [driver-name] 18 | (if-let [retval (find-driver driver-name)] 19 | retval 20 | (throw (ex-info (format "Failed to find driver. Perhaps a require is missing?" ) 21 | {:driver-name driver-name})))) 22 | 23 | 24 | (defn register-driver 25 | [driver] 26 | (swap! *registered-drivers* assoc (drv/driver-name driver) driver) 27 | (drv/driver-name driver)) 28 | 29 | 30 | (defn driver-names 31 | [] 32 | (->> (keys @*registered-drivers*) 33 | set)) 34 | 35 | 36 | ;;The cpu driver has a special place in that it can attach to things that 37 | ;;aren't in the ecosystem. 38 | (defn set-cpu-driver-name! 39 | [driver-name] 40 | (reset! *cpu-driver-name* driver-name)) 41 | 42 | 43 | (defn cpu-driver-name 44 | [] 45 | @*cpu-driver-name*) 46 | 47 | 48 | (defmacro current-ns->keyword 49 | "Use this to name your driver." 50 | [] 51 | `(keyword (str *ns*))) 52 | -------------------------------------------------------------------------------- /src/tech/compute/verify/driver.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.verify.driver 2 | (:require [clojure.test :refer :all] 3 | [tech.compute.driver :as drv] 4 | [tech.v2.datatype :as dtype] 5 | [tech.v2.datatype.functional :as dfn] 6 | [tech.compute.verify.utils :as verify-utils] 7 | [tech.compute :as compute] 8 | [tech.compute.context :as compute-ctx])) 9 | 10 | 11 | 12 | (defn simple-stream 13 | [driver datatype] 14 | (verify-utils/with-default-device-and-stream 15 | driver 16 | (let [{:keys [driver device stream]} (compute-ctx/options->context {}) 17 | buf-a (compute/allocate-host-buffer driver 10 datatype) 18 | output-buf-a (compute/allocate-host-buffer driver 10 datatype) 19 | buf-b (compute/allocate-device-buffer device 10 datatype) 20 | input-data (dtype/make-container :typed-buffer datatype (range 10)) 21 | output-data (dtype/make-container :typed-buffer datatype 10)] 22 | (dtype/copy! input-data 0 buf-a 0 10) 23 | (dtype/set-value! buf-a 0 100.0) 24 | (dtype/copy! buf-a 0 output-data 0 10) 25 | (compute/copy-device->device buf-a 0 buf-b 0 10) 26 | (compute/copy-device->device buf-b 0 output-buf-a 0 10) 27 | (compute/sync-with-host stream) 28 | (dtype/copy! output-buf-a 0 output-data 0 10) 29 | (is (dfn/equals [100.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0] 30 | output-data))))) 31 | -------------------------------------------------------------------------------- /src/tech/compute/verify/tensor.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.verify.tensor 2 | (:require [tech.compute.context :as compute-ctx] 3 | [tech.v2.tensor.impl :as dtt-impl] 4 | [tech.v2.datatype.functional :as dfn] 5 | [tech.v2.datatype :as dtype] 6 | [tech.v2.tensor :as dtt] 7 | [tech.compute.tensor :as ct] 8 | [tech.resource :as resource] 9 | [clojure.test :refer :all])) 10 | 11 | 12 | (defmacro verify-context 13 | [driver datatype & body] 14 | `(resource/stack-resource-context 15 | (compute-ctx/with-context 16 | {:driver ~driver} 17 | (dtt-impl/with-datatype 18 | ~datatype 19 | ~@body)))) 20 | 21 | 22 | (defn clone 23 | [driver datatype] 24 | (verify-context 25 | driver datatype 26 | (let [tensor (ct/->tensor (partition 3 (range 9))) 27 | dev-tens (ct/clone-to-device tensor) 28 | host-tens (ct/clone-to-host dev-tens)] 29 | (is (dfn/equals tensor host-tens)) 30 | (let [sub-tens (dtt/select tensor [0 1] [0 1]) 31 | dev-tens (ct/clone-to-device sub-tens) 32 | host-tens (ct/clone-to-host dev-tens)] 33 | (is (dfn/equals sub-tens host-tens)))))) 34 | 35 | 36 | (defn assign! 37 | [driver datatype] 38 | (verify-context 39 | driver datatype 40 | (let [tensor (ct/->tensor (partition 3 (range 9))) 41 | dev-tens (ct/new-tensor [3 3]) 42 | _ (ct/assign! dev-tens tensor) 43 | host-tens (ct/clone-to-host dev-tens)] 44 | (is (dfn/equals tensor host-tens))))) 45 | -------------------------------------------------------------------------------- /src/tech/compute/verify/utils.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.verify.utils 2 | (:require [tech.resource :as resource] 3 | [clojure.test :refer :all] 4 | [tech.v2.datatype.casting :as casting] 5 | [tech.compute :as compute] 6 | [tech.compute.context :as compute-ctx]) 7 | (:import [java.math BigDecimal MathContext])) 8 | 9 | 10 | (defn test-wrapper 11 | [test-fn] 12 | (resource/stack-resource-context 13 | ;;Turn on if you want much slower tests. 14 | (test-fn))) 15 | 16 | 17 | (defmacro with-default-device-and-stream 18 | [driver & body] 19 | `(resource/stack-resource-context 20 | (compute-ctx/with-context 21 | {:driver ~driver}) 22 | ~@body)) 23 | 24 | 25 | (def ^:dynamic *datatype* :float64) 26 | 27 | 28 | (defmacro datatype-list-tests 29 | [datatype-list test-name & body] 30 | `(do 31 | ~@(for [datatype datatype-list] 32 | (do 33 | `(deftest ~(symbol (str test-name "-" (name datatype))) 34 | (with-bindings {#'*datatype* ~datatype} 35 | ~@body)))))) 36 | 37 | 38 | 39 | (defmacro def-double-float-test 40 | [test-name & body] 41 | `(datatype-list-tests [:float64 :float32] ~test-name ~@body)) 42 | 43 | 44 | (defmacro def-int-long-test 45 | [test-name & body] 46 | `(datatype-list-tests [:int32 :uint32 :int64 :uint64] 47 | ~test-name 48 | ~@body)) 49 | 50 | 51 | (defmacro def-all-dtype-test 52 | [test-name & body] 53 | `(datatype-list-tests ~casting/numeric-types ~test-name ~@body)) 54 | 55 | 56 | (defmacro def-all-dtype-exception-unsigned 57 | "Some platforms can detect unsigned errors." 58 | [test-name & body] 59 | `(do 60 | (datatype-list-tests ~casting/host-numeric-types ~test-name ~@body) 61 | (datatype-list-tests ~casting/unsigned-int-types ~test-name 62 | (is (thrown? Throwable 63 | ~@body))))) 64 | -------------------------------------------------------------------------------- /src/tech/libs/neanderthal.clj: -------------------------------------------------------------------------------- 1 | (ns tech.libs.neanderthal 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.jna :as jna] 5 | [tech.v2.tensor.protocols :as dtt-proto]) 6 | (:import [uncomplicate.neanderthal.internal.api Block] 7 | [uncomplicate.commons.core Info] 8 | [com.sun.jna Pointer])) 9 | 10 | 11 | (extend-type Info 12 | dtype-proto/PDatatype 13 | (get-datatype [this] 14 | (let [{:keys [entry-type]} (.info this)] 15 | (if-let [retval (get {Double/TYPE :float64 16 | Float/TYPE :float32 17 | Long/TYPE :int64 18 | Integer/TYPE :int32} 19 | entry-type)] 20 | retval 21 | (throw (Exception. "Unrecognized type."))))) 22 | dtype-proto/PCountable 23 | (ecount [this] (:dim (.info this))) 24 | dtype-proto/PShape 25 | (shape [this] (let [info (.info this)] 26 | (if (contains? info :matrix-type) 27 | [(:m info) (:n info)] 28 | [(:dim info)]))) 29 | 30 | dtype-proto/PToBufferDesc 31 | (convertible-to-buffer-desc? [item] true) 32 | (->buffer-descriptor [item] 33 | (let [item-info (.info item) 34 | item-dtype (dtype-proto/get-datatype item) 35 | item-shape (dtype-proto/shape item)] 36 | (when-not (or (== 1 (count item-shape)) (get-in item-info [:storage :gapless])) 37 | (throw (Exception. "Only dense neanderthal matrixes supported"))) 38 | {:ptr (jna/->ptr-backing-store item) 39 | :datatype item-dtype 40 | :shape item-shape 41 | ;;TVM needs the device type 42 | :device-type (:device item-info) 43 | :strides (mapv #(* (casting/numeric-byte-width item-dtype) %) 44 | (if (= 2 (count item-shape)) 45 | (let [item-strides 46 | [(if (= :row (:layout item-info)) 47 | (second item-shape) 48 | (first item-shape)) 49 | 1]] 50 | (if (= :column (:layout item-info)) 51 | (reverse item-strides) 52 | item-strides)) 53 | [(:stride item-info)]))})) 54 | 55 | 56 | dtype-proto/PToReader 57 | (convertible-to-reader? [item] (= :cpu (:device (.info item)))) 58 | (->reader [item options] 59 | (dtype-proto/->reader (dtt-proto/convert-to-tensor item) options)) 60 | 61 | dtype-proto/PToWriter 62 | (convertible-to-writer? [item] (= :cpu (:device (.info item)))) 63 | (->writer [item options] 64 | (dtype-proto/->writer (dtt-proto/convert-to-tensor item) options))) 65 | 66 | 67 | (extend-type Block 68 | jna/PToPtr 69 | (is-convertible-to-ptr? [item] true) 70 | (->ptr-backing-store [item] 71 | (let [byte-width (casting/numeric-byte-width 72 | (dtype-proto/get-datatype item)) 73 | item-offset (* (.offset item) byte-width) 74 | ^Pointer ptr-val (jna/->ptr-backing-store (.buffer item))] 75 | (Pointer. (+ item-offset 76 | (Pointer/nativeValue ptr-val)))))) 77 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/argsort.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.argsort 2 | (:require [tech.v2.datatype.typecast :as typecast] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.protocols :as dtype-proto] 5 | [tech.v2.datatype.base :as dtype-base] 6 | [tech.v2.datatype.comparator :as dtype-comp] 7 | [tech.v2.datatype.operation-provider :as op-provider]) 8 | (:import [it.unimi.dsi.fastutil.bytes ByteArrays ByteComparator] 9 | [it.unimi.dsi.fastutil.shorts ShortArrays ShortComparator] 10 | [it.unimi.dsi.fastutil.ints IntArrays IntComparator] 11 | [it.unimi.dsi.fastutil.longs LongArrays LongComparator] 12 | [it.unimi.dsi.fastutil.floats FloatArrays FloatComparator] 13 | [it.unimi.dsi.fastutil.doubles DoubleArrays DoubleComparator] 14 | [tech.v2.datatype 15 | Comparator$ByteComp 16 | Comparator$ShortComp 17 | Comparator$IntComp 18 | Comparator$LongComp 19 | Comparator$FloatComp 20 | Comparator$DoubleComp] 21 | [java.util Comparator Arrays])) 22 | 23 | 24 | (set! *warn-on-reflection* true) 25 | (set! *unchecked-math* :warn-on-boxed) 26 | 27 | 28 | (defmacro make-argsort 29 | [datatype] 30 | `(fn [values# parallel?# reverse?# comparator#] 31 | (let [comparator# (or comparator# 32 | (dtype-comp/default-comparator ~datatype)) 33 | n-elems# (int (dtype-proto/ecount values#))] 34 | (if (= n-elems# 0) 35 | (int-array 0) 36 | (let [^"[I" index-array# (dtype-proto/make-container :java-array :int32 37 | (range n-elems#) 38 | {:unchecked? true}) 39 | values# (typecast/datatype->reader ~datatype values# true) 40 | value-comparator# (dtype-comp/datatype->comparator ~datatype comparator#) 41 | idx-comparator# (if reverse?# 42 | (dtype-comp/make-comparator 43 | :int32 (.compare value-comparator# 44 | (.read values# ~'rhs) 45 | (.read values# ~'lhs))) 46 | (dtype-comp/make-comparator 47 | :int32 (.compare value-comparator# 48 | (.read values# ~'lhs) 49 | (.read values# ~'rhs))))] 50 | (if parallel?# 51 | (IntArrays/parallelQuickSort index-array# ^IntComparator idx-comparator#) 52 | (IntArrays/quickSort index-array# ^IntComparator idx-comparator#)) 53 | index-array#))))) 54 | 55 | 56 | (def argsort-table (casting/make-base-no-boolean-datatype-table 57 | make-argsort)) 58 | 59 | 60 | (defn argsort 61 | [values {:keys [parallel? 62 | comparator 63 | datatype 64 | reverse?] 65 | :or {parallel? true} 66 | :as options}] 67 | (let [datatype (or datatype (dtype-base/get-datatype values)) 68 | sort-fn (get argsort-table (casting/safe-flatten datatype))] 69 | (sort-fn values parallel? reverse? comparator))) 70 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/argtypes.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.argtypes 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.shape :as dtype-shape]) 4 | (:import [tech.v2.datatype.protocols PToReader] 5 | [java.util RandomAccess])) 6 | 7 | 8 | (defn arg->arg-type 9 | [arg] 10 | (cond 11 | (dtype-shape/scalar? arg) 12 | :scalar 13 | (or (instance? tech.v2.datatype.protocols.PToReader arg) 14 | (instance? RandomAccess arg) 15 | (dtype-proto/convertible-to-reader? arg)) 16 | :reader 17 | (or (instance? Iterable arg) 18 | (dtype-proto/convertible-to-iterable? arg)) 19 | :iterable 20 | :else 21 | :scalar)) 22 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/base_macros.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.base-macros 2 | (:require [tech.v2.datatype.protocols :as dtype-proto])) 3 | 4 | (defmacro try-catch-any 5 | [try-body & catch-body] 6 | `(try 7 | ~try-body 8 | (catch Throwable ~(first catch-body) 9 | ~@(rest catch-body)))) 10 | 11 | 12 | (defmacro check-range 13 | [item offset elem-count] 14 | `(when-not (<= (+ ~offset ~elem-count) 15 | (dtype-proto/ecount ~item)) 16 | (throw (ex-info (format "%s offset + n-elems > ecount range violation" 17 | ~(name item)) 18 | {:offset ~offset 19 | :n-elems ~elem-count 20 | :length (dtype-proto/ecount ~item)})))) 21 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/base_macros.cljs: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.base-macros) 2 | 3 | (defmacro try-catch-any 4 | [try-body & catch-body] 5 | `(try 6 | ~try-body 7 | (catch :default ~(first catch-body) 8 | ~@(rest catch-body)))) 9 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/binary_search.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.binary-search 2 | (:require [tech.v2.datatype.typecast :as typecast] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.base :as dtype-base] 5 | [tech.v2.datatype.comparator :as dtype-comp])) 6 | 7 | 8 | (set! *warn-on-reflection* true) 9 | (set! *unchecked-math* :warn-on-boxed) 10 | 11 | 12 | (defmacro make-binary-search 13 | [datatype] 14 | `(fn [values# target# comparator#] 15 | (let [target# (casting/datatype->cast-fn :unknown ~datatype target#) 16 | values# (typecast/datatype->reader ~datatype values# true) 17 | comparator# (or comparator# 18 | (dtype-comp/make-comparator 19 | ~datatype (dtype-comp/default-compare-fn 20 | ~datatype ~'lhs ~'rhs))) 21 | comparator# (dtype-comp/datatype->comparator ~datatype comparator#) 22 | buf-ecount# (.size values#)] 23 | (if (= 0 buf-ecount#) 24 | [false 0] 25 | (loop [low# (int 0) 26 | high# (int buf-ecount#)] 27 | (if (< low# high#) 28 | (let [mid# (+ low# (quot (- high# low#) 2)) 29 | buf-data# (.read values# mid#) 30 | compare-result# (.compare comparator# buf-data# target#)] 31 | (if (= 0 compare-result#) 32 | (recur mid# mid#) 33 | (if (and (< compare-result# 0) 34 | (not= mid# low#)) 35 | (recur mid# high#) 36 | (recur low# mid#)))) 37 | (let [buf-val# (.read values# low#)] 38 | (if (<= (.compare comparator# target# buf-val#) 0) 39 | [(= target# buf-val#) low#] 40 | [false (unchecked-inc low#)])))))))) 41 | 42 | 43 | (def binary-search-table (casting/make-base-no-boolean-datatype-table 44 | make-binary-search)) 45 | 46 | 47 | (defn binary-search 48 | "Perform binary search returning long idx of matching value or insert position. 49 | Returns index of the element or the index where it should be inserted. Returns 50 | a tuple of [found? insert-or-elem-pos]" 51 | [values target {:keys [datatype 52 | comparator]}] 53 | (let [datatype (or datatype (dtype-base/get-datatype target))] 54 | (if-let [value-fn (get binary-search-table (casting/safe-flatten 55 | datatype))] 56 | (value-fn values target comparator) 57 | (throw (ex-info (format "No search mechanism found for datatype %s" datatype) 58 | {}))))) 59 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/clj_range.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.clj-range 2 | "Datatype bindings for clojure ranges." 3 | (:require [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.base :as base] 5 | [tech.v2.datatype.typecast :as typecast] 6 | [tech.v2.datatype.casting :as casting]) 7 | (:import [clojure.lang LongRange Range] 8 | [tech.v2.datatype LongReader] 9 | [java.lang.reflect Field])) 10 | 11 | 12 | (set! *warn-on-reflection* true) 13 | 14 | 15 | (def lr-step-field (doto (.getDeclaredField ^Class LongRange "step") 16 | (.setAccessible true))) 17 | 18 | 19 | (extend-type LongRange 20 | dtype-proto/PDatatype 21 | (get-datatype [rng] :int64) 22 | dtype-proto/PCountable 23 | (ecount [rng] (.count rng)) 24 | dtype-proto/PClone 25 | (clone [rng] rng) 26 | dtype-proto/PToReader 27 | (convertible-to-reader? [rng] true) 28 | (->reader [rng options] 29 | (let [start (long (first rng)) 30 | step (long (.get ^Field lr-step-field rng)) 31 | n-elems (.count rng)] 32 | (-> (reify 33 | LongReader 34 | (lsize [rdr] n-elems) 35 | (read [rdr idx] 36 | (-> (* step idx) 37 | (+ start))) 38 | dtype-proto/PRangeConvertible 39 | (convertible-to-range? [item] true) 40 | (->range [item options] (dtype-proto/->range rng options)) 41 | dtype-proto/PConstantTimeMinMax 42 | (has-constant-time-min-max? [item] true) 43 | (constant-time-min [item] (dtype-proto/constant-time-min rng)) 44 | (constant-time-max [item] (dtype-proto/constant-time-max rng))) 45 | (dtype-proto/->reader options))))) 46 | 47 | 48 | (def r-step-field (doto (.getDeclaredField ^Class Range "step") 49 | (.setAccessible true))) 50 | 51 | 52 | (defmacro range-reader-macro 53 | [datatype rng options] 54 | `(let [rng# ~rng 55 | start# (casting/datatype->cast-fn :unknown ~datatype (first rng#)) 56 | step# (casting/datatype->cast-fn :unknown ~datatype 57 | (.get ^Field r-step-field rng#)) 58 | n-elems# (.count rng#)] 59 | (-> (reify ~(typecast/datatype->reader-type datatype) 60 | (lsize [rdr#] n-elems#) 61 | (read [rdr# idx#] 62 | (casting/datatype->cast-fn :uknown ~datatype 63 | (-> (* step# idx#) 64 | (+ start#))))) 65 | (dtype-proto/->reader ~options)))) 66 | 67 | 68 | (extend-type Range 69 | dtype-proto/PDatatype 70 | (get-datatype [rng] (base/get-datatype (first rng))) 71 | dtype-proto/PCountable 72 | (ecount [rng] (.count rng)) 73 | dtype-proto/PClone 74 | (clone [rng] rng) 75 | dtype-proto/PToReader 76 | (convertible-to-reader? [rng] (contains? 77 | #{:int8 :int16 :int32 :int64 78 | :float32 :float64} 79 | (base/get-datatype rng))) 80 | (->reader [rng options] 81 | (case (base/get-datatype (first rng)) 82 | :int8 (range-reader-macro :int8 rng options) 83 | :int16 (range-reader-macro :int16 rng options) 84 | :int32 (range-reader-macro :int32 rng options) 85 | :int64 (range-reader-macro :int64 rng options) 86 | :float32 (range-reader-macro :float32 rng options) 87 | :float64 (range-reader-macro :float64 rng options)))) 88 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/direct_mapped.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.direct-mapped 2 | (:require [tech.jna :as jna]) 3 | (:import [tech.v2.datatype DirectMappedOps] 4 | [com.sun.jna Native])) 5 | 6 | 7 | 8 | (defn- setup-direct-mapping! 9 | [] 10 | (let [library (jna/load-library (jna/c-library-name))] 11 | (com.sun.jna.Native/register DirectMappedOps library) 12 | :ok)) 13 | 14 | (def direct-mapping (delay (setup-direct-mapping!))) 15 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/double_ops.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.double-ops 2 | (:require [primitive-math :as pmath])) 3 | 4 | 5 | (defmacro SIGNIFICAND-BITS 6 | [] 7 | `52) 8 | 9 | (defmacro SIGNIFICAND-MASK [] `0x000fffffffffffff) 10 | 11 | (defmacro IMPLICIT-BIT [] `(pmath/+ (SIGNIFICAND-MASK) 1)) 12 | 13 | (defmacro is-finite? 14 | [x] 15 | `(<= (Math/getExponent ~x) 16 | Double/MAX_EXPONENT)) 17 | 18 | (defn get-significand 19 | ^long [^double d] 20 | (when-not (is-finite? d) 21 | (throw (Exception. "not a normal value"))) 22 | (let [exponent (Math/getExponent d) 23 | bits (Double/doubleToRawLongBits d) 24 | bits (bit-and bits (SIGNIFICAND-MASK))] 25 | (if (== exponent (- Double/MIN_EXPONENT 1)) 26 | (bit-shift-left bits 1) 27 | (bit-or bits (IMPLICIT-BIT))))) 28 | 29 | 30 | ;; https://github.com/google/guava/blob/master/guava/src/com/google/common/math/DoubleMath.java#L279-L290 31 | (defmacro is-mathematical-integer? 32 | [x] 33 | `(let [x# ~x] 34 | (boolean 35 | (and (is-finite? x#) 36 | (or (pmath/== x# 0.0) 37 | (pmath/<= (- (SIGNIFICAND-BITS) 38 | (Long/numberOfTrailingZeros (get-significand x#))) 39 | (Math/getExponent x#))))))) 40 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/functional/interpreter.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.functional.interpreter 2 | (:require [tech.v2.datatype.functional.impl :as func-impl])) 3 | 4 | 5 | (set! *unchecked-math* :warn-on-boxed) 6 | (set! *warn-on-reflection* true) 7 | 8 | 9 | 10 | (defonce ^:dynamic *registered-language-fns* (atom {})) 11 | 12 | 13 | (defn register-symbol! 14 | [registration-atom sym-name sym-value] 15 | (-> (swap! (or registration-atom 16 | *registered-language-fns*) 17 | assoc sym-name sym-value) 18 | keys)) 19 | 20 | 21 | (defn get-operand 22 | "Return a map of (at least) 23 | {:type op-type 24 | :operand op-fn 25 | }" 26 | [{:keys [symbol-map]} op-kwd] 27 | (if-let [item-val (get symbol-map op-kwd)] 28 | item-val 29 | (if-let [retval (get @*registered-language-fns* op-kwd)] 30 | retval 31 | (throw (ex-info (format "Failed to find math operand: %s" op-kwd) 32 | {:operand op-kwd}))))) 33 | 34 | 35 | (defn eval-expr 36 | "Tiny simple interpreter." 37 | [env math-expr] 38 | (cond 39 | (sequential? math-expr) 40 | (if (symbol? (first math-expr)) 41 | (let [fn-name (first math-expr) 42 | ;;Force errors early 43 | expr-args (mapv (partial eval-expr env) (rest math-expr)) 44 | operand (get-operand env fn-name)] 45 | (try 46 | (apply operand env expr-args) 47 | (catch Throwable e 48 | (throw (ex-info (format "Operator %s failed:\n%s" math-expr (.getMessage e)) 49 | {:math-expression math-expr 50 | :error e}))))) 51 | (map partial eval-expr env math-expr)) 52 | (symbol? math-expr) 53 | (get-operand env math-expr) 54 | :else 55 | math-expr)) 56 | 57 | 58 | (defn symbol->str 59 | [sym] 60 | (if (namespace sym) 61 | (str (namespace sym) "/" (name sym)) 62 | (str (name sym)))) 63 | 64 | 65 | (defn ignore-env-fn 66 | [target-fn] 67 | (fn [_ & args] 68 | (apply target-fn args))) 69 | 70 | 71 | (defn register-base-symbols 72 | [& [registration-atom]] 73 | (let [registration-atom (or registration-atom *registered-language-fns*) 74 | builtin-set (set (concat (keys func-impl/all-builtins) 75 | [:indexed-reader 76 | :argsort 77 | :binary-search 78 | :argfilter 79 | :magnitude-squared 80 | :magnitude 81 | :dot-product 82 | :mean 83 | :median 84 | :geometric-mean 85 | :harmonic-mean 86 | :variance 87 | :variance-population 88 | :standard-deviation 89 | :standard-deviation-population 90 | :standard-error 91 | :skewness 92 | :skewness-population 93 | :kurtosis 94 | :kurtosis-population 95 | :pearsons-correlation 96 | :spearmans-correlation 97 | :kendalls-correlation 98 | :fixed-rolling-window]))] 99 | (doseq [kwd builtin-set] 100 | (if-let [public-var (resolve (symbol "tech.v2.datatype.functional" 101 | (name kwd)))] 102 | (swap! registration-atom assoc (symbol (name kwd)) 103 | (ignore-env-fn 104 | @public-var)) 105 | (throw (ex-info (format "Failed to find var: %s" (str kwd)) {})))) 106 | (keys @registration-atom))) 107 | 108 | 109 | (register-base-symbols) 110 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/iterable/concat.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.iterable.concat 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.typecast :as typecast])) 5 | 6 | 7 | (defmacro make-concat-iterable-impl 8 | [datatype] 9 | `(fn [datatype# concat-args#] 10 | (reify 11 | dtype-proto/PDatatype 12 | (get-datatype [item#] datatype#) 13 | Iterable 14 | (iterator [item#] 15 | (let [concat-args# (map #(typecast/datatype->iter ~datatype %) concat-args#) 16 | algo-data# (object-array [(first concat-args#) 17 | (rest concat-args#)])] 18 | (reify ~(typecast/datatype->iter-type datatype) 19 | (hasNext [item#] 20 | (boolean (not= nil (aget algo-data# 0)))) 21 | (~(typecast/datatype->iter-next-fn-name datatype) 22 | [item#] 23 | (let [src-iter# (typecast/datatype->fast-iter ~datatype 24 | (aget algo-data# 0)) 25 | retval# (typecast/datatype->iter-next-fn ~datatype src-iter#)] 26 | (when-not (.hasNext src-iter#) 27 | (aset algo-data# 0 (first (aget algo-data# 1))) 28 | (aset algo-data# 1 (rest (aget algo-data# 1)))) 29 | retval#)) 30 | (current [item#] 31 | (let [src-iter# (typecast/datatype->fast-iter 32 | ~datatype (aget algo-data# 0))] 33 | (.current src-iter#))))))))) 34 | 35 | 36 | (defmacro make-concat-iterable-table 37 | [] 38 | `(->> [~@(for [dtype casting/base-marshal-types] 39 | [dtype `(make-concat-iterable-impl ~dtype)])] 40 | (into {}))) 41 | 42 | 43 | (def concat-iterable-table (make-concat-iterable-table)) 44 | 45 | 46 | (defn iterable-concat 47 | [{:keys [datatype]} concat-iters] 48 | (let [datatype (or datatype 49 | (when-let [first-arg (first concat-iters)] 50 | (dtype-proto/get-datatype first-arg)) 51 | :float64) 52 | create-fn (get concat-iterable-table (casting/safe-flatten datatype))] 53 | (create-fn datatype concat-iters))) 54 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/iterable/const.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.iterable.const 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.typecast :as typecast] 5 | [tech.v2.datatype.nio-access :as access])) 6 | 7 | 8 | 9 | (defmacro make-const-iter 10 | [datatype] 11 | (let [host-type (casting/safe-flatten datatype)] 12 | `(fn [item#] 13 | (let [item# (access/checked-full-write-cast 14 | item# :unknown ~datatype 15 | ~host-type)] 16 | (reify 17 | Iterable 18 | (iterator [item] 19 | (reify ~(typecast/datatype->iter-type host-type) 20 | (getDatatype [iter#] ~datatype) 21 | (hasNext [iter#] true) 22 | (~(typecast/datatype->iter-next-fn-name host-type) [iter#] item#) 23 | (current [iter#] item#))) 24 | dtype-proto/PDatatype 25 | (get-datatype [item] ~datatype)))))) 26 | 27 | 28 | (defmacro make-const-iter-table 29 | [] 30 | `(->> [~@(for [dtype casting/base-marshal-types] 31 | [dtype `(make-const-iter ~dtype)])] 32 | (into {}))) 33 | 34 | 35 | (def const-iter-table (make-const-iter-table)) 36 | 37 | 38 | (defn make-const-iterable 39 | [item datatype] 40 | (if-let [iter-fn (get const-iter-table (casting/flatten-datatype datatype))] 41 | (iter-fn item) 42 | (throw (ex-info (format "Failed to find iter for datatype %s" datatype) {})))) 43 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/iterable/masked.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.iterable.masked 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.typecast :as typecast])) 5 | 6 | 7 | (defmacro make-masked-iterable-impl 8 | [datatype] 9 | `(fn [datatype# values# mask# unchecked?#] 10 | (reify 11 | dtype-proto/PDatatype 12 | (get-datatype [item#] datatype#) 13 | Iterable 14 | (iterator [item#] 15 | (let [values# (typecast/datatype->iter ~datatype values# unchecked?#) 16 | mask# (typecast/datatype->iter :boolean mask# true)] 17 | (while (and (.hasNext mask#) 18 | (= false (.current mask#))) 19 | (.nextBoolean mask#) 20 | (typecast/datatype->iter-next-fn ~datatype values#)) 21 | (reify ~(typecast/datatype->iter-type datatype) 22 | (hasNext [item#] (and (.hasNext mask#) 23 | (.current mask#) 24 | (.hasNext values#))) 25 | (~(typecast/datatype->iter-next-fn-name datatype) 26 | [item#] 27 | (let [retval# (.current values#)] 28 | (when (.hasNext mask#) 29 | (.nextBoolean mask#) 30 | (typecast/datatype->iter-next-fn ~datatype values#) 31 | (while (and (.hasNext mask#) 32 | (= false (.current mask#))) 33 | (.nextBoolean mask#) 34 | (typecast/datatype->iter-next-fn ~datatype values#))) 35 | retval#)) 36 | (current [item#] (.current values#)))))))) 37 | 38 | 39 | (defmacro make-masked-iterable-table 40 | [] 41 | `(->> [~@(for [dtype casting/base-marshal-types] 42 | [dtype `(make-masked-iterable-impl ~dtype)])] 43 | (into {}))) 44 | 45 | 46 | (def masked-iterable-table (make-masked-iterable-table)) 47 | 48 | 49 | (defn iterable-mask 50 | [{:keys [datatype unchecked?]} mask-iter values] 51 | (let [datatype (or datatype (dtype-proto/get-datatype values)) 52 | mask-fn (get masked-iterable-table (casting/safe-flatten datatype))] 53 | (mask-fn datatype values mask-iter unchecked?))) 54 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/iterator.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.iterator 2 | (:require [tech.v2.datatype.typecast :as typecast] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.protocols :as dtype-proto] 5 | [tech.v2.datatype.argtypes :as argtypes] 6 | [tech.v2.datatype.iterable.const :as iter-const]) 7 | (:import [tech.v2.datatype ObjectIter ByteIter ShortIter 8 | IntIter LongIter FloatIter DoubleIter 9 | BooleanIter IteratorObjectIter])) 10 | 11 | 12 | (set! *warn-on-reflection* true) 13 | (set! *unchecked-math* :warn-on-boxed) 14 | 15 | 16 | (defmacro make-marshal-iterator-macro 17 | [src-dtype dest-dtype] 18 | `(fn [iterator# datatype# unchecked?#] 19 | (let [src-iterator# (typecast/datatype->iter ~src-dtype iterator# 20 | unchecked?#)] 21 | (if unchecked?# 22 | (reify ~(typecast/datatype->iter-type dest-dtype) 23 | (getDatatype [item#] datatype#) 24 | (hasNext [item#] (.hasNext src-iterator#)) 25 | (~(typecast/datatype->iter-next-fn-name dest-dtype) [item#] 26 | (let [temp-val# (typecast/datatype->iter-next-fn ~src-dtype src-iterator#)] 27 | (casting/datatype->unchecked-cast-fn ~src-dtype ~dest-dtype temp-val#))) 28 | (current [item#] 29 | (let [temp-val# (.current src-iterator#)] 30 | (casting/datatype->unchecked-cast-fn ~src-dtype ~dest-dtype temp-val#)))) 31 | (reify ~(typecast/datatype->iter-type dest-dtype) 32 | (getDatatype [item#] datatype#) 33 | (hasNext [item#] (.hasNext src-iterator#)) 34 | (~(typecast/datatype->iter-next-fn-name dest-dtype) [item#] 35 | (let [temp-val# (typecast/datatype->iter-next-fn ~src-dtype src-iterator#)] 36 | (casting/datatype->cast-fn ~src-dtype ~dest-dtype temp-val#))) 37 | (current [item#] 38 | (let [temp-val# (.current src-iterator#)] 39 | (casting/datatype->cast-fn ~src-dtype ~dest-dtype temp-val#)))))))) 40 | 41 | 42 | (def marshal-iterator-table (casting/make-marshalling-item-table 43 | make-marshal-iterator-macro)) 44 | 45 | 46 | (defn make-marshal-iterator 47 | [item datatype unchecked?] 48 | (let [item-dtype (dtype-proto/get-datatype item)] 49 | (if (= datatype item-dtype) 50 | item 51 | (let [iter-fn (get marshal-iterator-table [(casting/safe-flatten item-dtype) 52 | (casting/safe-flatten datatype)])] 53 | (iter-fn item datatype unchecked?))))) 54 | 55 | 56 | (extend-type Iterable 57 | dtype-proto/PToIterable 58 | (convertible-to-iterable? [item] true) 59 | (->iterable [item {:keys [datatype unchecked?] :as options}] 60 | (if-let [src-reader (dtype-proto/as-reader item options)] 61 | src-reader 62 | (reify 63 | Iterable 64 | (iterator [iter-item] 65 | (-> (IteratorObjectIter. (.iterator item) :object) 66 | (make-marshal-iterator datatype unchecked?))) 67 | dtype-proto/PDatatype 68 | (get-datatype [item] datatype))))) 69 | 70 | 71 | (defn ->iterable 72 | [item] 73 | (if (= :scalar (argtypes/arg->arg-type item)) 74 | (iter-const/make-const-iterable item (dtype-proto/get-datatype item)) 75 | item)) 76 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/jna.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.jna 2 | (:require [tech.v2.datatype.base :as dtype-base] 3 | [tech.v2.datatype.nio-buffer] 4 | [tech.v2.datatype.typed-buffer :as typed-buf] 5 | [tech.v2.datatype.protocols :as dtype-proto] 6 | [tech.v2.datatype.casting :as casting] 7 | [tech.jna :as jna] 8 | [tech.resource :as resource]) 9 | (:import [com.sun.jna Pointer Native Function NativeLibrary] 10 | [com.sun.jna.ptr PointerByReference] 11 | [java.nio ByteBuffer Buffer])) 12 | 13 | 14 | (set! *warn-on-reflection* true) 15 | (set! *unchecked-math* :warn-on-boxed) 16 | 17 | 18 | (defn offset-pointer 19 | ^Pointer [^Pointer ptr, ^long offset] 20 | (Pointer. (+ offset (Pointer/nativeValue ptr)))) 21 | 22 | 23 | (defn make-jna-pointer 24 | "Use with care..." 25 | ^Pointer [^long address] 26 | (Pointer. address)) 27 | 28 | 29 | (defn pointer->address 30 | ^long [^Pointer ptr] 31 | (Pointer/nativeValue ptr)) 32 | 33 | 34 | (defn typed-pointer->ptr 35 | ^Pointer [typed-pointer] 36 | (jna/->ptr-backing-store typed-pointer)) 37 | 38 | 39 | (defn pointer->nio-buffer 40 | [^Pointer ptr datatype byte-len] 41 | (let [buffer (.getByteBuffer ptr 0 byte-len)] 42 | (case datatype 43 | :int8 buffer 44 | :int16 (.asShortBuffer buffer) 45 | :int32 (.asIntBuffer buffer) 46 | :int64 (.asLongBuffer buffer) 47 | :float32 (.asFloatBuffer buffer) 48 | :float64 (.asDoubleBuffer buffer)))) 49 | 50 | 51 | (defn unsafe-ptr->typed-pointer 52 | "This is fairly unsafe. The byte-len and datatype must match 53 | the actual pointer data. But the GC can link the data to the 54 | resultant buffer." 55 | [^Pointer data ^long byte-len datatype] 56 | (let [nio-buf (pointer->nio-buffer data 57 | (casting/datatype->host-datatype datatype) 58 | byte-len)] 59 | (if (= datatype (casting/datatype->host-datatype datatype)) 60 | nio-buf 61 | (typed-buf/->TypedBuffer datatype nio-buf)))) 62 | 63 | 64 | (defn unsafe-address->typed-pointer 65 | "This is really unsafe. The GC won't be able to link Whatever produced the address 66 | to the produced pointer." 67 | [^long address ^long byte-len datatype] 68 | (unsafe-ptr->typed-pointer (make-jna-pointer address) byte-len datatype)) 69 | 70 | 71 | (defn unsafe-free-ptr 72 | [^Pointer ptr] 73 | (Native/free (Pointer/nativeValue ptr))) 74 | 75 | 76 | (defn make-typed-pointer 77 | "Make a typed pointer. Aside from the usual option :unchecked?, there is a new option 78 | :untracked? which means to explicitly avoid using the gc tracking system 79 | to track this pointer." 80 | [datatype elem-count-or-seq & [options]] 81 | (let [n-elems (if (number? elem-count-or-seq) 82 | (long elem-count-or-seq) 83 | (dtype-base/ecount elem-count-or-seq)) 84 | byte-len (* n-elems (dtype-base/datatype->byte-size datatype)) 85 | data (Native/malloc byte-len) 86 | retval (unsafe-address->typed-pointer data byte-len datatype)] 87 | (if-not (number? elem-count-or-seq) 88 | (dtype-proto/copy-raw->item! elem-count-or-seq retval 0 options) 89 | (when-not (:skip-init? options) 90 | (dtype-proto/set-constant! retval 0 0 n-elems))) 91 | ;;This will be freed if either the resource context is released *or* the return 92 | ;;value goes out of scope. In For some use-cases, the returned item should be 93 | ;;untracked and the callers will assume responsibility for freeing the data. 94 | (when-not (:untracked? options) 95 | (resource/track retval #(Native/free data) [:gc])) 96 | 97 | retval)) 98 | 99 | 100 | (defmethod dtype-proto/make-container :native-buffer 101 | [_ datatype elem-count options] 102 | (make-typed-pointer datatype elem-count options)) 103 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/mutable/iterable_to_list.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.mutable.iterable-to-list 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast])) 5 | 6 | 7 | (defmacro make-iter->list-table 8 | [] 9 | `(->> [~@(for [dtype casting/base-marshal-types] 10 | [dtype `(fn [iter# output# unchecked?#] 11 | (let [iter# (typecast/datatype->iter ~dtype iter# unchecked?#) 12 | output# (or output# 13 | (dtype-proto/make-container 14 | :list ~dtype 0)) 15 | mutable# (typecast/datatype->mutable ~dtype output# 16 | unchecked?#)] 17 | (while (.hasNext iter#) 18 | (.append mutable# (typecast/datatype->iter-next-fn 19 | ~dtype iter#))) 20 | output#))])] 21 | (into {}))) 22 | 23 | 24 | (def iter->list-table (make-iter->list-table)) 25 | 26 | 27 | (defn iterable->list 28 | [src-iterable dst-list {:keys [datatype unchecked?]}] 29 | (let [datatype (or datatype (dtype-proto/get-datatype src-iterable)) 30 | dst-list (or dst-list (dtype-proto/make-container :list datatype 0 {})) 31 | iter-fn (get iter->list-table (casting/safe-flatten datatype))] 32 | (iter-fn src-iterable dst-list unchecked?))) 33 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/object_datatypes.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.object-datatypes 2 | (:require [tech.v2.datatype.array :as dtype-ary] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.datatype.protocols :as dtype-proto]) 5 | (:import [clojure.lang Keyword Symbol] 6 | [java.util UUID])) 7 | 8 | 9 | 10 | (defmacro add-object-datatype 11 | [obj-cls datatype constructor] 12 | (let [ary-type (type (make-array (resolve obj-cls) 0))] 13 | `(do 14 | (casting/add-extended-datatype! ~datatype 15 | #(or (nil? %) 16 | (instance? ~obj-cls %)) 17 | ~constructor) 18 | ;;Ensure the scalar pathway works 19 | (clojure.core/extend 20 | ~obj-cls 21 | dtype-proto/PDatatype 22 | {:get-datatype (constantly ~datatype)}) 23 | (dtype-ary/extend-object-array-type ~ary-type) 24 | (dtype-ary/add-object-array-datatype-override! ~obj-cls ~datatype) 25 | (dtype-ary/add-array-constructor! 26 | ~datatype 27 | #(dtype-ary/make-object-array-of-type ~obj-cls %1 28 | (merge {:constructor ~constructor} 29 | %2)))))) 30 | 31 | (defn- construct-object 32 | ([] nil) 33 | ([item] item)) 34 | 35 | (defn- construct-keyword 36 | ([] nil) 37 | ([item] (keyword item))) 38 | 39 | (defn- construct-symbol 40 | ([] nil) 41 | ([item] (symbol item))) 42 | 43 | (defn- construct-uuid 44 | ([] nil) 45 | ([item] 46 | (cond 47 | (instance? UUID item) item 48 | (string? item) (UUID/fromString item) 49 | (nil? item) item 50 | :else 51 | (throw (Exception. (format "Unable to construct UUID from %s" item)))))) 52 | 53 | 54 | (add-object-datatype Object :object construct-object) 55 | (add-object-datatype String :string str) 56 | (add-object-datatype Keyword :keyword construct-keyword) 57 | (add-object-datatype Symbol :symbol construct-symbol) 58 | (add-object-datatype UUID :uuid construct-uuid) 59 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/pprint.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.pprint 2 | (:require [tech.v2.datatype.protocols :as dtype-proto])) 3 | 4 | 5 | ;; pretty-printing utilities for matrices 6 | (def ^:dynamic *number-format* "%.4G") 7 | 8 | 9 | (defn format-num [x] 10 | (if (integer? x) 11 | (str x) 12 | (format *number-format* (double x)))) 13 | 14 | 15 | (defn format-object 16 | [x] 17 | (if (number? x) 18 | (format-num x) 19 | (str x))) 20 | 21 | 22 | (defmulti reader-converter 23 | "Given a item that is of a datatype that is unprintable or that prints incorrectly 24 | return a new reader of a datatype that will print correctly (or just a reader of 25 | strings is fine). This is sometimes called for iterables also." 26 | (fn [item] 27 | (when item 28 | (dtype-proto/get-datatype item)))) 29 | 30 | 31 | (defmethod reader-converter :default 32 | [item] 33 | item) 34 | 35 | 36 | (defn print-reader-data 37 | [rdr & {:keys [formatter] 38 | :or {formatter format-object}}] 39 | (let [rdr (reader-converter rdr) 40 | ^StringBuilder builder 41 | (->> (dtype-proto/->reader rdr {}) 42 | (reduce (fn [^StringBuilder builder val] 43 | (.append builder 44 | (formatter val)) 45 | (.append builder ", ")) 46 | (StringBuilder.)))] 47 | (.toString builder))) 48 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/primitive.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.primitive 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.array :as dtype-ary]) 4 | (:import [tech.v2.datatype ObjectReader])) 5 | 6 | 7 | 8 | (defmacro implement-scalar-primitive 9 | [cls datatype] 10 | `(clojure.core/extend 11 | ~cls 12 | dtype-proto/PDatatype 13 | {:get-datatype (fn [item#] ~datatype)} 14 | dtype-proto/PCountable 15 | {:ecount (fn [item#] 1)} 16 | dtype-proto/PToReader 17 | ;;Reader conversion of primitives is inefficient so we allow it 18 | ;;but do not advertise it 19 | {:convertible-to-reader? (constantly false) 20 | :->reader (fn [item# options#] 21 | (-> (dtype-ary/make-array-of-type ~datatype [item#] true) 22 | (dtype-proto/->reader options#)))})) 23 | 24 | 25 | (implement-scalar-primitive Byte :int8) 26 | (implement-scalar-primitive Short :int16) 27 | (implement-scalar-primitive Integer :int32) 28 | (implement-scalar-primitive Long :int64) 29 | (implement-scalar-primitive Float :float32) 30 | (implement-scalar-primitive Double :float64) 31 | (implement-scalar-primitive Boolean :boolean) 32 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/readers/const.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.readers.const 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.typecast :as typecast] 4 | [tech.v2.datatype.protocols :as dtype-proto] 5 | [tech.v2.datatype.monotonic-range :as dtype-range])) 6 | 7 | 8 | (set! *warn-on-reflection* true) 9 | 10 | 11 | (defmacro make-single-elem-range 12 | [datatype elem] 13 | (when (casting/integer-type? datatype) 14 | `(dtype-range/make-range 15 | (long ~elem) 16 | (unchecked-inc (long ~elem))))) 17 | 18 | 19 | (defmacro make-const-reader-macro 20 | [datatype] 21 | `(fn [item# num-elems# datatype#] 22 | (let [num-elems# (long (or num-elems# Long/MAX_VALUE)) 23 | item# (casting/datatype->cast-fn 24 | :unknown ~datatype item#)] 25 | (reify 26 | ~(typecast/datatype->reader-type datatype) 27 | (getDatatype [reader#] datatype#) 28 | (lsize [reader#] num-elems#) 29 | (read [reader# idx#] item#) 30 | dtype-proto/PConstantTimeMinMax 31 | (has-constant-time-min-max? [this#] true) 32 | (constant-time-min [this#] item#) 33 | (constant-time-max [this#] item#) 34 | dtype-proto/PRangeConvertible 35 | (convertible-to-range? [this#] 36 | (and (== 1 num-elems#) 37 | (casting/integer-type? ~datatype))) 38 | (->range [this# options#] 39 | (when (== 1 num-elems#) 40 | (make-single-elem-range ~datatype item#))))))) 41 | 42 | 43 | (def const-reader-table (casting/make-base-datatype-table 44 | make-const-reader-macro)) 45 | 46 | 47 | (defn make-const-reader 48 | [item datatype & [num-elems]] 49 | (let [datatype (or datatype 50 | (when item (dtype-proto/get-datatype item)) 51 | :object)] 52 | (if-let [reader-fn (get const-reader-table (casting/flatten-datatype datatype))] 53 | (reader-fn item num-elems datatype) 54 | (throw (ex-info (format "Failed to find reader for datatype %s" datatype) {}))))) 55 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/readers/range.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.readers.range 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.typecast :as typecast])) 4 | 5 | 6 | (defmacro make-range-reader 7 | [datatype] 8 | (when-not (casting/numeric-type? datatype) 9 | (throw (ex-info (format "Datatype (%s) is not a numeric type" 10 | datatype) {}))) 11 | `(fn [start# end# increment#] 12 | (let [start# (casting/datatype->cast-fn :unknown ~datatype start#) 13 | end# (casting/datatype->cast-fn :unknown ~datatype end#) 14 | increment# (casting/datatype->cast-fn :unkown ~datatype increment#) 15 | n-elems# (Math/round (double (/ (- end# start#) 16 | increment#)))] 17 | 18 | (reify ~(typecast/datatype->reader-type datatype) 19 | (getDatatype [item#] ~datatype) 20 | (lsize [item#] n-elems#) 21 | (read [item# idx#] 22 | (when-not (< idx# n-elems#) 23 | (throw (ex-info (format "Index out of range: %s >= %s" idx# n-elems#)))) 24 | (casting/datatype->unchecked-cast-fn 25 | :unknown ~datatype 26 | (+ (* increment# idx#) 27 | start#))))))) 28 | 29 | 30 | (defmacro make-range-reader-table 31 | [] 32 | `(->> [~@(for [dtype (->> casting/base-marshal-types 33 | (filter casting/numeric-type?))] 34 | [dtype `(make-range-reader ~dtype)])] 35 | (into {}))) 36 | 37 | 38 | (def range-reader-table (make-range-reader-table)) 39 | 40 | (defn reader-range 41 | [datatype start end & [increment]] 42 | (if-let [reader-fn (get range-reader-table (casting/safe-flatten datatype))] 43 | (reader-fn start end (or increment 1)) 44 | (throw (ex-info (format "Failed to find reader fn for datatype %s" datatype) 45 | {})))) 46 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/readers/reverse.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.readers.reverse 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast] 5 | [tech.v2.datatype.reader :as reader])) 6 | 7 | 8 | (defmacro make-reverse-reader 9 | [datatype] 10 | `(fn [src-reader#] 11 | (let [src-reader# (typecast/datatype->reader ~datatype src-reader#) 12 | n-elems# (.lsize src-reader#) 13 | n-elems-m1# (- n-elems# 1) 14 | src-dtype# (dtype-proto/get-datatype src-reader#)] 15 | (reader/make-derived-reader ~datatype src-dtype# {:unchecked? true} src-reader# 16 | (.read src-reader# 17 | (- n-elems-m1# ~'idx)) 18 | dtype-proto/->reader 19 | n-elems#)))) 20 | 21 | 22 | (def reverse-reader-table (casting/make-base-datatype-table make-reverse-reader)) 23 | 24 | 25 | (defn reverse-reader 26 | ([src-reader {:keys [datatype]}] 27 | (let [datatype (or datatype (dtype-proto/get-datatype src-reader)) 28 | create-fn (get reverse-reader-table (casting/safe-flatten datatype))] 29 | (create-fn src-reader))) 30 | ([src-reader] 31 | (reverse-reader src-reader {}))) 32 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/readers/update.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.readers.update 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast] 5 | [tech.v2.datatype.bitmap :as bitmap]) 6 | (:import [java.util List Map] 7 | [org.roaringbitmap RoaringBitmap])) 8 | 9 | 10 | (set! *warn-on-reflection* true) 11 | (set! *unchecked-math* :warn-on-boxed) 12 | 13 | (defmacro make-reader-impl 14 | [datatype] 15 | `(fn [src-reader# update-map#] 16 | (let [dtype# (dtype-proto/get-datatype src-reader#) 17 | src-reader# (typecast/datatype->reader ~datatype src-reader#) 18 | n-elems# (.lsize src-reader#) 19 | ^Map update-map# (typecast/->java-map update-map#) 20 | ^RoaringBitmap bitmap# (if (dtype-proto/convertible-to-bitmap? update-map#) 21 | (dtype-proto/as-roaring-bitmap update-map#) 22 | (bitmap/->bitmap (keys update-map#)))] 23 | (reify ~(typecast/datatype->reader-type datatype) 24 | (getDatatype [item#] dtype#) 25 | (lsize [item#] n-elems#) 26 | (read [item# idx#] 27 | ;;The assumption here is that when there are very few sparse values the 28 | ;;bitmap lookup is both a lot faster and doesn't box the index. 29 | (if (.contains bitmap# idx#) 30 | (casting/datatype->unchecked-cast-fn :unknown ~datatype 31 | (.get update-map# idx#)) 32 | (.read src-reader# idx#))))))) 33 | 34 | 35 | (def reader-table (casting/make-base-datatype-table make-reader-impl)) 36 | 37 | 38 | (defn update-reader 39 | "Create a new reader that uses values from the update-map else uses values 40 | from the src-reader if the update map values do not exist." 41 | [src-reader update-map] 42 | (let [reader-fn (get reader-table (casting/safe-flatten 43 | (dtype-proto/get-datatype src-reader)))] 44 | (reader-fn src-reader update-map))) 45 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/record.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.record 2 | "Records that have a c-struct packed binary component 3 | and an object component. 4 | Record definitions are static datastructures that describe the 5 | layout of the binary componont in addition to the datatypes 6 | of the columns.") 7 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/shape.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.shape 2 | (:require [tech.v2.datatype.protocols :as dtype-proto] 3 | [tech.v2.datatype.typecast :as typecast]) 4 | (:import [java.util List] 5 | [java.nio Buffer 6 | ByteBuffer ShortBuffer IntBuffer LongBuffer 7 | FloatBuffer DoubleBuffer])) 8 | 9 | 10 | (set! *warn-on-reflection* true) 11 | 12 | 13 | (defn scalar? 14 | [item] 15 | (or (number? item) 16 | (string? item) 17 | (and 18 | (not (when (instance? Class (type item)) 19 | (.isArray ^Class (type item)))) 20 | (not (dtype-proto/convertible-to-iterable? item))))) 21 | 22 | 23 | (declare shape) 24 | 25 | 26 | (extend-type Object 27 | dtype-proto/PCountable 28 | (ecount [item] 29 | (cond 30 | (scalar? item) 31 | 0 32 | (and (instance? Class (type item)) 33 | (.isArray ^Class (type item))) 34 | (apply * (dtype-proto/shape item)) 35 | :else 36 | (count item))) 37 | dtype-proto/PShape 38 | (shape [item] 39 | (cond 40 | (scalar? item) 41 | nil 42 | (and (instance? Class (type item)) 43 | (.isArray ^Class (type item))) 44 | (let [n-elems (count item)] 45 | (-> (concat [n-elems] 46 | (when (> n-elems 0) 47 | (let [first-elem (first item)] 48 | (shape first-elem)))) 49 | vec)) 50 | :else 51 | [(dtype-proto/ecount item)]))) 52 | 53 | 54 | (extend-type List 55 | dtype-proto/PShape 56 | (shape [item] 57 | (if (seqable? (first item)) 58 | (->> (concat [(.size item)] 59 | (dtype-proto/shape (first item))) 60 | vec) 61 | [(.size item)]))) 62 | 63 | 64 | (extend-type Buffer 65 | dtype-proto/PCountable 66 | (ecount [item] 67 | (.remaining item))) 68 | 69 | 70 | 71 | (def array-class-names 72 | {"[B" :int8 73 | "[S" :int16 74 | "[I" :int32 75 | "[J" :int64 76 | "[F" :float32 77 | "[D" :float64 78 | "[Z" :boolean}) 79 | 80 | 81 | (defmacro extend-ary-cls 82 | [ary-name ary-dtype] 83 | `(clojure.core/extend 84 | (Class/forName ~ary-name) 85 | dtype-proto/PDatatype 86 | {:get-datatype (constantly ~ary-dtype)} 87 | dtype-proto/PCountable 88 | {:ecount (fn [item#] (alength (typecast/datatype->array-cast-fn ~ary-dtype item#)))})) 89 | 90 | 91 | (defmacro extend-all-array-classes 92 | [] 93 | `(do 94 | ~@(->> array-class-names 95 | (map (fn [[ary-name ary-dtype]] 96 | `(extend-ary-cls ~ary-name ~ary-dtype)))))) 97 | 98 | 99 | (extend-all-array-classes) 100 | 101 | 102 | (defmacro extend-buffer 103 | [datatype] 104 | `(clojure.core/extend 105 | (resolve (typecast/datatype->buffer-type ~datatype)) 106 | dtype-proto/PDatatype 107 | {:get-datatype (constantly ~datatype)} 108 | dtype-proto/PCountable 109 | {:ecount (fn [buffer#] (.remaining (typecast/datatype->buffer-cast-fn ~datatype buffer#)))})) 110 | 111 | 112 | (def buffer-extend-map (typecast/extend-host-numeric-datatypes extend-buffer)) 113 | 114 | 115 | (defn ecount 116 | "Type hinted element count" 117 | ^long [item] 118 | (if-not item 119 | 0 120 | (long (dtype-proto/ecount item)))) 121 | 122 | 123 | (defn shape 124 | [item] 125 | (if-not item 126 | nil 127 | (dtype-proto/shape item))) 128 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/sparse/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.sparse.protocols) 2 | 3 | 4 | (defprotocol PSparse 5 | (index-seq [item] 6 | "Return a sequence of records of 7 | :global-index 8 | :data-index") 9 | (sparse-value [item]) 10 | (sparse-ecount [item] 11 | "Number of sparse elements in this datastructure. Constant time query.") 12 | 13 | (readers [item] 14 | "Return a map containing 15 | { 16 | :indexes - int32 reader of indexes in global space. 17 | :data - (optional) reader of data values. 18 | }") 19 | (iterables [item] 20 | "Return a map containing: 21 | { 22 | :indexes int32 iterable of indexes in global space. 23 | :data iterable of values in global space. 24 | }")) 25 | 26 | 27 | (defn any-non-sparse? 28 | [item] 29 | (boolean (first (index-seq item)))) 30 | 31 | 32 | (defn index-reader 33 | [item] 34 | (:indexes (readers item))) 35 | 36 | 37 | (defn index-iterable 38 | [item] 39 | (:indexes (iterables item))) 40 | 41 | 42 | (defn data-reader 43 | [item] 44 | (:data (readers item))) 45 | 46 | 47 | (defn data-iterable 48 | [item] 49 | (:data (iterables item))) 50 | 51 | 52 | (defprotocol PToSparse 53 | (convertible-to-sparse? [item]) 54 | (->sparse [item])) 55 | 56 | 57 | (extend-type Object 58 | PToSparse 59 | (convertible-to-sparse? [item] false)) 60 | 61 | 62 | (defn sparse-convertible? 63 | [item] 64 | (convertible-to-sparse? item)) 65 | 66 | 67 | (defn as-sparse 68 | [item] 69 | (when (sparse-convertible? item) 70 | (->sparse item))) 71 | 72 | 73 | (defn safe-index-seq 74 | [item] 75 | (when-let [sparse-item (as-sparse item)] 76 | (index-seq sparse-item))) 77 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/type_resolution.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.type-resolution) 2 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/vector_of.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.vector-of 2 | "Bindings to the vectors produced by vector-of" 3 | (:require [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast] 5 | [tech.v2.datatype.casting :as casting]) 6 | (:import [clojure.core Vec ArrayManager])) 7 | 8 | (set! *warn-on-reflection* true) 9 | 10 | 11 | (defmacro ^:private specific-list-reader 12 | [datatype n-elems list-data] 13 | `(reify ~(typecast/datatype->reader-type datatype) 14 | (lsize [item#] ~n-elems) 15 | (read [item# idx#] 16 | (casting/datatype->unchecked-cast-fn 17 | :unknown 18 | ~datatype 19 | (.get ~list-data idx#))))) 20 | 21 | (defmacro ^:private list-reader 22 | ([datatype item] 23 | `(let [item# (typecast/object-list-cast ~item) 24 | n-elems# (long (.size item#))] 25 | (case ~datatype 26 | :boolean (specific-list-reader :boolean n-elems# item#) 27 | :int8 (specific-list-reader :int8 n-elems# item#) 28 | :int16 (specific-list-reader :int16 n-elems# item#) 29 | :int32 (specific-list-reader :int32 n-elems# item#) 30 | :int64 (specific-list-reader :int64 n-elems# item#) 31 | :float32 (specific-list-reader :float32 n-elems# item#) 32 | :float64 (specific-list-reader :float64 n-elems# item#)))) 33 | ([item] 34 | `(list-reader (dtype-proto/get-datatype ~item) ~item))) 35 | 36 | 37 | (extend-type Vec 38 | dtype-proto/PDatatype 39 | (get-datatype [item] 40 | (dtype-proto/get-datatype 41 | (.array ^ArrayManager (.am item) 1))) 42 | dtype-proto/PClone 43 | (clone [item] item) 44 | dtype-proto/PToReader 45 | (convertible-to-reader? [item] true) 46 | (->reader [item options] 47 | (let [item-dtype (dtype-proto/get-datatype item) 48 | new-reader (list-reader item-dtype item) 49 | final-dtype (or (:datatype options) item-dtype)] 50 | (if (= final-dtype item-dtype) 51 | new-reader 52 | (dtype-proto/->reader new-reader options)))) 53 | dtype-proto/PBuffer 54 | (sub-buffer [item offset length] 55 | (dtype-proto/sub-buffer (dtype-proto/->reader item nil) offset length))) 56 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/writers/concat.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.writers.concat 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.base :as dtype-base] 5 | [tech.v2.datatype.typecast :as typecast]) 6 | (:import [java.util List])) 7 | 8 | 9 | 10 | (defmacro make-concat-writer-impl 11 | [datatype] 12 | `(fn [datatype# concat-args#] 13 | (let [^List writer-args# (mapv #(dtype-proto/->writer % {:datatype datatype#}) 14 | concat-args#) 15 | total-size# (long (apply + (map #(dtype-base/ecount %) writer-args#)))] 16 | (reify 17 | dtype-proto/PDatatype 18 | (get-datatype [item#] datatype#) 19 | ~(typecast/datatype->writer-type datatype) 20 | (lsize [rdr#] total-size#) 21 | (write [rdr# idx# value#] 22 | (loop [rdr-idx# 0 23 | idx# idx#] 24 | (let [cur-rdr# (typecast/datatype->writer ~datatype 25 | (.get writer-args# rdr-idx#)) 26 | cur-lsize# (.lsize cur-rdr#)] 27 | (if (< idx# cur-lsize#) 28 | (.write cur-rdr# idx# value#) 29 | (recur (inc rdr-idx#) (- idx# cur-lsize#)))))))))) 30 | 31 | 32 | (def concat-writer-table (casting/make-base-datatype-table make-concat-writer-impl)) 33 | 34 | 35 | (defn concat-writers 36 | ([options writers] 37 | (let [datatype (or (:datatype options) (dtype-base/get-datatype (first writers))) 38 | writer-fn (get concat-writer-table (casting/safe-flatten datatype))] 39 | (writer-fn datatype writers))) 40 | ([writers] 41 | (concat-writers {} writers))) 42 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/writers/indexed.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.writers.indexed 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast])) 5 | 6 | 7 | (declare make-indexed-writer) 8 | 9 | 10 | 11 | (defmacro make-indexed-writer-impl 12 | [datatype] 13 | `(fn [indexes# values# unchecked?#] 14 | (let [idx-reader# (typecast/datatype->reader :int32 indexes# true) 15 | values# (typecast/datatype->writer ~datatype values# unchecked?#) 16 | writer-dtype# (dtype-proto/get-datatype values#) 17 | n-elems# (.lsize idx-reader#)] 18 | (reify 19 | ~(typecast/datatype->writer-type datatype) 20 | (getDatatype [writer#] writer-dtype#) 21 | (lsize [writer#] n-elems#) 22 | (write [writer# idx# value#] 23 | (.write values# (.read idx-reader# idx#) value#)) 24 | dtype-proto/PToBackingStore 25 | (->backing-store-seq [writer#] 26 | (dtype-proto/->backing-store-seq values#)) 27 | dtype-proto/PBuffer 28 | (sub-buffer [writer# offset# length#] 29 | (-> (dtype-proto/sub-buffer indexes# offset# length#) 30 | (make-indexed-writer values# {:unchecked? unchecked?#}))))))) 31 | 32 | 33 | (def indexed-writer-creators (casting/make-base-datatype-table 34 | make-indexed-writer-impl)) 35 | 36 | 37 | (defn make-indexed-writer 38 | [indexes values {:keys [datatype unchecked?]}] 39 | (let [datatype (or datatype (dtype-proto/get-datatype values)) 40 | writer-fn (get indexed-writer-creators (casting/safe-flatten datatype))] 41 | (writer-fn indexes values unchecked?))) 42 | -------------------------------------------------------------------------------- /src/tech/v2/datatype/writers/iterable_to_writer.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.writers.iterable-to-writer 2 | (:require [tech.v2.datatype.casting :as casting] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.typecast :as typecast])) 5 | 6 | 7 | (defmacro make-iterable-write-fn 8 | [datatype] 9 | `(fn [dst# src# unchecked?#] 10 | (let [dst-writer# (typecast/datatype->writer ~datatype dst# true) 11 | src-iter# (typecast/datatype->iter ~datatype src# unchecked?#)] 12 | (loop [idx# (int 0)] 13 | (if (.hasNext src-iter#) 14 | (do 15 | (.write dst-writer# idx# (typecast/datatype->iter-next-fn 16 | ~datatype src-iter#)) 17 | (recur (unchecked-inc idx#))) 18 | dst#))))) 19 | 20 | 21 | (def iterable-writer-table (casting/make-base-datatype-table 22 | make-iterable-write-fn)) 23 | 24 | 25 | (defn iterable->writer 26 | [dst-writer src-iterable & {:keys [datatype unchecked?]}] 27 | (let [datatype (or datatype (dtype-proto/get-datatype src-iterable)) 28 | writer-fn (get iterable-writer-table (casting/safe-flatten datatype))] 29 | (writer-fn dst-writer src-iterable unchecked?))) 30 | -------------------------------------------------------------------------------- /src/tech/v2/libs/blas.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.libs.blas 2 | (:require [tech.jna :as jna] 3 | [tech.v2.datatype.typecast :refer [ensure-ptr-like]] 4 | [clojure.tools.logging :as log])) 5 | 6 | 7 | (def ^:dynamic *system-blas-lib-name* "blas") 8 | 9 | 10 | (defn ^:private system-blas-lib 11 | [] 12 | (try 13 | (jna/load-library *system-blas-lib-name*) 14 | (catch Throwable e 15 | (log/warnf "Failed to load native blas: %s" (.getMessage e)) 16 | nil))) 17 | 18 | (defn has-blas? 19 | [] 20 | (boolean (system-blas-lib))) 21 | 22 | 23 | (defn openblas? 24 | [] 25 | (boolean 26 | (when (system-blas-lib) 27 | (jna/find-function "openblas_get_num_threads" *system-blas-lib-name*)))) 28 | 29 | 30 | (jna/def-jna-fn *system-blas-lib-name* openblas_get_num_threads 31 | "Get number of openblas threads" 32 | Integer) 33 | 34 | 35 | (jna/def-jna-fn *system-blas-lib-name* openblas_set_num_threads 36 | "Set number of openblas threads" 37 | nil 38 | [num-threads int]) 39 | 40 | 41 | ;; typedef enum CBLAS_ORDER {CblasRowMajor=101, CblasColMajor=102} CBLAS_ORDER; 42 | ;; typedef enum CBLAS_TRANSPOSE {CblasNoTrans=111, CblasTrans=112, CblasConjTrans=113, CblasConjNoTrans=114} CBLAS_TRANSPOSE; 43 | ;; typedef enum CBLAS_UPLO {CblasUpper=121, CblasLower=122} CBLAS_UPLO; 44 | ;; typedef enum CBLAS_DIAG {CblasNonUnit=131, CblasUnit=132} CBLAS_DIAG; 45 | ;; typedef enum CBLAS_SIDE {CblasLeft=141, CblasRight=142} CBLAS_SIDE; 46 | 47 | (def enums 48 | {:row-major 101 :column-major 102 49 | :no-transpose 111 :transpose 112 :conjugate-transpose 113 :conjugate-no-transpose 114 50 | :upper 121 :lower 122 51 | :non-unit 131 :unit 132 52 | :left 141 :right 142}) 53 | 54 | (defn enum-value 55 | [enum-name] 56 | (int 57 | (if-let [retval (get enums enum-name)] 58 | retval 59 | (throw (ex-info "Failed to find enum:" 60 | {:enum-name enum-name}))))) 61 | 62 | ;; void cblas_sgemm(OPENBLAS_CONST enum CBLAS_ORDER Order, 63 | ;; OPENBLAS_CONST enum CBLAS_TRANSPOSE TransA, 64 | ;; OPENBLAS_CONST enum CBLAS_TRANSPOSE TransB, 65 | ;; OPENBLAS_CONST blasint M, 66 | ;; OPENBLAS_CONST blasint N, 67 | ;; OPENBLAS_CONST blasint K, 68 | ;; OPENBLAS_CONST float alpha, 69 | ;; OPENBLAS_CONST float *A, 70 | ;; OPENBLAS_CONST blasint lda, 71 | ;; OPENBLAS_CONST float *B, 72 | ;; OPENBLAS_CONST blasint ldb, 73 | ;; OPENBLAS_CONST float beta, 74 | ;; float *C, OPENBLAS_CONST blasint ldc); 75 | 76 | 77 | (defn- ensure-blas 78 | [] 79 | (when-not (has-blas?) 80 | (throw (ex-info "System blas is unavailable." {})))) 81 | 82 | 83 | (defn bool->blas-transpose 84 | [trans?] 85 | (int 86 | (enum-value (if trans? :transpose :no-transpose)))) 87 | 88 | 89 | (jna/def-jna-fn *system-blas-lib-name* cblas_sgemm 90 | "float32 gemm" 91 | nil 92 | [order enum-value] 93 | [trans-a? bool->blas-transpose] 94 | [trans-b? bool->blas-transpose] 95 | [M int] 96 | [N int] 97 | [K int] 98 | [alpha float] 99 | [A ensure-ptr-like] 100 | [lda int] 101 | [B ensure-ptr-like] 102 | [ldb int] 103 | [beta float] 104 | [C ensure-ptr-like] 105 | [ldc int]) 106 | 107 | 108 | (jna/def-jna-fn *system-blas-lib-name* cblas_dgemm 109 | "float64 gemm" 110 | nil 111 | [order enum-value] 112 | [trans-a? bool->blas-transpose] 113 | [trans-b? bool->blas-transpose] 114 | [M int] 115 | [N int] 116 | [K int] 117 | [alpha double] 118 | [A ensure-ptr-like] 119 | [lda int] 120 | [B ensure-ptr-like] 121 | [ldb int] 122 | [beta double] 123 | [C ensure-ptr-like] 124 | [ldc int]) 125 | 126 | 127 | (jna/def-jna-fn *system-blas-lib-name* cblas_sgemv 128 | "float32 gemv" 129 | nil 130 | [order enum-value] 131 | [trans-a? bool->blas-transpose] 132 | [M int] 133 | [N int] 134 | [alpha float] 135 | [A ensure-ptr-like] 136 | [lda int] 137 | [x ensure-ptr-like] 138 | [incx int] 139 | [beta float] 140 | [y ensure-ptr-like] 141 | [incy int]) 142 | 143 | 144 | (jna/def-jna-fn *system-blas-lib-name* cblas_dgemv 145 | "float64 gemv" 146 | nil 147 | [order enum-value] 148 | [trans-a? bool->blas-transpose] 149 | [M int] 150 | [N int] 151 | [alpha double] 152 | [A ensure-ptr-like] 153 | [lda int] 154 | [x ensure-ptr-like] 155 | [incx int] 156 | [beta double] 157 | [y ensure-ptr-like] 158 | [incy int]) 159 | -------------------------------------------------------------------------------- /src/tech/v2/tensor.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.tensor.impl :as impl] 4 | [tech.v2.tensor.dimensions :as dims] 5 | [tech.v2.tensor.protocols :as tens-proto] 6 | ;;Ensure printing things works. 7 | [tech.v2.tensor.pprint] 8 | [tech.v2.datatype.functional.impl :as func-impl] 9 | [tech.v2.datatype.binary-op :as binary-op] 10 | ;;registers a highly optimized copy function 11 | [tech.v2.tensor.tensor-copy])) 12 | 13 | 14 | (func-impl/export-symbols tech.v2.tensor.impl 15 | ->tensor 16 | new-tensor 17 | clone 18 | reshape 19 | select 20 | transpose 21 | broadcast 22 | rotate 23 | slice 24 | tensor? 25 | ensure-tensor 26 | ensure-buffer-descriptor 27 | buffer-descriptor->tensor 28 | tensor-force 29 | tensor-container-type 30 | tensor-buffer-type 31 | mutable? 32 | matrix-matrix-dispatch 33 | ->jvm) 34 | 35 | 36 | (defn as-buffer-descriptor 37 | "Convenience function. Also present in tech.v2.datatype, just here because 38 | ensure-buffer-descriptor is in this namespace." 39 | [tens] 40 | (dtype/as-buffer-descriptor tens)) 41 | 42 | 43 | (defn tensor->dimensions 44 | [tens] 45 | (tens-proto/dimensions tens)) 46 | 47 | 48 | (defn tensor->buffer 49 | [tens] 50 | (tens-proto/buffer tens)) 51 | 52 | 53 | (defn as-datatype 54 | [tens new-dtype] 55 | (impl/construct-tensor 56 | (dtype/->reader tens new-dtype) 57 | (dims/dimensions (dtype/shape tens)))) 58 | 59 | 60 | (defn dimensions-dense? 61 | [tensor] 62 | (dims/dense? (tensor->dimensions tensor))) 63 | 64 | 65 | (defn rows 66 | [tens] 67 | (let [[n-rows _] (dtype/shape tens)] 68 | (->> (range n-rows) 69 | (map #(select tens % :all))))) 70 | 71 | 72 | (defn columns 73 | [tens] 74 | (let [[_ n-cols] (dtype/shape tens)] 75 | (->> (range n-cols) 76 | (map #(select tens :all %))))) 77 | 78 | 79 | 80 | (defn matrix-multiply 81 | "lhs - 2 dimensional tensor. 82 | rhs - Either 2 dimensional tensor or 1 dimensional vector. 83 | alpha - multiply result by alpha. 84 | reduction operators - *,+" 85 | [lhs rhs & [alpha]] 86 | (when-not (= (dtype/get-datatype lhs) 87 | (dtype/get-datatype rhs)) 88 | (throw (ex-info (format "Argument datatype mismatch: %s vs %s" 89 | (name (dtype/get-datatype lhs)) 90 | (name (dtype/get-datatype rhs))) 91 | {}))) 92 | (impl/matrix-matrix-dispatch alpha lhs rhs 93 | (:* binary-op/builtin-binary-ops) 94 | (:+ binary-op/builtin-binary-ops) 95 | {})) 96 | -------------------------------------------------------------------------------- /src/tech/v2/tensor/color_gradients/build_gradients.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.color-gradients.build-gradients 2 | (:require [clojure.java.io :as io] 3 | [clojure.java.shell :as sh] 4 | [clojure.edn :as edn] 5 | [tech.libs.buffered-image :as bufimg] 6 | [tech.v2.tensor :as dtt] 7 | [tech.v2.datatype :as dtype]) 8 | (:import [java.net URL])) 9 | 10 | 11 | 12 | (def gradient-root-url (:base-url (edn/read-string 13 | (slurp "build-data/mathematica-gradients.edn")))) 14 | 15 | (def gradient-map (edn/read-string (slurp "build-data/parsed-gradients.edn"))) 16 | 17 | 18 | (defn image->gradient-line 19 | [img-partial-path] 20 | (let [img (bufimg/load (URL. (str gradient-root-url img-partial-path))) 21 | _ (bufimg/save! img "PNG" "temp.png") 22 | sh-val (sh/sh "mogrify" "-trim" "temp.png") 23 | _ (when-not (= 0 (long (:exit sh-val))) 24 | (throw (Exception. (str (:err sh-val))))) 25 | img (bufimg/load "temp.png") 26 | ;;Ensure we know the format of the image. 27 | dst-img (bufimg/new-image (.getWidth img) (.getHeight img) :byte-bgr) 28 | _ (bufimg/draw-image! img dst-img)] 29 | (-> dst-img 30 | (bufimg/as-ubyte-tensor) 31 | (dtt/select 10 :all :all)))) 32 | 33 | 34 | (defn build-gradient-datastructure 35 | [] 36 | (let [base-structure 37 | (->> gradient-map 38 | (map-indexed vector) 39 | (map (fn [[idx [img-name img-path]]] 40 | (let [gradient-line (image->gradient-line img-path)] 41 | [img-name {:tensor-index idx 42 | :gradient-shape (dtype/shape gradient-line) 43 | :gradient gradient-line}]))) 44 | (into {})) 45 | full-gradient-tensor-shape [(count base-structure) 46 | (->> (vals base-structure) 47 | (map (comp first :gradient-shape)) 48 | (apply max)) 49 | 3] 50 | final-img (bufimg/new-image (second full-gradient-tensor-shape) 51 | (first full-gradient-tensor-shape) 52 | :byte-bgr) 53 | final-tensor (dtt/ensure-tensor final-img) 54 | _ (doseq [{:keys [tensor-index gradient-shape gradient]} 55 | (vals base-structure)] 56 | (dtype/copy! gradient (dtt/select final-tensor 57 | tensor-index 58 | (range 0 (first gradient-shape)) 59 | :all))) 60 | final-gradient-structure (->> base-structure 61 | (map (fn [[k v]] 62 | [k (dissoc v :gradient)])) 63 | (into {}))] 64 | (spit "resources/gradients.edn" (pr-str final-gradient-structure)) 65 | (bufimg/save! final-img "PNG" "resources/gradients.png") 66 | :ok)) 67 | -------------------------------------------------------------------------------- /src/tech/v2/tensor/dimensions/select.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.dimensions.select 2 | "Selecting subsets from a larger set of dimensions leads to its own algebra." 3 | (:require [tech.v2.datatype.index-algebra :as idx-alg] 4 | [tech.v2.datatype.protocols :as dtype-proto]) 5 | (:import [tech.v2.datatype LongReader] 6 | [java.util ArrayList List] 7 | [it.unimi.dsi.fastutil.longs LongArrayList])) 8 | 9 | (set! *unchecked-math* :warn-on-boxed) 10 | (set! *warn-on-reflection* true) 11 | 12 | 13 | (defn- expand-dimension 14 | ^LongReader [dim] 15 | (idx-alg/dimension->reader dim)) 16 | 17 | 18 | (defn apply-select-arg-to-dimension 19 | "Given a dimension and select argument, create a new dimension with 20 | the selection applied." 21 | [dim select-arg] 22 | (if (= select-arg :all) 23 | dim 24 | (idx-alg/select dim select-arg))) 25 | 26 | 27 | (defn select 28 | "Apply a select seq to a dimension. 29 | Return new shape, stride, offset array 30 | along with new buffer offset and if it can be calculated a new 31 | buffer length." 32 | [select-seq shape-vec stride-vec] 33 | (let [^List select-vec (vec select-seq) 34 | ^List shape-vec shape-vec 35 | ^List stride-vec stride-vec 36 | result-shape (ArrayList.) 37 | result-stride (LongArrayList.) 38 | n-elems (.size select-vec)] 39 | (when-not (== (.size select-vec) 40 | (.size shape-vec)) 41 | (throw (Exception. "Shape,select vecs do not match"))) 42 | (loop [idx 0 43 | buffer-offset 0] 44 | (if (< idx n-elems) 45 | (let [new-shape-val (apply-select-arg-to-dimension 46 | (.get shape-vec idx) 47 | (.get select-vec idx)) 48 | ;;scalar means to evaluate the result immediately and do not 49 | ;;add to result dims. 50 | new-shape-scalar? (:select-scalar? (meta new-shape-val)) 51 | stride-val (long (.get stride-vec idx)) 52 | ;;This resets ranges to start at zero at the cost of causing our host 53 | ;;to offset the buffer. In the case where the range is incrementing 54 | ;;by 1, however, this leads to a native mapping to underlying buffers. 55 | [cmin new-shape-val] 56 | (if (and (not (number? new-shape-val)) 57 | (dtype-proto/convertible-to-range? new-shape-val)) 58 | (let [cmin (long (dtype-proto/constant-time-min new-shape-val))] 59 | [cmin (-> (dtype-proto/range-offset new-shape-val (- cmin)) 60 | (idx-alg/simplify-range->direct))]) 61 | [nil new-shape-val]) 62 | buffer-offset (long (if cmin 63 | (+ buffer-offset 64 | (* stride-val (long cmin))) 65 | buffer-offset))] 66 | (when-not new-shape-scalar? 67 | (.add result-shape new-shape-val) 68 | (.add result-stride stride-val)) 69 | (recur (unchecked-inc idx) buffer-offset)) 70 | {:shape result-shape 71 | :strides result-stride 72 | :offset buffer-offset})))) 73 | -------------------------------------------------------------------------------- /src/tech/v2/tensor/dimensions/shape.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.dimensions.shape 2 | "A shape vector entry can be a number of things. We want to be precise 3 | with handling them and abstract that handling so new things have a clear 4 | path." 5 | (:require [tech.v2.tensor.utils 6 | :refer [when-not-error reversev map-reversev]] 7 | [tech.v2.datatype :as dtype] 8 | [tech.v2.datatype.index-algebra :as idx-alg] 9 | [tech.v2.datatype.bitmap :as bitmap] 10 | [tech.v2.datatype.readers.range :as rdr-range] 11 | [tech.v2.datatype.readers.indexed :as indexed-rdr] 12 | [tech.v2.datatype.typecast :as typecast] 13 | [tech.v2.datatype.monotonic-range :as dtype-range] 14 | [tech.v2.datatype.functional :as dfn] 15 | [tech.v2.datatype.protocols :as dtype-proto] 16 | [tech.v2.datatype.casting :as casting]) 17 | (:import [tech.v2.datatype LongReader] 18 | [java.util Map] 19 | [clojure.lang MapEntry])) 20 | 21 | 22 | (set! *unchecked-math* :warn-on-boxed) 23 | (set! *warn-on-reflection* true) 24 | 25 | 26 | (defn shape-entry->count 27 | "Return a vector of counts of each shape." 28 | ^long [shape-entry] 29 | (if (number? shape-entry) 30 | (long shape-entry) 31 | (dtype/ecount shape-entry))) 32 | 33 | 34 | (defn shape->count-vec 35 | ^longs [shape-vec] 36 | (mapv shape-entry->count shape-vec)) 37 | 38 | 39 | (defn direct-shape? 40 | [shape] 41 | (every? idx-alg/direct? shape)) 42 | 43 | 44 | (defn indirect-shape? 45 | [shape] 46 | (boolean (some (complement idx-alg/direct?) shape))) 47 | 48 | 49 | (defn ecount 50 | "Return the element count indicated by the dimension map" 51 | ^long [shape] 52 | (let [count-vec (shape->count-vec shape) 53 | n-items (count count-vec)] 54 | (loop [idx 0 55 | sum 1] 56 | (if (< idx n-items) 57 | (recur (unchecked-inc idx) 58 | (* (long (count-vec idx)) 59 | sum)) 60 | sum)))) 61 | 62 | 63 | (defn- ensure-direct 64 | [shape-seq] 65 | (when-not (direct-shape? shape-seq) 66 | (throw (ex-info "Index buffers not supported for this operation." {}))) 67 | shape-seq) 68 | 69 | 70 | (defn ->2d 71 | "Given dimensions, return new dimensions with the lowest (fastest-changing) dimension 72 | unchanged and the rest of the dimensions multiplied into the higher dimension." 73 | [shape] 74 | (when-not-error (seq shape) 75 | "Invalid shape in dimension map" 76 | {:shape shape}) 77 | (if (= 1 (count shape)) 78 | [1 (first shape)] 79 | [(apply * (ensure-direct (drop-last shape))) (last shape)])) 80 | 81 | 82 | (defn ->batch 83 | "Given dimensions, return new dimensions with the lowest (fastest-changing) dimension 84 | unchanged and the rest of the dimensions multiplied into the higher dimension." 85 | [shape] 86 | (when-not-error (seq shape) 87 | "Invalid shape in dimension map" 88 | {:shape shape}) 89 | (if (= 1 (count shape)) 90 | [1 (first shape)] 91 | [(first shape) (apply * (ensure-direct (drop 1 shape)))])) 92 | -------------------------------------------------------------------------------- /src/tech/v2/tensor/protocols.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.protocols 2 | (:require [tech.v2.datatype.protocols :as dtype-proto])) 3 | 4 | 5 | (defprotocol PTensor 6 | (is-tensor? [item]) 7 | (dimensions [item]) 8 | (buffer [item])) 9 | 10 | 11 | (defprotocol PTensorPrinter 12 | (print-tensor [item])) 13 | 14 | 15 | (defprotocol PToTensor 16 | (convertible-to-tensor? [item]) 17 | (convert-to-tensor [item])) 18 | 19 | 20 | (defn as-tensor 21 | [item] 22 | (when (convertible-to-tensor? item) 23 | (convert-to-tensor item))) 24 | 25 | 26 | (defprotocol PToTensorReader 27 | (convertible-to-tensor-reader? [item]) 28 | (->tensor-reader [item options])) 29 | 30 | 31 | (defprotocol PToTensorWriter 32 | (convertible-to-tensor-writer? [item]) 33 | (->tensor-writer [item options])) 34 | 35 | 36 | (extend-type Object 37 | PTensor 38 | (is-tensor? [item] false) 39 | PToTensorReader 40 | (convertible-to-tensor-reader? [item] false)) 41 | -------------------------------------------------------------------------------- /src/tech/v2/tensor/utils.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.utils) 2 | 3 | 4 | (defmacro when-not-error 5 | [expr error-msg extra-data] 6 | `(when-not ~expr 7 | (throw (ex-info ~error-msg ~extra-data)))) 8 | 9 | 10 | (defn reversev 11 | [item-seq] 12 | (if (vector? item-seq) 13 | (let [len (count item-seq) 14 | retval (transient [])] 15 | (loop [idx 0] 16 | (if (< idx len) 17 | (do 18 | (conj! retval (item-seq (- len idx 1))) 19 | (recur (inc idx))) 20 | (persistent! retval)))) 21 | (vec (reverse item-seq)))) 22 | 23 | 24 | (defn map-reversev 25 | [map-fn item-seq] 26 | (if (vector? item-seq) 27 | (let [len (count item-seq) 28 | retval (transient [])] 29 | (loop [idx 0] 30 | (if (< idx len) 31 | (do 32 | (conj! retval (map-fn (item-seq (- len idx 1)))) 33 | (recur (inc idx))) 34 | (persistent! retval)))) 35 | (vec (reverse item-seq)))) 36 | 37 | 38 | (defn all-combinations 39 | [item-seq] 40 | (let [item (first item-seq) 41 | rest-items (rest item-seq)] 42 | (if (seq rest-items) 43 | (concat (map vector (repeat item) rest-items) 44 | (lazy-seq (all-combinations rest-items))) 45 | nil))) 46 | -------------------------------------------------------------------------------- /test/data/test.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techascent/tech.datatype/8cc83d771d9621d580fd5d4d0625005bd7ab0e0c/test/data/test.jpg -------------------------------------------------------------------------------- /test/tech/compute/cpu/driver_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.cpu.driver-test 2 | (:require [tech.compute.cpu.driver :as cpu] 3 | [clojure.test :refer :all] 4 | [tech.compute.verify.utils :refer [def-all-dtype-test 5 | def-double-float-test] :as test-utils] 6 | [tech.compute.verify.driver :as verify-driver])) 7 | 8 | 9 | (use-fixtures :each test-utils/test-wrapper) 10 | 11 | 12 | (deftest simple-stream 13 | (verify-driver/simple-stream (cpu/driver) test-utils/*datatype*)) 14 | -------------------------------------------------------------------------------- /test/tech/compute/cpu/tensor_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.compute.cpu.tensor-test 2 | (:require [tech.compute.cpu.driver :as cpu] 3 | [tech.compute.verify.tensor :as verify-tens] 4 | [clojure.test :refer :all])) 5 | 6 | 7 | (deftest clone 8 | (verify-tens/clone (cpu/driver) :float64)) 9 | 10 | 11 | (deftest assign! 12 | (verify-tens/assign! (cpu/driver) :float64)) 13 | -------------------------------------------------------------------------------- /test/tech/libs/buffered_image_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.libs.buffered-image-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.functional :as dfn] 4 | [tech.v2.tensor :as dtt] 5 | [tech.v2.tensor.typecast :as tens-typecast] 6 | [tech.libs.buffered-image :as bufimg] 7 | [clojure.test :refer :all])) 8 | 9 | 10 | (deftest base-img-tensor-test 11 | (let [test-img (bufimg/new-image 4 4 :int-rgb) 12 | test-writer (tens-typecast/datatype->tensor-writer 13 | :uint8 14 | (bufimg/as-ubyte-tensor test-img))] 15 | (.write3d test-writer 2 2 2 255) 16 | (is (= [0 0 0 0 0 0 0 0 0 0 16711680 0 0 0 0 0] 17 | (dtype/->vector test-img))))) 18 | 19 | 20 | (deftest tensor-and-draw-image 21 | (let [test-img (bufimg/load "test/data/test.jpg") 22 | new-img (bufimg/new-image 512 512 :int-argb) 23 | _ (bufimg/draw-image! test-img new-img :dst-y-offset 128) 24 | tens-img (bufimg/new-image 512 512 :int-argb) 25 | 26 | copy-tens (dtt/select (bufimg/as-ubyte-tensor tens-img) 27 | (range 128 (+ 128 (.getHeight test-img))) 28 | :all 29 | [0 1 2]) 30 | alpha-tens (dtt/select (bufimg/as-ubyte-tensor tens-img) 31 | (range 128 (+ 128 (.getHeight test-img))) 32 | :all 33 | 3) 34 | _ (is (= (dtype/shape copy-tens) 35 | (dtype/shape test-img))) 36 | _ (dtype/copy! test-img copy-tens) 37 | _ (dtype/set-constant! alpha-tens 255)] 38 | (is (dfn/equals (mapv (comp long dfn/mean) [new-img tens-img]) 39 | [-5240767 -5240767])))) 40 | 41 | 42 | (deftest channel-order 43 | (let [test-img (bufimg/load "test/data/test.jpg") 44 | [height width _n-chan] (dtype/shape test-img) 45 | channel-fn 46 | (fn [img-fmt] 47 | (let [new-img (bufimg/new-image height width img-fmt) 48 | _ (bufimg/draw-image! test-img new-img) 49 | first-pix (-> (bufimg/as-ubyte-tensor new-img) 50 | (dtt/select 0 0 :all)) 51 | chan-map (bufimg/image-channel-map new-img)] 52 | (->> [:r :g :b :a] 53 | (mapv (fn [chan-name] 54 | (if (contains? chan-map chan-name) 55 | (dtype/get-value first-pix 56 | (get chan-map chan-name)) 57 | 255)))))) 58 | img-types [:byte-bgr :byte-abgr 59 | :int-argb :int-bgr :int-rgb]] 60 | (doseq [img-type img-types] 61 | (is (= [170 170 172 255] 62 | (channel-fn img-type)) 63 | (format "Format %s failed channel order test" img-type))))) 64 | 65 | 66 | (defn strided-tensor-copy-time-test 67 | [] 68 | (let [src-tens (-> (bufimg/new-image 2048 2048 :byte-abgr) 69 | (dtt/select (range 256 (* 2 256)) 70 | (range 256 (* 2 256)) 71 | :all)) 72 | dst-tens (dtt/ensure-tensor (bufimg/new-image 256 256 :byte-abgr))] 73 | ;; (dtype/copy! src-tens dst-tens) 74 | (time (dotimes [iter 100] 75 | (dtype/copy! src-tens dst-tens))) 76 | :ok 77 | )) 78 | -------------------------------------------------------------------------------- /test/tech/v2/apl/sparse_game_of_life.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.apl.sparse-game-of-life 2 | (:require [tech.v2.apl.game-of-life :as apl-gol] 3 | [tech.v2.tensor.impl :as tens-impl] 4 | [tech.v2.tensor :as tens] 5 | [tech.v2.datatype.functional :as dtype-fn] 6 | [tech.v2.datatype :as dtype] 7 | [clojure.test :refer :all])) 8 | 9 | 10 | (comment 11 | ;;Temporarily dropping most sparse support. 12 | (with-bindings {#'tens-impl/*container-type* :sparse} 13 | (def range-tens (-> (tens/reshape (vec (range 9)) [3 3]) 14 | ;;Have to set actual numeric datatype or the sparse value ends 15 | ;;up as 'nil' 16 | (tens/clone :datatype :int8))) 17 | 18 | (def bool-tens (-> range-tens 19 | (apl-gol/membership [1 2 3 4 7]) 20 | ;;convert to zeros/ones for display. 21 | (tens/clone :datatype :int8))) 22 | 23 | 24 | (def take-tens (apl-gol/apl-take bool-tens [5 7])) 25 | 26 | 27 | (def right-rotate (apl-gol/rotate-vertical take-tens -2)) 28 | 29 | (def down-rotate (apl-gol/rotate-horizontal right-rotate -1)) 30 | 31 | (def R-matrix down-rotate) 32 | 33 | 34 | (def rotate-arg [1 0 -1]) 35 | 36 | (def group-rotated (->> rotate-arg 37 | (map (partial apl-gol/rotate-vertical down-rotate)))) 38 | 39 | (def table-rotated (->> rotate-arg 40 | (mapcat (fn [rot-amount] 41 | (->> group-rotated 42 | (mapv #(apl-gol/rotate-horizontal 43 | % rot-amount))))))) 44 | 45 | 46 | (def summed (apply dtype-fn/+ table-rotated)) 47 | 48 | 49 | (def next-gen (apl-gol/game-of-life-operator R-matrix summed)) 50 | 51 | 52 | (def half-RR (apl-gol/apl-take R-matrix [-10 -20])) 53 | 54 | 55 | (def RR (-> (apl-gol/apl-take R-matrix [-10 -20]) 56 | (apl-gol/apl-take [15 35])))) 57 | 58 | (defn make-big-rr 59 | [] 60 | (with-bindings {#'tens-impl/*container-type* :sparse} 61 | (-> (apl-gol/apl-take R-matrix [-1000 -2000]) 62 | (apl-gol/apl-take [3840 2160])))) 63 | 64 | 65 | (defn bigRR-time-test 66 | [] 67 | (dotimes [iter 5] 68 | (time 69 | (->> (apl-gol/life-seq (make-big-rr)) 70 | (take 1000) 71 | last)))) 72 | 73 | 74 | 75 | (deftest game-of-life-test 76 | (let [end-matrix (->> (apl-gol/life-seq RR) 77 | (take 1000) 78 | last)] 79 | (is (= :sparse (dtype/buffer-type (tens-impl/tensor->buffer end-matrix)))) 80 | (is (= apl-gol/end-state (tens/->jvm end-matrix)))))) 81 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/array_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.array-test 2 | (:require [tech.v2.datatype.array :as ary] 3 | [tech.v2.datatype.nio-buffer :as nio-buf] 4 | [tech.v2.datatype.typed-buffer :as typed-buf] 5 | [tech.v2.datatype :as dtype] 6 | [clojure.test :refer :all])) 7 | 8 | 9 | (deftest array-basics 10 | (let [src-ary (short-array (range 10)) 11 | byte-data (byte-array 10) 12 | float-data (float-array 10)] 13 | (dtype/copy! src-ary byte-data) 14 | (is (= (vec (range 10)) 15 | (dtype/->vector byte-data))) 16 | (is (= (mapv float (range 10)) 17 | (-> (dtype/copy! src-ary float-data) 18 | (dtype/->vector)))) 19 | (let [sub-ary (dtype/sub-buffer src-ary 3 3)] 20 | (dtype/set-value! sub-ary 1 20) 21 | (is (= [3 20 5] (dtype/->vector sub-ary))) 22 | (is (= [0 1 2 3 20 5 6 7 8 9] (dtype/->vector src-ary))) 23 | (dtype/set-constant! sub-ary 0 0 3) 24 | (is (= [0 0 0] (dtype/->vector sub-ary))) 25 | (is (= [0 1 2 0 0 0 6 7 8 9] (dtype/->vector src-ary)))) 26 | (let [new-shorts (short-array (range 4)) 27 | new-indexes (int-array [0 2 4 6])] 28 | (dtype/write-indexes! src-ary 29 | (dtype/->buffer-backing-store new-indexes) 30 | (dtype/->buffer-backing-store new-shorts)) 31 | (is (= [0 1 1 0 2 0 3 7 8 9] (dtype/->vector src-ary)))))) 32 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/bitmap_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.bitmap-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | [tech.v2.datatype.bitmap :as bitmap] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | (deftest basic-bitmap-test 9 | (let [ba (bitmap/->bitmap (range 5)) 10 | bb (bitmap/->bitmap (filter even? (range 10)))] 11 | (is (= (set #{0 4 2}) (set (dtype-proto/set-and ba bb)))) 12 | (is (= (set #{0 1 4 6 3 2 8}) (set (dtype-proto/set-or ba bb)))) 13 | (is (= (set #{7 6 9 5 8}) (set (dtype-proto/set-offset ba 5)))) 14 | (is (= (vec (range 5)) 15 | (vec (dtype/->reader ba))))) 16 | (let [big-range (seq (range 3000000000 3000000005)) 17 | ba (bitmap/->bitmap big-range)] 18 | (is (= (vec big-range) 19 | (vec (dtype/->reader ba)))) 20 | (is (= (set big-range) 21 | (set (dtype/->reader ba))))) 22 | (is (= #{} 23 | (set (bitmap/->bitmap nil))))) 24 | 25 | 26 | (deftest empty-bitmap-as-random-access-reader 27 | (is (== 0 (dtype/ecount (bitmap/bitmap->efficient-random-access-reader 28 | (bitmap/->bitmap)))))) 29 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/byte_conversions_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.byte-conversions-test 2 | (:require [clojure.test :refer [deftest is]] 3 | [tech.v2.datatype :as dtype]) 4 | (:import [tech.v2.datatype ByteConversions])) 5 | 6 | 7 | 8 | (deftest short-test 9 | (let [test-val Short/MAX_VALUE 10 | le-ary (byte-array 2) 11 | be-ary (byte-array 2)] 12 | (ByteConversions/shortToWriterLE test-val (dtype/->writer le-ary) 0) 13 | (ByteConversions/shortToWriterBE test-val (dtype/->writer be-ary) 0) 14 | (is (== test-val (ByteConversions/shortFromReaderLE (dtype/->reader le-ary) 0))) 15 | (is (== test-val (ByteConversions/shortFromReaderBE (dtype/->reader be-ary) 0))) 16 | (is (not= (vec le-ary) (vec be-ary))))) 17 | 18 | 19 | (deftest int-test 20 | (let [test-val Integer/MAX_VALUE 21 | le-ary (byte-array 4) 22 | be-ary (byte-array 4)] 23 | (ByteConversions/intToWriterLE test-val (dtype/->writer le-ary) 0) 24 | (ByteConversions/intToWriterBE test-val (dtype/->writer be-ary) 0) 25 | (is (== test-val (ByteConversions/intFromReaderLE (dtype/->reader le-ary) 0))) 26 | (is (== test-val (ByteConversions/intFromReaderBE (dtype/->reader be-ary) 0))) 27 | (is (not= (vec le-ary) (vec be-ary))))) 28 | 29 | 30 | (deftest float-test 31 | (let [test-val (float Integer/MAX_VALUE) 32 | le-ary (byte-array 4) 33 | be-ary (byte-array 4)] 34 | (ByteConversions/floatToWriterLE test-val (dtype/->writer le-ary) 0) 35 | (ByteConversions/floatToWriterBE test-val (dtype/->writer be-ary) 0) 36 | (is (== test-val (ByteConversions/floatFromReaderLE (dtype/->reader le-ary) 0))) 37 | (is (== test-val (ByteConversions/floatFromReaderBE (dtype/->reader be-ary) 0))) 38 | (is (not= (vec le-ary) (vec be-ary))))) 39 | 40 | 41 | (deftest long-test 42 | (let [test-val Long/MAX_VALUE 43 | le-ary (byte-array 8) 44 | be-ary (byte-array 8)] 45 | (ByteConversions/longToWriterLE test-val (dtype/->writer le-ary) 0) 46 | (ByteConversions/longToWriterBE test-val (dtype/->writer be-ary) 0) 47 | (is (== test-val (ByteConversions/longFromReaderLE (dtype/->reader le-ary) 0))) 48 | (is (== test-val (ByteConversions/longFromReaderBE (dtype/->reader be-ary) 0))) 49 | (is (not= (vec le-ary) (vec be-ary))))) 50 | 51 | 52 | (deftest double-test 53 | (let [test-val (double Long/MAX_VALUE) 54 | le-ary (byte-array 8) 55 | be-ary (byte-array 8)] 56 | (ByteConversions/doubleToWriterLE test-val (dtype/->writer le-ary) 0) 57 | (ByteConversions/doubleToWriterBE test-val (dtype/->writer be-ary) 0) 58 | (is (== test-val (ByteConversions/doubleFromReaderLE (dtype/->reader le-ary) 0))) 59 | (is (== test-val (ByteConversions/doubleFromReaderBE (dtype/->reader be-ary) 0))) 60 | (is (not= (vec le-ary) (vec be-ary))))) 61 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/cheatsheet_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.cheatsheet-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.casting :as casting] 4 | [tech.v2.tensor :as tens] 5 | [tech.v2.datatype.functional :as dtype-fn] 6 | [clojure.test :refer :all])) 7 | 8 | 9 | 10 | (deftest cheatsheet-test 11 | (let [float-data (dtype/copy! (range 10) (float-array 10))] 12 | (is (= [0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0] 13 | (dtype/->vector float-data)))) 14 | (let [test-data (dtype-fn/+ (range 10 0 -1) 5) 15 | indexes (dtype-fn/argsort test-data)] 16 | (is (= [6 7 8 9 10 11 12 13 14 15] 17 | (dtype-fn/indexed-reader indexes (vec test-data))))) 18 | 19 | (let [test-tens (tens/->tensor (partition 3 (range 9))) 20 | ;;This actually tests quite a lot but type promotion is one 21 | ;;thing. 22 | result-tens (dtype-fn/+ 2 (tens/select test-tens [1 0] :all))] 23 | (is (= [[5.000 6.000 7.000] 24 | [2.000 3.000 4.000]] 25 | (tens/->jvm result-tens))))) 26 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/index_algebra_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.index-algebra-test 2 | (:require [tech.v2.datatype.index-algebra :as idx-alg] 3 | [tech.v2.datatype.protocols :as dtype-proto] 4 | ;;This has to be loaded to fill out the protocol matrix. 5 | [tech.v2.datatype]) 6 | (:require [clojure.test :refer :all])) 7 | 8 | 9 | 10 | 11 | (deftest idx-alg-test 12 | (let [item (-> [1 2 3 4] 13 | (idx-alg/offset 2) 14 | (idx-alg/broadcast 8) 15 | (idx-alg/select (range 1 5)))] 16 | (is (= [4 1 2 3] (vec item))) 17 | (is (= (vec (flatten (repeat 2 [4 1 2 3]))) 18 | (idx-alg/broadcast item 8))) 19 | (let [new-item (-> (idx-alg/broadcast item 8) 20 | (idx-alg/offset 3) 21 | (idx-alg/select (range 2 6)))] 22 | (is (= [1 2 3 4] new-item)) 23 | (is (dtype-proto/convertible-to-range? new-item))) 24 | (let [new-item (idx-alg/select item 2)] 25 | (is (= [2] new-item)) 26 | (is (:select-scalar? (meta new-item)))))) 27 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/javacpp_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.javacpp-test 2 | (:require [clojure.test :refer :all] 3 | [tech.v2.datatype.javacpp :as jcpp-dtype] 4 | [tech.v2.datatype :as dtype] 5 | [tech.v2.datatype.jna :as dtype-jna] 6 | [tech.resource :as resource]) 7 | ;;required to load some of the javacpp help functions; they are small functions 8 | ;;compiled into each bound library. 9 | (:import [org.bytedeco.javacpp opencv_core])) 10 | 11 | ;;Force loading of the class to make unit tests work 12 | (println opencv_core/ACCESS_FAST) 13 | 14 | 15 | (deftest ensure-fast-copy-paths 16 | (resource/stack-resource-context 17 | (let [int-data (int-array [1 2 3 4 5 6]) 18 | result (jcpp-dtype/make-pointer-of-type :float32 6) 19 | short-data (short-array 6)] 20 | (dtype/copy! int-data 0 result 0 6) 21 | (dtype/copy! result 0 short-data 0 6) 22 | (is (= [1 2 3 4 5 6] 23 | (vec short-data)))))) 24 | 25 | (deftest typed-pointers 26 | (resource/stack-resource-context 27 | (let [src-data (range 255 235 -1) 28 | typed-ptr (jcpp-dtype/make-typed-pointer :uint8 src-data) 29 | result-data (short-array 20) 30 | byte-data (byte-array 20) 31 | signed-data (range -1 -21 -1)] 32 | (dtype/copy! typed-ptr result-data) 33 | (is (= (vec result-data) 34 | (vec src-data))) 35 | (dtype/copy! typed-ptr 0 byte-data 0 20 {:unchecked? true}) 36 | (is (= (vec signed-data) 37 | (vec byte-data)))))) 38 | 39 | 40 | (deftest datatype-base-->array 41 | (resource/stack-resource-context 42 | (let [base-ptr (jcpp-dtype/make-pointer-of-type :float32 (range 10)) 43 | typed-ptr (jcpp-dtype/make-typed-pointer :int32 10)] 44 | ;;These should be nil but should not cause exceptions. 45 | (is (and (not (dtype/->array base-ptr)) 46 | (not (dtype/->array typed-ptr)))) 47 | (dtype/copy! base-ptr typed-ptr) 48 | (is (= (vec (range 10)) 49 | (dtype/->vector typed-ptr)))))) 50 | 51 | 52 | (deftest jcpp-ptr-interface-specification 53 | (resource/stack-resource-context 54 | (let [base-ptr (jcpp-dtype/make-pointer-of-type :float32 (range 10))] 55 | (is (instance? org.bytedeco.javacpp.Pointer (dtype/clone base-ptr))) 56 | (is (= (dtype/->vector base-ptr) 57 | (dtype/->vector (dtype/clone base-ptr))))))) 58 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/jna_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.jna-test 2 | (:require [clojure.test :refer :all] 3 | [tech.v2.datatype :as dtype] 4 | [tech.v2.datatype.protocols :as dtype-proto] 5 | [tech.v2.datatype.base :as base] 6 | [tech.v2.datatype.jna :as dtype-jna] 7 | [tech.jna :as jna] 8 | [tech.v2.datatype.typed-buffer :as typed-buffer] 9 | [tech.parallel.for :as parallel-for])) 10 | 11 | 12 | (set! *warn-on-reflection* true) 13 | (set! *unchecked-math* :warn-on-boxed) 14 | 15 | 16 | (deftest jna-things-are-typed-pointers 17 | (let [test-buf (dtype-jna/make-typed-pointer :uint8 (range 255 245 -1))] 18 | (is (= :uint8 (dtype-proto/get-datatype test-buf))) 19 | (is (typed-buffer/typed-buffer? test-buf)) 20 | (is (identical? test-buf (typed-buffer/->typed-buffer test-buf))) 21 | (let [data-buf (int-array (dtype/ecount test-buf))] 22 | (dtype/copy! test-buf data-buf) 23 | (is (= (vec (range 255 245 -1)) 24 | (dtype/->vector data-buf)))))) 25 | 26 | 27 | (deftest copy-time-test 28 | (testing "Run perf regression of times spent to copy data" 29 | (let [num-items (long 100000) 30 | src-data (float-array (range num-items)) 31 | dst-data (float-array num-items) 32 | array-copy (fn [] 33 | (parallel-for/parallel-for 34 | idx num-items 35 | (aset dst-data idx (aget src-data idx)))) 36 | 37 | src-ptr (dtype-jna/make-typed-pointer :float32 src-data) 38 | dst-ptr (dtype-jna/make-typed-pointer :float32 num-items) 39 | 40 | ptr-ptr-copy (fn [] 41 | (dtype/copy! src-ptr 0 dst-ptr 0 num-items)) 42 | 43 | array-ptr-copy (fn [] 44 | (dtype/copy! src-data 0 dst-ptr 0 num-items)) 45 | fns {:array-hand-copy array-copy 46 | :ptr-ptr-copy ptr-ptr-copy 47 | :array-ptr-copy array-ptr-copy} 48 | run-timed-fns (fn [] 49 | (->> fns 50 | (map (fn [[fn-name time-fn]] 51 | [fn-name (with-out-str 52 | (time 53 | (dotimes [iter 400] 54 | (time-fn))))])) 55 | (into {}))) 56 | warmup (run-timed-fns) 57 | times (run-timed-fns)] 58 | (println times)))) 59 | 60 | 61 | (deftest set-constant! 62 | (let [test-buf (dtype-jna/make-typed-pointer :int64 5)] 63 | (dtype/set-constant! test-buf 0 1 (dtype/ecount test-buf)) 64 | (is (= [1 1 1 1 1] (dtype/->vector test-buf))) 65 | (is (= [1 1 1 1 1] (-> (dtype/clone test-buf) 66 | dtype/->vector))) 67 | (is (not= 0 (-> (dtype/clone test-buf) 68 | dtype/->jna-ptr 69 | dtype-jna/pointer->address))))) 70 | 71 | 72 | (deftest simple-init-ptr 73 | (let [test-ptr (dtype-jna/make-typed-pointer :int64 [2 2])] 74 | (is (= [2 2] 75 | (dtype/->vector test-ptr)))) 76 | (let [test-ptr (dtype-jna/make-typed-pointer :float64 [2 2])] 77 | (is (= [2.0 2.0] 78 | (dtype/->vector test-ptr))))) 79 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/mmap_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.mmap-test 2 | (:require [tech.v2.datatype.mmap :as mmap] 3 | [tech.v2.datatype :as dtype] 4 | [tech.resource :as resource] 5 | [clojure.test :refer [deftest is]])) 6 | 7 | 8 | 9 | (deftest base-mmap-data 10 | (resource/stack-resource-context 11 | (let [proj-mmap (mmap/mmap-file "project.clj") 12 | initial-str (String. (dtype/->array-copy proj-mmap))] 13 | (is (.startsWith initial-str"(defproject"))))) 14 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/reader_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.reader-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.unary-op :as unary-op] 4 | [clojure.test :refer :all])) 5 | 6 | 7 | (deftest reader-sub-buffers 8 | (is (= [5 6] 9 | (-> (unary-op/unary-reader :int32 (+ x 2) [1 2 3 4]) 10 | (dtype/sub-buffer 2 2) 11 | vec)))) 12 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/simple_ops_perf_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.simple-ops-perf-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.typecast :as typecast] 4 | [tech.v2.datatype.functional :as dfn] 5 | [tech.v2.datatype.comparator :as comp] 6 | [tech.parallel.for :as parallel-for] 7 | [criterium.core :as crit]) 8 | (:import [it.unimi.dsi.fastutil.ints IntArrays IntComparator] 9 | [java.util Comparator Arrays])) 10 | 11 | 12 | (set! *unchecked-math* :warn-on-boxed) 13 | 14 | 15 | (def n-elems 1000000) 16 | (def src-data (range n-elems)) 17 | (def src-ary (long-array src-data)) 18 | (def src-rdr (typecast/datatype->reader :int64 src-data)) 19 | (def dst-data (long-array n-elems)) 20 | 21 | 22 | (defn hardcode-copy 23 | [] 24 | (parallel-for/parallel-for 25 | idx 26 | (long n-elems) 27 | (aset ^longs dst-data idx (unchecked-multiply 2 (aget ^longs src-ary idx))))) 28 | 29 | 30 | (defn reader-copy 31 | [rdr] 32 | (let [rdr (typecast/datatype->reader :int64 rdr)] 33 | (parallel-for/parallel-for 34 | idx 35 | (long n-elems) 36 | (aset ^longs dst-data idx (.read rdr idx))))) 37 | 38 | 39 | (defn reader-writer-copy 40 | [rdr wtr] 41 | (let [rdr (typecast/datatype->reader :int64 rdr) 42 | wtr (typecast/datatype->writer :int64 wtr)] 43 | (parallel-for/parallel-for 44 | idx 45 | (long n-elems) 46 | (.write wtr 47 | idx (.read rdr idx))))) 48 | 49 | (def obj-data (object-array (shuffle (range n-elems)))) 50 | 51 | (defn arrays-comparator-sort 52 | [ary-data] 53 | (let [test-comp (comparator <)] 54 | (Arrays/sort ary-data test-comp) 55 | (dtype/make-container :java-array :int64 (dtype/->reader ary-data :int64)))) 56 | 57 | 58 | (def int-data (int-array (shuffle (range n-elems)))) 59 | 60 | (defn int-arrays-quicksort 61 | [] 62 | (let [test-comp (comp/make-comparator 63 | :int32 (- lhs rhs))] 64 | (IntArrays/quickSort int-data test-comp))) 65 | 66 | 67 | (defn parallel-int-arrays-quicksort 68 | [] 69 | (let [test-comp (comp/make-comparator 70 | :int32 (- lhs rhs))] 71 | (IntArrays/parallelQuickSort int-data test-comp))) 72 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/streams_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.streams-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.typecast :as typecast] 4 | [clojure.test :refer [deftest is]])) 5 | 6 | 7 | (defn typed-stream-sum 8 | [rdr] 9 | (case (dtype/get-datatype rdr) 10 | :boolean (-> (typecast/datatype->reader :boolean rdr) 11 | (.typedStream) 12 | (.sum)) 13 | :int8 (-> (typecast/datatype->reader :int8 rdr) 14 | (.typedStream) 15 | (.sum)) 16 | :int16 (-> (typecast/datatype->reader :int16 rdr) 17 | (.typedStream) 18 | (.sum)) 19 | :int32 (-> (typecast/datatype->reader :int32 rdr) 20 | (.typedStream) 21 | (.sum)) 22 | :int64 (-> (typecast/datatype->reader :int64 rdr) 23 | (.typedStream) 24 | (.sum)) 25 | :float32 (-> (typecast/datatype->reader :float32 rdr) 26 | (.typedStream) 27 | (.sum)) 28 | :float64 (-> (typecast/datatype->reader :float64 rdr) 29 | (.typedStream) 30 | (.sum)) 31 | (-> (typecast/datatype->reader :object rdr) 32 | (.typedStream) 33 | (.reduce (reify java.util.function.BinaryOperator 34 | (apply [this lhs rhs] 35 | (+ lhs rhs))))))) 36 | 37 | 38 | (deftest basic-datatype-streams 39 | (let [dtype-list [:int8 :int16 :int32 :int64 40 | :float32 :float64] 41 | readers (concat (->> dtype-list 42 | (map #(dtype/make-container :java-array % (range 10))) 43 | (map dtype/->reader)) 44 | [(dtype/->reader (vec (range 10))) 45 | (dtype/->reader (boolean-array (map even? (range 10))))])] 46 | (is (= [45 47 | 45 48 | 45 49 | 45 50 | 45.0 51 | 45.0 52 | (java.util.Optional/of 45) 53 | 5] 54 | (vec (mapv typed-stream-sum readers)))))) 55 | -------------------------------------------------------------------------------- /test/tech/v2/datatype/vector_of_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.datatype.vector-of-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.functional :as dfn] 4 | [clojure.test :refer [deftest is]])) 5 | 6 | 7 | 8 | (deftest base-as-vector-test 9 | (let [vec-dtypes [:boolean :byte :short :int :long 10 | :float :double]] 11 | (doseq [dtype vec-dtypes] 12 | (let [src-seq (if-not (= :boolean dtype) 13 | (range 10) 14 | (flatten (repeat 5 [true false]))) 15 | lhs (dtype/make-container :java-array dtype src-seq) 16 | rhs (apply vector-of dtype src-seq)] 17 | (is (= (dtype/get-datatype lhs) 18 | (dtype/get-datatype rhs))) 19 | (is (= (vec lhs) 20 | (vec (dtype/->reader rhs)))))))) 21 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/broadcast_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.broadcast-test 2 | (:require [tech.v2.tensor :as dtt] 3 | [tech.v2.datatype :as dtype] 4 | [tech.v2.datatype.functional :as dfn] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | (deftest simple-broadcast 9 | (let [bcast-tens (dtt/broadcast (dtt/->tensor [5 6 7]) 10 | [3 3])] 11 | (is (dfn/equals [5 6 7 5 6 7 5 6 7] 12 | (dtype/->reader bcast-tens))) 13 | (is (dfn/equals (dtt/->tensor [[5 6 7] [5 6 7] [5 6 7]]) 14 | bcast-tens)))) 15 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/color_gradient_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.color-gradient-test 2 | (:require [tech.v2.tensor :as dtt] 3 | [tech.v2.tensor.color-gradients :as cgrad] 4 | [tech.v2.datatype.functional :as dfn] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | ;;Sometimes data has NAN's or INF's 9 | (def test-nan-tens (dtt/->tensor (repeat 128 10 | (->> (range 0 512) 11 | (map-indexed (fn [idx val] 12 | (if (even? idx) 13 | Double/NaN 14 | (double val)))) 15 | vec)))) 16 | 17 | 18 | (deftest color-grad-test 19 | (is (dfn/equals (dtt/->tensor [[ 0 0 0 0] 20 | [255 42 34 209] 21 | [ 0 0 0 0] 22 | [255 42 34 209]]) 23 | (-> (cgrad/colorize test-nan-tens :temperature-map 24 | :alpha? true 25 | :check-invalid? true 26 | :invert-gradient? true 27 | :data-min 0 28 | :data-max 512) 29 | (dtt/select 0 (range 4) :all))))) 30 | 31 | 32 | (deftest colorize->clj-test 33 | (is (= [[[32 252 0] [21 255 25] [0 255 185] [0 225 251] [0 95 255] [0 0 255]] 34 | [[32 252 0] [21 255 25] [0 255 185] [0 225 251] [0 95 255] [0 0 255]]] 35 | (cgrad/colorize->clj [[0.0 0.2 0.4 0.6 0.8 1.0] 36 | [0.0 0.2 0.4 0.6 0.8 1.0]] 37 | :green-red)))) 38 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/copy_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.copy-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.datatype.functional :as dfn] 4 | [tech.v2.tensor :as dtt] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | (deftest tensor-copy-test 9 | (let [new-tens (repeat 2 (dtt/->tensor [[2] [3]])) 10 | dest-tens (dtt/new-tensor [2 2])] 11 | (dtype/copy-raw->item! (->> new-tens 12 | (map #(dtt/select % 0 :all))) 13 | (dtt/select dest-tens 0 :all)) 14 | (is (dfn/equals dest-tens (dtt/->tensor [[2 2] [0 0]]))) 15 | (dtype/copy-raw->item! (->> new-tens 16 | (map #(dtt/select % 1 :all))) 17 | (dtt/select dest-tens 1 :all)) 18 | (is (dfn/equals dest-tens (dtt/->tensor [[2 2] [3 3]]))))) 19 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/dimensions_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.dimensions-test 2 | (:require [tech.v2.tensor.dimensions :as ct-dims] 3 | [clojure.test :refer :all])) 4 | 5 | 6 | (defn ->raw 7 | [dimensions] 8 | (->> (select-keys dimensions [:shape :strides]) 9 | (map (fn [[k v]] 10 | [k (vec v)])) 11 | (into {}))) 12 | 13 | 14 | (deftest in-place-reshape-test 15 | (is (= {:shape [6 2] 16 | :strides [2 1]} 17 | (->raw 18 | (ct-dims/in-place-reshape 19 | (ct-dims/dimensions [3 2 2] [4 2 1]) [6 2])))) 20 | (is (= {:shape [3 4] 21 | :strides [4 1]} 22 | (->raw 23 | (ct-dims/in-place-reshape 24 | (ct-dims/dimensions [3 2 2] [4 2 1]) 25 | [3 4])))) 26 | #_(is (= {:shape [3 4] 27 | :strides [5 1]} 28 | (->raw 29 | (ct-dims/in-place-reshape 30 | (ct-dims/dimensions [3 2 2] [5 2 1]) 31 | [3 4])))) 32 | #_(is (= {:shape [20 8] :strides [10 1]} 33 | (->raw 34 | (ct-dims/in-place-reshape 35 | (ct-dims/dimensions [4 5 8] [50 10 1]) 36 | [20 8])))) 37 | #_(is (= {:shape [20 8 1 1] :strides [10 1 1 1]} 38 | (->raw 39 | (ct-dims/in-place-reshape 40 | (ct-dims/dimensions [4 5 8] [50 10 1]) 41 | [20 8 1 1])))) 42 | #_(is (= {:shape [1 1 20 8] :strides [200 200 10 1]} 43 | (->raw 44 | (ct-dims/in-place-reshape 45 | (ct-dims/dimensions [4 5 8] [50 10 1]) 46 | [1 1 20 8])))) 47 | (is (= {:shape [169 5] :strides [5 1]} 48 | (->raw 49 | (ct-dims/in-place-reshape 50 | (ct-dims/dimensions [845] [1]) 51 | [169 5])))) 52 | ;;This test is just f-ed up. But the thing is that if the dimensions are dense then 53 | ;;in-place reshape that preserves ecount is possible; it is just an arbitrary 54 | ;;reinterpretation of the data. 55 | (is (= {:shape [10 4 9] :strides [36 9 1]} 56 | (->raw 57 | (ct-dims/in-place-reshape 58 | (ct-dims/dimensions [10 1 18 2] [36 36 2 1]) 59 | [10 4 9])))) 60 | #_(is (= {:shape [845 1] :strides [25 1]} 61 | (->raw 62 | (ct-dims/in-place-reshape 63 | (ct-dims/dimensions [13 13 5 1] [1625 125 25 1]) 64 | [845 1])))) 65 | (is (= {:shape [1 1] :strides [1 1]} 66 | (->raw 67 | (ct-dims/in-place-reshape 68 | (ct-dims/dimensions [1] [1]) 69 | [1 1]))))) 70 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/mmul_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.mmul-test 2 | (:require [tech.v2.datatype :as dtype] 3 | [tech.v2.tensor :as tens] 4 | [tech.v2.tensor.select-test :refer [tensor-default-context]] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | (defn do-basic-mm 9 | [container-type datatype] 10 | (let [n-elems (* 2 5) 11 | test-mat (tens/->tensor (partition 5 (range n-elems)) 12 | :container-type container-type 13 | :datatype datatype)] 14 | (is (= [[30 80] [80 255]] 15 | (-> (tens/matrix-multiply test-mat (tens/transpose test-mat [1 0])) 16 | (tens/->jvm :datatype :int32))) 17 | (with-out-str (println container-type datatype))) 18 | 19 | 20 | (is (= [[25 30 35 40 45] 21 | [30 37 44 51 58] 22 | [35 44 53 62 71] 23 | [40 51 62 73 84] 24 | [45 58 71 84 97]] 25 | (-> (tens/matrix-multiply (tens/transpose test-mat [1 0]) test-mat) 26 | (tens/->jvm :datatype :int32))) 27 | (with-out-str (println container-type datatype))))) 28 | 29 | 30 | (deftest basic-mm 31 | (do-basic-mm :list :float64) 32 | (do-basic-mm :list :int32) 33 | ;;sparse is disabled for now 34 | (comment 35 | (do-basic-mm :sparse :int32) 36 | (do-basic-mm :sparse :int16))) 37 | -------------------------------------------------------------------------------- /test/tech/v2/tensor/select_test.clj: -------------------------------------------------------------------------------- 1 | (ns tech.v2.tensor.select-test 2 | (:require [tech.v2.tensor :as tens] 3 | [tech.v2.tensor.impl :as impl] 4 | [tech.v2.datatype :as dtype] 5 | [clojure.test :refer :all])) 6 | 7 | 8 | (defmacro tensor-default-context 9 | [container-type datatype & body] 10 | `(with-bindings {#'impl/*datatype* ~datatype 11 | #'impl/*container-type* ~container-type} 12 | ~@body)) 13 | 14 | (defn ->jvm-flatten 15 | [tensor] 16 | (-> (tens/->jvm tensor :datatype :int32) 17 | flatten)) 18 | 19 | 20 | (defn select 21 | [container-type datatype] 22 | (tensor-default-context 23 | container-type datatype 24 | (let [mat-tens (tens/->tensor (repeat 2 (partition 3 (range 9))))] 25 | (let [sel-tens (tens/select mat-tens :all :all [1 2])] 26 | (is (= (vec (flatten (repeat 2 [1 2 4 5 7 8]))) 27 | (->jvm-flatten sel-tens))) 28 | (is (= [2 3 2] 29 | (dtype/shape sel-tens)))) 30 | (let [sel-tens (tens/select mat-tens :all :all [2])] 31 | (is (= (flatten (repeat 2 [2 5 8])) 32 | (->jvm-flatten sel-tens))) 33 | (is (= [2 3 1] 34 | (dtype/shape sel-tens)))) 35 | (let [sel-tens (tens/select mat-tens :all :all 2)] 36 | (is (= (flatten (repeat 2 [2 5 8])) 37 | (->jvm-flatten sel-tens))) 38 | (is (= [2 3] 39 | (dtype/shape sel-tens))) 40 | (is (not (tens/dimensions-dense? sel-tens)))) 41 | (let [sel-tens (tens/select mat-tens :all [1 2] :all)] 42 | (is (= (flatten (repeat 2 [3 4 5 6 7 8])) 43 | (->jvm-flatten sel-tens))) 44 | (is (= [2 2 3] 45 | (dtype/shape sel-tens))) 46 | (is (not (tens/dimensions-dense? sel-tens)))) 47 | (let [sel-tens (tens/select mat-tens :all [2] :all)] 48 | (is (= (flatten (repeat 2 [6 7 8])) 49 | (->jvm-flatten sel-tens))) 50 | (is (= [2 1 3] 51 | (dtype/shape sel-tens))) 52 | (is (not (tens/dimensions-dense? sel-tens)))) 53 | (let [sel-tens (tens/select mat-tens :all 0 :all)] 54 | (is (= (flatten (repeat 2 [0 1 2])) 55 | (->jvm-flatten sel-tens))) 56 | (is (= [2 3] 57 | (dtype/shape sel-tens))) 58 | (is (not (tens/dimensions-dense? sel-tens)))) 59 | 60 | (let [sel-tens (tens/select mat-tens [1] [1] :all)] 61 | (is (= [3 4 5] 62 | (->jvm-flatten sel-tens))) 63 | (is (= [1 1 3] 64 | (dtype/shape sel-tens))) 65 | (is (tens/dimensions-dense? sel-tens))) 66 | 67 | (let [sel-tens (tens/select mat-tens 1 1 :all)] 68 | (is (= [3 4 5] 69 | (->jvm-flatten sel-tens))) 70 | (is (= [3] 71 | (dtype/shape sel-tens))) 72 | (is (tens/dimensions-dense? sel-tens))) 73 | 74 | (let [sel-tens (tens/select mat-tens 1 :all 2)] 75 | (is (= [2 5 8] 76 | (->jvm-flatten sel-tens))) 77 | (is (= [3] 78 | (dtype/shape sel-tens))) 79 | (is (not (tens/dimensions-dense? sel-tens))))))) 80 | 81 | 82 | 83 | (deftest dense-select 84 | (select :typed-buffer :float32)) 85 | 86 | 87 | (comment 88 | ;;sparse is disabled for now. 89 | (deftest sparse-select 90 | (select :sparse :float32))) 91 | --------------------------------------------------------------------------------