Family IDs start at 128 to allow separation from sketches-core for as long as possible without 33 | * inducing a mutual dependency between packages.
34 | * 35 | * @author Lee Rhodes 36 | * @author Jon Malkin 37 | */ 38 | public enum MatrixFamily { 39 | /** 40 | * The Frequent Directions sketch is used for approximate Singular Value Decomposition (MatrixOps) of a 41 | * matrix. 42 | */ 43 | MATRIX(128, "Matrix", 2, 3), 44 | /** 45 | * Select Frequent Directions Family 46 | */ 47 | FREQUENTDIRECTIONS(129, "FrequentDirections", 2, 4); 48 | 49 | 50 | private static final Map31 | * The low significance bytes of this long items structure are on the right. Multi-byte 32 | * integers (int and long) are stored in native byte order. All byte 33 | * values are treated as unsigned.
34 | * 35 | *An empty Frequent Directions sketch requires 16 bytes. A non-empty sketch requires 32 bytes 36 | * of preamble. The matrix is dense and is expected to dominate storage.
37 | * 38 | *39 | * Long || Start Byte Adr: 40 | * Adr: 41 | * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | 42 | * 0 ||----------Sketch Size (k)----------| Flags | FamID | SerVer | Preamble_Longs | 43 | * 44 | * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | 45 | * 1 ||-----------Num. Columns------------|------Current Num Rows-------------------------| 46 | * 47 | * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 | 48 | * 2 ||-----------------------------Total Records Seen (n)--------------------------------| 49 | * 50 | * || 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 | 51 | * 3 ||------------------------------Total SV Adjustment----------------------------------| 52 | *53 | * 54 | * @author Jon Malkin 55 | */ 56 | @SuppressWarnings("restriction") 57 | public final class PreambleUtil { 58 | 59 | /** 60 | * The java line separator character as a String. 61 | */ 62 | private static final String LS = System.getProperty("line.separator"); 63 | 64 | private PreambleUtil() {} 65 | 66 | // ###### DO NOT MESS WITH THIS FROM HERE ... 67 | // Preamble byte Addresses 68 | static final int PREAMBLE_LONGS_BYTE = 0; 69 | static final int SER_VER_BYTE = 1; 70 | static final int FAMILY_BYTE = 2; 71 | static final int FLAGS_BYTE = 3; 72 | static final int K_INT = 4; 73 | static final int NUM_ROWS_INT = 8; 74 | static final int NUM_COLUMNS_INT = 12; 75 | static final int N_LONG = 16; 76 | static final int SV_ADJUSTMENT_DOUBLE = 24; 77 | 78 | // flag bit masks 79 | static final int EMPTY_FLAG_MASK = 4; 80 | 81 | // Other constants 82 | static final int SER_VER = 1; 83 | 84 | 85 | /** 86 | * Returns a human readable string summary of the preamble state of the given Memory. 87 | * Note: other than making sure that the given Memory size is large 88 | * enough for just the preamble, this does not do much value checking of the contents of the 89 | * preamble as this is primarily a tool for debugging the preamble visually. 90 | * 91 | * @param mem the given Memory. 92 | * @return the summary preamble string. 93 | */ 94 | public static String preambleToString(final Memory mem) { 95 | 96 | final int preLongs = getAndCheckPreLongs(mem); // make sure we can get the assumed preamble 97 | final MatrixFamily family = MatrixFamily.idToFamily(extractFamilyID(mem)); 98 | 99 | final int serVer = extractSerVer(mem); 100 | final int flags = extractFlags(mem); 101 | final String flagsStr = Integer.toBinaryString(flags) + ", " + flags; 102 | final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0; 103 | 104 | final int k = extractK(mem); 105 | final int d = extractNumColumns(mem); 106 | final int numRows = extractNumRows(mem); 107 | 108 | long n = 0; 109 | double svAdjustment = 0.0; 110 | if (!isEmpty) { 111 | n = extractN(mem); 112 | svAdjustment = extractSVAdjustment(mem); 113 | } 114 | 115 | final StringBuilder sb = new StringBuilder(); 116 | sb.append(LS) 117 | .append("### START ") 118 | .append(family.getFamilyName().toUpperCase()) 119 | .append(" PREAMBLE SUMMARY").append(LS) 120 | .append("Byte 0: Preamble Longs : ").append(preLongs).append(LS) 121 | .append("Byte 1: Serialization Version: ").append(serVer).append(LS) 122 | .append("Byte 2: Family : ").append(family.toString()).append(LS) 123 | .append("Byte 3: Flags Field : ").append(flagsStr).append(LS) 124 | .append(" EMPTY : ").append(isEmpty).append(LS) 125 | .append("Bytes 4-7 : Sketch Size (k) : ").append(k).append(LS) 126 | .append("Bytes 8-11: Num Rows : ").append(numRows).append(LS) 127 | .append("Bytes 12-15: Num Dimensions : ").append(d).append(LS); 128 | 129 | if (!isEmpty) { 130 | sb.append("Bytes 16-23: Items Seen(n) : ").append(n).append(LS); 131 | sb.append("Bytes 24-31: SV Adjustment : ").append(svAdjustment).append(LS); 132 | } 133 | 134 | final long numBytes = (long)numRows * d * Double.BYTES; 135 | sb.append("TOTAL Sketch Bytes : ").append(mem.getCapacity()).append(LS) 136 | .append(" Preamble Bytes : ").append(preLongs << 3).append(LS) 137 | .append(" Data Bytes : ").append(numBytes).append(LS) 138 | .append("### END ") 139 | .append(family.getFamilyName().toUpperCase()) 140 | .append(" PREAMBLE SUMMARY").append(LS); 141 | return sb.toString(); 142 | } 143 | 144 | // Extraction methods 145 | 146 | static int extractPreLongs(final Memory mem) { 147 | return mem.getInt(PREAMBLE_LONGS_BYTE) & 0xFF; 148 | } 149 | 150 | static int extractSerVer(final Memory mem) { 151 | return mem.getInt(SER_VER_BYTE) & 0xFF; 152 | } 153 | 154 | static int extractFamilyID(final Memory mem) { 155 | return mem.getByte(FAMILY_BYTE) & 0xFF; 156 | } 157 | 158 | static int extractFlags(final Memory mem) { 159 | return mem.getByte(FLAGS_BYTE) & 0xFF; 160 | } 161 | 162 | static int extractK(final Memory mem) { 163 | return mem.getInt(K_INT); 164 | } 165 | 166 | static int extractNumRows(final Memory mem) { 167 | return mem.getInt(NUM_ROWS_INT); 168 | } 169 | 170 | static int extractNumColumns(final Memory mem) { 171 | return mem.getInt(NUM_COLUMNS_INT); 172 | } 173 | 174 | static long extractN(final Memory mem) { 175 | return mem.getLong(N_LONG); 176 | } 177 | 178 | static double extractSVAdjustment(final Memory mem) { 179 | return mem.getDouble(SV_ADJUSTMENT_DOUBLE); 180 | } 181 | 182 | 183 | // Insertion methods 184 | 185 | static void insertPreLongs(final Object memObj, final long memAddr, final int preLongs) { 186 | unsafe.putByte(memObj, memAddr + PREAMBLE_LONGS_BYTE, (byte) preLongs); 187 | } 188 | 189 | static void insertSerVer(final Object memObj, final long memAddr, final int serVer) { 190 | unsafe.putByte(memObj, memAddr + SER_VER_BYTE, (byte) serVer); 191 | } 192 | 193 | static void insertFamilyID(final Object memObj, final long memAddr, final int matrixFamId) { 194 | unsafe.putByte(memObj, memAddr + FAMILY_BYTE, (byte) matrixFamId); 195 | } 196 | 197 | static void insertFlags(final Object memObj, final long memAddr, final int flags) { 198 | unsafe.putByte(memObj, memAddr + FLAGS_BYTE, (byte) flags); 199 | } 200 | 201 | static void insertK(final Object memObj, final long memAddr, final int k) { 202 | unsafe.putInt(memObj, memAddr + K_INT, k); 203 | } 204 | 205 | static void insertNumRows(final Object memObj, final long memAddr, final int numRows) { 206 | unsafe.putInt(memObj, memAddr + NUM_ROWS_INT, numRows); 207 | } 208 | 209 | static void insertNumColumns(final Object memObj, final long memAddr, final int numColumns) { 210 | unsafe.putInt(memObj, memAddr + NUM_COLUMNS_INT, numColumns); 211 | } 212 | 213 | static void insertN(final Object memObj, final long memAddr, final long n) { 214 | unsafe.putLong(memObj, memAddr + N_LONG, n); 215 | } 216 | 217 | static void insertSVAdjustment(final Object memObj, final long memAddr, final double adj) { 218 | unsafe.putDouble(memObj, memAddr + SV_ADJUSTMENT_DOUBLE, adj); 219 | } 220 | 221 | 222 | /** 223 | * Checks Memory for capacity to hold the preamble and returns the extracted preLongs. 224 | * @param mem the given Memory 225 | * @return the extracted prelongs value. 226 | */ 227 | static int getAndCheckPreLongs(final Memory mem) { 228 | final long cap = mem.getCapacity(); 229 | if (cap < Long.BYTES) { throwNotBigEnough(cap, Long.BYTES); } 230 | final int preLongs = extractPreLongs(mem); 231 | final int required = Math.max(preLongs << 3, Long.BYTES); 232 | if (cap < required) { throwNotBigEnough(cap, required); } 233 | return preLongs; 234 | } 235 | 236 | private static void throwNotBigEnough(final long cap, final int required) { 237 | throw new IllegalArgumentException( 238 | "Possible Corruption: Size of byte array or Memory not large enough: Size: " + cap 239 | + ", Required: " + required); 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/decomposition/SVDAlgo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.decomposition; 21 | 22 | /** 23 | * This class allows a choice of algorithms for Singular Value Decomposition. The options are: 24 | *
This package is dedicated to streaming algorithms that enable approximate matrix 22 | * decompositions.
23 | * 24 | *These sketches are mergeable and can be serialized and deserialized to/from a compact 25 | * form.
26 | */ 27 | package org.apache.datasketches.vector.decomposition; 28 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/Matrix.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.LS; 23 | 24 | import org.ojalgo.matrix.store.Primitive64Store; 25 | 26 | import org.apache.datasketches.memory.Memory; 27 | import org.apache.datasketches.vector.MatrixFamily; 28 | 29 | /** 30 | * Provides an implementation-agnostic wrapper around Matrix classes. 31 | * 32 | * @author Jon Malkin 33 | */ 34 | public abstract class Matrix { 35 | int numRows_; 36 | int numCols_; 37 | 38 | /** 39 | * Loads matrix from srcMem, assuming storage in column-major order to ensure portability. 40 | * Does not necessarily encode matrix size; do not expect size checks based on passed-in 41 | * parameters. 42 | * 43 | * @param srcMem Memory wrapping the matrix 44 | * @param type Matrix implementation type to use 45 | * @return The heapified matrix 46 | */ 47 | public static Matrix heapify(final Memory srcMem, final MatrixType type) { 48 | switch (type) { 49 | case OJALGO: 50 | return MatrixImplOjAlgo.heapifyInstance(srcMem); 51 | default: 52 | return null; 53 | } 54 | } 55 | 56 | /** 57 | * Wraps an object without allocating memory. This method will throw an exception if the mtx 58 | * Object is not of the same type as the implementing class's native format. 59 | * @param mtx Matrix object to wrap 60 | * @return A Matrix object 61 | */ 62 | public static Matrix wrap(final Object mtx) { 63 | if (mtx == null) { 64 | return null; 65 | } else if (mtx instanceof Primitive64Store) { 66 | return MatrixImplOjAlgo.wrap((Primitive64Store) mtx); 67 | } 68 | else { 69 | throw new IllegalArgumentException("wrap() does not currently support " 70 | + mtx.getClass().toString()); 71 | } 72 | } 73 | 74 | /** 75 | * Gets a builder to be able to create instances of Matrix objects 76 | * @return a MatrixBuilder object 77 | */ 78 | public static MatrixBuilder builder() { 79 | return new MatrixBuilder(); 80 | } 81 | 82 | /** 83 | * Returns the raw data object backing this Matrix, as an Object. Must be cast to the 84 | * appropriate type (assuming knowledge of the implementation) to be used. 85 | * @return An Object pointing to the raw data backing this Matrix 86 | */ 87 | public abstract Object getRawObject(); 88 | 89 | /** 90 | * Serializes the Matrix in a custom format as a byte array 91 | * @return A byte[] conttaining a serialized Matrix 92 | */ 93 | public abstract byte[] toByteArray(); 94 | 95 | /** 96 | * Serializes a sub-Matrix by storing only the first numRows and numCols rows and columns, 97 | * respsectively. 98 | * @param numRows Number of rows to write 99 | * @param numCols Number of columns to write 100 | * @return A byte[] containing the serialized sub-Matrix. 101 | */ 102 | public abstract byte[] toCompactByteArray(int numRows, int numCols); 103 | 104 | /** 105 | * Returns a single element from the Matrix 106 | * @param row Row index of target element (0-based) 107 | * @param col Column index of target elemtn (0-based) 108 | * @return Matrix value at (row, column) 109 | */ 110 | public abstract double getElement(int row, int col); 111 | 112 | /** 113 | * Returns a copy of an entire row of the Matrix 114 | * @param row Row index to return (0-based) 115 | * @return A double[] representing the Matrix row 116 | */ 117 | public abstract double[] getRow(int row); 118 | 119 | /** 120 | * Returns a copy of an entire column of the Matrix 121 | * @param col Column index to return (0-based) 122 | * @return A double[] representing the Matrix column 123 | */ 124 | public abstract double[] getColumn(int col); 125 | 126 | /** 127 | * Sets a single element inthe Matrix 128 | * @param row Row index of target element (0-based) 129 | * @param col Column index of target element (0-based) 130 | * @param value The value to insert into the Matrix at (row, column) 131 | */ 132 | public abstract void setElement(int row, int col, double value); 133 | 134 | /** 135 | * Sets an entire row of the Matrix, by copying data from the input 136 | * @param row Target row index (0-based) 137 | * @param values Array of values to write into the Matrix 138 | */ 139 | public abstract void setRow(int row, double[] values); 140 | 141 | /** 142 | * Sets an entire column of the Matrix, by copying data from the input 143 | * @param column Target column index (0-based) 144 | * @param values Array of values to write into the Matrix 145 | */ 146 | public abstract void setColumn(int column, double[] values); 147 | 148 | /** 149 | * Gets the number of rows in the Matrix 150 | * @return Configured number of rows in the Matrix 151 | */ 152 | public long getNumRows() { return numRows_; } 153 | 154 | /** 155 | * Gets the number of columns in the Matrix 156 | * @return Configured number of columns in the Matrix 157 | */ 158 | public long getNumColumns() { return numCols_; } 159 | 160 | /** 161 | * Gets serialized size of the Matrix, in bytes. 162 | * @return Number of bytes needed for a serialized Matrix 163 | */ 164 | public int getSizeBytes() { 165 | final int preBytes = MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES; 166 | final int mtxBytes = (numRows_ * numCols_) * Double.BYTES; 167 | return preBytes + mtxBytes; 168 | } 169 | 170 | /** 171 | * Gets serialized size of the Matrix in compact form, in bytes. 172 | * @param rows Number of rows to select for writing 173 | * @param cols Number of columns to select for writing 174 | * @return Number of bytes needed to serialize the first (rows, cols) of this Matrix 175 | */ 176 | public int getCompactSizeBytes(final int rows, final int cols) { 177 | final int nRows = Math.min(rows, numRows_); 178 | final int nCols = Math.min(cols, numCols_); 179 | 180 | if ((nRows < 1) || (nCols < 1)) { 181 | return MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES; 182 | } else if ((nRows == numRows_) && (nCols == numCols_)) { 183 | return getSizeBytes(); 184 | } 185 | 186 | final int preBytes = MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES; 187 | final int mtxBytes = (nRows * nCols) * Double.BYTES; 188 | return preBytes + mtxBytes; 189 | } 190 | 191 | /** 192 | * Writes information about this Matrix to a String. 193 | * @return A human-readable representation of a Matrix 194 | */ 195 | @Override 196 | public String toString() { 197 | final StringBuilder sb = new StringBuilder(); 198 | 199 | sb.append(" Matrix data :").append(LS); 200 | sb.append(this.getClass().getName()); 201 | sb.append(" < ").append(numRows_).append(" x ").append(numCols_).append(" >"); 202 | 203 | // First element 204 | sb.append("\n{ { ").append(getElement(0, 0)); 205 | 206 | // Rest of the first row 207 | for (int j = 1; j < numCols_; j++) { 208 | sb.append(",\t").append(getElement(0, j)); 209 | } 210 | 211 | // For each of the remaining rows 212 | for (int i = 1; i < numRows_; i++) { 213 | 214 | // First column 215 | sb.append(" },\n{ ").append(getElement(i, 0)); 216 | 217 | // Remaining columns 218 | for (int j = 1; j < numCols_; j++) { 219 | sb.append(",\t").append(getElement(i, j)); 220 | } 221 | } 222 | 223 | // Finish 224 | sb.append(" } }").append(LS); 225 | 226 | return sb.toString(); 227 | } 228 | 229 | /** 230 | * Gets the matrix type 231 | * @return the matrix type 232 | */ 233 | public abstract MatrixType getMatrixType(); 234 | } 235 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/MatrixBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | /** 23 | * Provides a builder for Matrix objects. 24 | */ 25 | public class MatrixBuilder { 26 | 27 | private MatrixType type_ = MatrixType.OJALGO; // default type 28 | 29 | /** 30 | * Default no-op constructor. 31 | */ 32 | public MatrixBuilder() {} 33 | 34 | /** 35 | * Sets the underlying type of object to use with any Matrix objects created. 36 | * @param type One of the supported types 37 | * @return This MatrixBuilder object 38 | */ 39 | public MatrixBuilder setType(final MatrixType type) { 40 | type_ = type; 41 | return this; 42 | } 43 | 44 | /** 45 | * Returns a value from an enum defining the type of object backing any Matrix objects created. 46 | * @return An item from the MatrixType enum. 47 | */ 48 | public MatrixType getBackingType() { 49 | return type_; 50 | } 51 | 52 | /** 53 | * Instantiates a new, empty matrix of the target size 54 | * 55 | * @param numRows Number of rows in matrix 56 | * @param numCols Number of columns in matrix 57 | * @return An empty matrix of the requested size 58 | */ 59 | public Matrix build(final int numRows, final int numCols) { 60 | switch (type_) { 61 | case OJALGO: 62 | return MatrixImplOjAlgo.newInstance(numRows, numCols); 63 | 64 | default: 65 | throw new IllegalArgumentException("OJALGO is currently the only supported MatrixTypes"); 66 | } 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/MatrixImplOjAlgo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.COMPACT_FLAG_MASK; 23 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractFamilyID; 24 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractFlags; 25 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractNumColumns; 26 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractNumColumnsUsed; 27 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractNumRows; 28 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractNumRowsUsed; 29 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractPreLongs; 30 | import static org.apache.datasketches.vector.matrix.MatrixPreambleUtil.extractSerVer; 31 | 32 | import org.ojalgo.matrix.store.Primitive64Store; 33 | 34 | import org.apache.datasketches.memory.Memory; 35 | import org.apache.datasketches.memory.WritableMemory; 36 | import org.apache.datasketches.vector.MatrixFamily; 37 | 38 | /** 39 | * Implements the ojAlgo Matrix operations. 40 | */ 41 | public final class MatrixImplOjAlgo extends Matrix { 42 | private Primitive64Store mtx_; 43 | 44 | private MatrixImplOjAlgo(final int numRows, final int numCols) { 45 | mtx_ = Primitive64Store.FACTORY.make(numRows, numCols); 46 | numRows_ = numRows; 47 | numCols_ = numCols; 48 | } 49 | 50 | private MatrixImplOjAlgo(final Primitive64Store mtx) { 51 | mtx_ = mtx; 52 | numRows_ = (int) mtx.countRows(); 53 | numCols_ = (int) mtx.countColumns(); 54 | } 55 | 56 | static Matrix newInstance(final int numRows, final int numCols) { 57 | return new MatrixImplOjAlgo(numRows, numCols); 58 | } 59 | 60 | static Matrix heapifyInstance(final Memory srcMem) { 61 | final int minBytes = MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES; 62 | final long memCapBytes = srcMem.getCapacity(); 63 | if (memCapBytes < minBytes) { 64 | throw new IllegalArgumentException("Source Memory too small: " + memCapBytes 65 | + " < " + minBytes); 66 | } 67 | 68 | final int preLongs = extractPreLongs(srcMem); 69 | final int serVer = extractSerVer(srcMem); 70 | final int familyID = extractFamilyID(srcMem); 71 | 72 | if (serVer != 1) { 73 | throw new IllegalArgumentException("Invalid SerVer reading srcMem. Expected 1, found: " 74 | + serVer); 75 | } 76 | if (familyID != MatrixFamily.MATRIX.getID()) { 77 | throw new IllegalArgumentException("srcMem does not point to a Matrix"); 78 | } 79 | 80 | final int flags = extractFlags(srcMem); 81 | final boolean isCompact = (flags & COMPACT_FLAG_MASK) > 0; 82 | 83 | int nRows = extractNumRows(srcMem); 84 | int nCols = extractNumColumns(srcMem); 85 | 86 | final MatrixImplOjAlgo matrix = new MatrixImplOjAlgo(nRows, nCols); 87 | if (isCompact) { 88 | nRows = extractNumRowsUsed(srcMem); 89 | nCols = extractNumColumnsUsed(srcMem); 90 | } 91 | 92 | int memOffset = preLongs * Long.BYTES; 93 | for (int c = 0; c < nCols; ++c) { 94 | for (int r = 0; r < nRows; ++r) { 95 | matrix.mtx_.set(r, c, srcMem.getDouble(memOffset)); 96 | memOffset += Double.BYTES; 97 | } 98 | } 99 | 100 | return matrix; 101 | } 102 | 103 | static Matrix wrap(final Primitive64Store mtx) { 104 | return new MatrixImplOjAlgo(mtx); 105 | } 106 | 107 | @Override 108 | public Object getRawObject() { 109 | return mtx_; 110 | } 111 | 112 | @Override 113 | public byte[] toByteArray() { 114 | final int preLongs = 2; 115 | final long numElements = mtx_.count(); 116 | assert numElements == (mtx_.countColumns() * mtx_.countRows()); 117 | 118 | final int outBytes = (int) (((long)preLongs * Long.BYTES) + (numElements * Double.BYTES)); 119 | final byte[] outByteArr = new byte[outBytes]; 120 | final WritableMemory memOut = WritableMemory.wrap(outByteArr); 121 | final Object memObj = memOut.getArray(); 122 | final long memAddr = memOut.getCumulativeOffset(0L); 123 | 124 | MatrixPreambleUtil.insertPreLongs(memObj, memAddr, preLongs); 125 | MatrixPreambleUtil.insertSerVer(memObj, memAddr, MatrixPreambleUtil.SER_VER); 126 | MatrixPreambleUtil.insertFamilyID(memObj, memAddr, MatrixFamily.MATRIX.getID()); 127 | MatrixPreambleUtil.insertFlags(memObj, memAddr, 0); 128 | MatrixPreambleUtil.insertNumRows(memObj, memAddr, (int) mtx_.countRows()); 129 | MatrixPreambleUtil.insertNumColumns(memObj, memAddr, (int) mtx_.countColumns()); 130 | memOut.putDoubleArray(preLongs << 3, mtx_.data, 0, (int) numElements); 131 | 132 | return outByteArr; 133 | } 134 | 135 | @Override 136 | public byte[] toCompactByteArray(final int numRows, final int numCols) { 137 | // TODO: row/col limit checks 138 | 139 | final int preLongs = 3; 140 | 141 | // for non-compact we can do an array copy, so save as non-compact if using the entire matrix 142 | final long numElements = (long) numRows * numCols; 143 | final boolean isCompact = numElements < mtx_.count(); 144 | if (!isCompact) { 145 | return toByteArray(); 146 | } 147 | 148 | assert numElements < mtx_.count(); 149 | 150 | final int outBytes = (int) (((long)preLongs * Long.BYTES) + (numElements * Double.BYTES)); 151 | final byte[] outByteArr = new byte[outBytes]; 152 | final WritableMemory memOut = WritableMemory.wrap(outByteArr); 153 | final Object memObj = memOut.getArray(); 154 | final long memAddr = memOut.getCumulativeOffset(0L); 155 | 156 | MatrixPreambleUtil.insertPreLongs(memObj, memAddr, preLongs); 157 | MatrixPreambleUtil.insertSerVer(memObj, memAddr, MatrixPreambleUtil.SER_VER); 158 | MatrixPreambleUtil.insertFamilyID(memObj, memAddr, MatrixFamily.MATRIX.getID()); 159 | MatrixPreambleUtil.insertFlags(memObj, memAddr, COMPACT_FLAG_MASK); 160 | MatrixPreambleUtil.insertNumRows(memObj, memAddr, (int) mtx_.countRows()); 161 | MatrixPreambleUtil.insertNumColumns(memObj, memAddr, (int) mtx_.countColumns()); 162 | MatrixPreambleUtil.insertNumRowsUsed(memObj, memAddr, numRows); 163 | MatrixPreambleUtil.insertNumColumnsUsed(memObj, memAddr, numCols); 164 | 165 | // write elements in column-major order 166 | long offsetBytes = (long)preLongs * Long.BYTES; 167 | for (int c = 0; c < numCols; ++c) { 168 | for (int r = 0; r < numRows; ++r) { 169 | memOut.putDouble(offsetBytes, mtx_.doubleValue(r, c)); 170 | offsetBytes += Double.BYTES; 171 | } 172 | } 173 | 174 | return outByteArr; 175 | } 176 | 177 | @Override 178 | public double getElement(final int row, final int col) { 179 | return mtx_.doubleValue(row, col); 180 | } 181 | 182 | @Override 183 | public double[] getRow(final int row) { 184 | final int cols = (int) mtx_.countColumns(); 185 | final double[] result = new double[cols]; 186 | for (int c = 0; c < cols; ++c) { 187 | result[c] = mtx_.doubleValue(row, c); 188 | } 189 | return result; 190 | } 191 | 192 | @Override 193 | public double[] getColumn(final int col) { 194 | final int rows = (int) mtx_.countRows(); 195 | final double[] result = new double[rows]; 196 | for (int r = 0; r < rows; ++r) { 197 | result[r] = mtx_.doubleValue(r, col); 198 | } 199 | return result; 200 | } 201 | 202 | @Override 203 | public void setElement(final int row, final int col, final double value) { 204 | mtx_.set(row, col, value); 205 | } 206 | 207 | @Override 208 | public void setRow(final int row, final double[] values) { 209 | if (values.length != mtx_.countColumns()) { 210 | throw new IllegalArgumentException("Invalid number of elements for row. Expected " 211 | + mtx_.countColumns() + ", found " + values.length); 212 | } 213 | 214 | for (int i = 0; i < mtx_.countColumns(); ++i) { 215 | mtx_.set(row, i, values[i]); 216 | } 217 | } 218 | 219 | @Override 220 | public void setColumn(final int column, final double[] values) { 221 | if (values.length != mtx_.countRows()) { 222 | throw new IllegalArgumentException("Invalid number of elements for column. Expected " 223 | + mtx_.countRows() + ", found " + values.length); 224 | } 225 | 226 | for (int i = 0; i < mtx_.countRows(); ++i) { 227 | mtx_.set(i, column, values[i]); 228 | } 229 | } 230 | 231 | @Override 232 | public MatrixType getMatrixType() { 233 | return MatrixType.OJALGO; 234 | } 235 | 236 | } 237 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/MatrixPreambleUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.apache.datasketches.memory.UnsafeUtil.unsafe; 23 | 24 | import org.apache.datasketches.memory.Memory; 25 | import org.apache.datasketches.vector.MatrixFamily; 26 | 27 | /** 28 | * This class defines the preamble items structure and provides basic utilities for some of the 29 | * key fields for a Matrix 30 | * 31 | *32 | * The low significance bytes of this long items structure are on the right. Multi-byte 33 | * integers (int and long) are stored in native byte order. All byte 34 | * values are treated as unsigned.
35 | * 36 | *An empty or non-compact Matrix requires 16 bytes. A compact under-full matrix requires 37 | * 24 bytes of preamble.
38 | * 39 | *40 | * Long || Start Byte Adr: 41 | * Adr: 42 | * || 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 | 43 | * 0 ||-------------Reserved--------------| Flags | FamID | SerVer | Preamble_Longs | 44 | * 45 | * || 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | 46 | * 1 ||-----------Num. Columns------------|-------------Num. Rows-------------------------| 47 | * 48 | * || 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 | 49 | * 2 ||---------Num. Columns Used---------|----------Num. Rows Used-----------------------| 50 | *51 | * 52 | * @author Jon Malkin 53 | */ 54 | @SuppressWarnings("restriction") 55 | public final class MatrixPreambleUtil { 56 | 57 | /** 58 | * The java line separator character as a String. 59 | */ 60 | static final String LS = System.getProperty("line.separator"); 61 | 62 | private MatrixPreambleUtil() {} 63 | 64 | // ###### DO NOT MESS WITH THIS FROM HERE ... 65 | // Preamble byte Addresses 66 | private static final int PREAMBLE_LONGS_BYTE = 0; 67 | private static final int SER_VER_BYTE = 1; 68 | private static final int FAMILY_BYTE = 2; 69 | private static final int FLAGS_BYTE = 3; 70 | private static final int NUM_ROWS_INT = 8; 71 | private static final int NUM_COLUMNS_INT = 12; 72 | private static final int ROWS_USED_INT = 16; 73 | private static final int COLS_USED_INT = 20; 74 | 75 | // flag bit masks 76 | //static final int EMPTY_FLAG_MASK = 4; 77 | static final int COMPACT_FLAG_MASK = 8; 78 | 79 | // Other constants 80 | static final int SER_VER = 1; 81 | 82 | /** 83 | * Returns a human readable string summary of the preamble state of the given Memory. 84 | * Note: other than making sure that the given Memory size is large 85 | * enough for just the preamble, this does not do much value checking of the contents of the 86 | * preamble as this is primarily a tool for debugging the preamble visually. 87 | * 88 | * @param mem the given Memory. 89 | * @return the summary preamble string. 90 | */ 91 | public static String preambleToString(final Memory mem) { 92 | 93 | final int preLongs = getAndCheckPreLongs(mem); // make sure we can get the assumed preamble 94 | final MatrixFamily family = MatrixFamily.idToFamily(extractFamilyID(mem)); 95 | 96 | final int serVer = extractSerVer(mem); 97 | if (serVer != SER_VER) { 98 | throw new IllegalArgumentException("Invalid serialization version in memory region. " 99 | + "Found: " + serVer); 100 | } 101 | 102 | final int flags = extractFlags(mem); 103 | final String flagsStr = Integer.toBinaryString(flags) + ", " + flags; 104 | //final boolean isEmpty = (flags & EMPTY_FLAG_MASK) > 0; 105 | final boolean isCompact = (flags & COMPACT_FLAG_MASK) > 0; 106 | 107 | final int numRows = extractNumRows(mem); 108 | final int numCols = extractNumColumns(mem); 109 | 110 | int numRowsUsed = numRows; 111 | int numColsUsed = numCols; 112 | if (isCompact) { 113 | numRowsUsed = extractNumRowsUsed(mem); 114 | numColsUsed = extractNumColumnsUsed(mem); 115 | } 116 | 117 | final StringBuilder sb = new StringBuilder(); 118 | sb.append(LS) 119 | .append("### START ") 120 | .append(family.getFamilyName().toUpperCase()) 121 | .append(" PREAMBLE SUMMARY").append(LS) 122 | .append("Byte 0: Preamble Longs : ").append(preLongs).append(LS) 123 | .append("Byte 1: Serialization Version: ").append(serVer).append(LS) 124 | .append("Byte 2: Family : ").append(family.toString()).append(LS) 125 | .append("Byte 3: Flags Field : ").append(flagsStr).append(LS) 126 | //.append(" EMPTY : ").append(isEmpty).append(LS) 127 | .append(" COMPACT : ").append(isCompact).append(LS) 128 | .append("Bytes 8-11: Num Rows : ").append(numRows).append(LS) 129 | .append("Bytes 12-15: Num Columns : ").append(numCols).append(LS); 130 | 131 | if (isCompact) { 132 | sb.append("Bytes 16-23: Num Rows Used : ").append(numRowsUsed).append(LS); 133 | sb.append("Bytes 24-31: Num Columns Used : ").append(numColsUsed).append(LS); 134 | } 135 | 136 | return sb.toString(); 137 | } 138 | 139 | // Extraction methods 140 | 141 | static int extractPreLongs(final Memory mem) { 142 | return mem.getInt(PREAMBLE_LONGS_BYTE) & 0xFF; 143 | } 144 | 145 | static int extractSerVer(final Memory mem) { 146 | return mem.getInt(SER_VER_BYTE) & 0xFF; 147 | } 148 | 149 | static int extractFamilyID(final Memory mem) { 150 | return mem.getByte(FAMILY_BYTE) & 0xFF; 151 | } 152 | 153 | static int extractFlags(final Memory mem) { 154 | return mem.getByte(FLAGS_BYTE) & 0xFF; 155 | } 156 | 157 | static int extractNumRows(final Memory mem) { 158 | return mem.getInt(NUM_ROWS_INT); 159 | } 160 | 161 | static int extractNumColumns(final Memory mem) { 162 | return mem.getInt(NUM_COLUMNS_INT); 163 | } 164 | 165 | static int extractNumRowsUsed(final Memory mem) { 166 | return mem.getInt(ROWS_USED_INT); 167 | } 168 | 169 | static int extractNumColumnsUsed(final Memory mem) { 170 | return mem.getInt(COLS_USED_INT); 171 | } 172 | 173 | // Insertion methods 174 | 175 | static void insertPreLongs(final Object memObj, final long memAddr, final int preLongs) { 176 | unsafe.putByte(memObj, memAddr + PREAMBLE_LONGS_BYTE, (byte) preLongs); 177 | } 178 | 179 | static void insertSerVer(final Object memObj, final long memAddr, final int serVer) { 180 | unsafe.putByte(memObj, memAddr + SER_VER_BYTE, (byte) serVer); 181 | } 182 | 183 | static void insertFamilyID(final Object memObj, final long memAddr, final int matrixFamId) { 184 | unsafe.putByte(memObj, memAddr + FAMILY_BYTE, (byte) matrixFamId); 185 | } 186 | 187 | static void insertFlags(final Object memObj, final long memAddr, final int flags) { 188 | unsafe.putByte(memObj, memAddr + FLAGS_BYTE, (byte) flags); 189 | } 190 | 191 | static void insertNumRows(final Object memObj, final long memAddr, final int numRows) { 192 | unsafe.putInt(memObj, memAddr + NUM_ROWS_INT, numRows); 193 | } 194 | 195 | static void insertNumColumns(final Object memObj, final long memAddr, final int numColumns) { 196 | unsafe.putInt(memObj, memAddr + NUM_COLUMNS_INT, numColumns); 197 | } 198 | 199 | static void insertNumRowsUsed(final Object memObj, final long memAddr, final int rowsUsed) { 200 | unsafe.putInt(memObj, memAddr + ROWS_USED_INT, rowsUsed); 201 | } 202 | 203 | static void insertNumColumnsUsed(final Object memObj, final long memAddr, final int columnsUsed) { 204 | unsafe.putInt(memObj, memAddr + COLS_USED_INT, columnsUsed); 205 | } 206 | 207 | 208 | /** 209 | * Checks Memory for capacity to hold the preamble and returns the extracted preLongs. 210 | * @param mem the given Memory 211 | * @return the extracted prelongs value. 212 | */ 213 | private static int getAndCheckPreLongs(final Memory mem) { 214 | final long cap = mem.getCapacity(); 215 | if (cap < Long.BYTES) { throwNotBigEnough(cap, Long.BYTES); } 216 | final int preLongs = extractPreLongs(mem); 217 | final int required = Math.max(preLongs << 3, Long.BYTES); 218 | if (cap < required) { throwNotBigEnough(cap, required); } 219 | return preLongs; 220 | } 221 | 222 | private static void throwNotBigEnough(final long cap, final int required) { 223 | throw new IllegalArgumentException( 224 | "Possible Corruption: Size of byte array or Memory not large enough: Size: " + cap 225 | + ", Required: " + required); 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/MatrixType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | /** 23 | * The MatrixType enum 24 | */ 25 | public enum MatrixType { 26 | /** 27 | * Select the ojAlgo 28 | */ 29 | OJALGO(1, "ojAlgo"); 30 | 31 | private int id_; 32 | private String name_; 33 | 34 | MatrixType(final int id, final String name) { 35 | id_ = id; 36 | name_ = name; 37 | } 38 | 39 | /** 40 | * Gets the ID 41 | * @return the ID 42 | */ 43 | public int getId() { return id_; } 44 | 45 | /** 46 | * Gets the name 47 | * @return the name 48 | */ 49 | public String getName() { return name_; } 50 | 51 | @Override 52 | public String toString() { return name_; } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/matrix/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | *
This package contains a Matrix class that wraps one of several underlying matrix 22 | * implementations. It can be used to provide a stable public API independent of the 23 | * specific linear algebra package used for computation.
24 | * 25 | *These Matrix objects can be serialized and deserialized to/from a compact form.
26 | */ 27 | package org.apache.datasketches.vector.matrix; 28 | -------------------------------------------------------------------------------- /src/main/java/org/apache/datasketches/vector/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | /** 21 | * This is the parent package for all vector sketch algorithms. Any classes at this level are 22 | * used by more than one sub-package. 23 | */ 24 | package org.apache.datasketches.vector; 25 | -------------------------------------------------------------------------------- /src/test/java/org/apache/datasketches/vector/decomposition/FrequentDirectionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.decomposition; 21 | 22 | import static org.testng.Assert.assertEquals; 23 | import static org.testng.Assert.assertNotNull; 24 | import static org.testng.Assert.assertNull; 25 | import static org.testng.Assert.assertTrue; 26 | import static org.testng.Assert.fail; 27 | 28 | import java.util.Arrays; 29 | 30 | import org.testng.annotations.Test; 31 | 32 | import org.apache.datasketches.memory.Memory; 33 | import org.apache.datasketches.memory.WritableMemory; 34 | import org.apache.datasketches.vector.MatrixFamily; 35 | import org.apache.datasketches.vector.matrix.Matrix; 36 | 37 | @SuppressWarnings("javadoc") 38 | public class FrequentDirectionsTest { 39 | 40 | @Test 41 | public void instantiateFD() { 42 | final int k = 32; 43 | final int d = 256; 44 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 45 | 46 | assertNotNull(fd); 47 | assertTrue(fd.isEmpty()); 48 | assertEquals(fd.getK(), k); 49 | assertEquals(fd.getD(), d); 50 | assertEquals(fd.getN(), 0); 51 | assertNull(fd.getResult()); 52 | 53 | // error conditions 54 | // d = 0 55 | try { 56 | FrequentDirections.newInstance(k, 0); 57 | fail(); 58 | } catch (final IllegalArgumentException e) { 59 | // expected 60 | } 61 | 62 | // k = -1 63 | try { 64 | FrequentDirections.newInstance(-1, d); 65 | fail(); 66 | } catch (final IllegalArgumentException e) { 67 | // expected 68 | } 69 | 70 | // d < 2k (not handled in reduceRank() 71 | try { 72 | FrequentDirections.newInstance(d, d); 73 | fail(); 74 | } catch (final IllegalArgumentException e) { 75 | // expected 76 | } 77 | } 78 | 79 | @Test 80 | public void checkSymmUpdate() { 81 | final int k = 4; 82 | final int d = 16; // should be > 2k 83 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 84 | fd.setSVDAlgo(SVDAlgo.SYM); // default, but force anyway 85 | 86 | runUpdateTest(fd); 87 | } 88 | 89 | @Test 90 | public void checkFullSVDUpdate() { 91 | final int k = 4; 92 | final int d = 16; // should be > 2k 93 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 94 | fd.setSVDAlgo(SVDAlgo.FULL); 95 | 96 | runUpdateTest(fd); 97 | } 98 | 99 | private static void runUpdateTest(final FrequentDirections fd) { 100 | final int k = fd.getK(); 101 | final int d = fd.getD(); 102 | 103 | // creates matrix with increasing values along diagonal 104 | final double[] input = new double[d]; 105 | for (int i = 0; i < (2 * k); ++i) { 106 | if (i > 0) { 107 | input[i - 1] = 0.0; 108 | } 109 | input[i] = i * 1.0; 110 | fd.update(input); 111 | } 112 | fd.update((double[]) null); // should be a no-op and not impact next lines 113 | assertEquals(fd.getNumRows(), 2 * k); 114 | assertEquals(fd.getN(), 2 * k); 115 | 116 | input[(2 * k) - 1] = 0.0; 117 | input[2 * k] = 2.0 * k; 118 | fd.update(input); // trigger reduceRank(), then add 1 more row 119 | assertEquals(fd.getNumRows(), k); 120 | } 121 | 122 | 123 | @Test 124 | public void updateWithTooFewDimensions() { 125 | final int k = 4; 126 | final int d = 16; // should be > 2k 127 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 128 | 129 | final double[] input = new double[d - 3]; 130 | try { 131 | fd.update(input); 132 | fail(); 133 | } catch (final IllegalArgumentException e) { 134 | // expected 135 | } 136 | } 137 | 138 | @Test 139 | public void mergeSketches() { 140 | final int k = 5; 141 | final int d = 12; // should be > 2k 142 | final int initialRows = 7; 143 | final FrequentDirections fd1 = FrequentDirections.newInstance(k, d); 144 | final FrequentDirections fd2 = FrequentDirections.newInstance(k, d); 145 | 146 | // two diagonal matrices 147 | final double[] input = new double[d]; 148 | for (int i = 0; i < initialRows; ++i) { 149 | if (i > 0) { 150 | input[i - 1] = 0.0; 151 | } 152 | //input[i] = (2 * k) - (i * 1.0); 153 | input[i] = i * 1.0; 154 | fd1.update(input); 155 | 156 | input[i] = (i * 1.0) - (2 * k); 157 | fd2.update(input); 158 | } 159 | 160 | // the next two lines are no-ops 161 | fd1.update((FrequentDirections) null); 162 | fd1.update(FrequentDirections.newInstance(k, d)); 163 | assertEquals(fd1.getNumRows(), initialRows); 164 | assertEquals(fd1.getN(), initialRows); 165 | 166 | assertEquals(fd2.getNumRows(), initialRows); 167 | assertEquals(fd2.getN(), initialRows); 168 | 169 | fd1.update(fd2); 170 | final int expectedRows = (((2 * initialRows) % k) + k) - 1; // assumes 2 * initialRows > k 171 | assertEquals(fd1.getNumRows(), expectedRows); 172 | assertEquals(fd1.getN(), 2 * initialRows); 173 | 174 | final Matrix result = fd1.getResult(false); 175 | assertNotNull(result); 176 | assertEquals(result.getNumRows(), 2 * k); 177 | 178 | println(fd1.toString(true, true, true)); 179 | } 180 | 181 | @Test 182 | public void checkCompensativeResultSymSVD() { 183 | final int k = 4; 184 | final int d = 10; // should be > 2k 185 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 186 | fd.setSVDAlgo(SVDAlgo.SYM); 187 | 188 | runCompensativeResultTest(fd); 189 | } 190 | 191 | @Test 192 | public void checkCompensativeResultFullSVD() { 193 | final int k = 4; 194 | final int d = 10; // should be > 2k 195 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 196 | fd.setSVDAlgo(SVDAlgo.FULL); 197 | 198 | runCompensativeResultTest(fd); 199 | } 200 | 201 | private static void runCompensativeResultTest(final FrequentDirections fd) { 202 | final int d = fd.getD(); 203 | final int k = fd.getK(); 204 | 205 | // diagonal matrix for easy checking 206 | final double[] input = new double[d]; 207 | for (int i = 0; i < (k + 1); ++i) { 208 | if (i > 0) { 209 | input[i - 1] = 0.0; 210 | } 211 | input[i] = (i + 1) * 1.0; 212 | fd.update(input); 213 | } 214 | 215 | Matrix m = fd.getResult(); 216 | for (int i = 0; i < (k + 1); ++i) { 217 | assertEquals(m.getElement(i,i), 1.0 * (i + 1), 1e-6); 218 | } 219 | 220 | // without compensation, but force rank reduction and check projection at the same time 221 | fd.forceReduceRank(); 222 | m = fd.getResult(); 223 | final Matrix p = fd.getProjectionMatrix(); 224 | double[] sv = fd.getSingularValues(false); 225 | for (int i = k; i > 1; --i) { 226 | final double val = Math.abs(m.getElement(k - i, i)); 227 | final double expected = Math.sqrt(((i + 1) * (i + 1)) - fd.getSvAdjustment()); 228 | assertEquals(val, expected, 1e-6); 229 | assertEquals(sv[k - i], expected, 1e-10); 230 | assertEquals(Math.abs(p.getElement(k - i, i)), 1.0, 1e-6); 231 | } 232 | assertEquals(m.getElement(k, 1), 0.0, 0.0); // might return -0.0 233 | assertEquals(p.getElement(k, 1), 0.0, 0.0); // might return -0.0 234 | 235 | // with compensation 236 | m = fd.getResult(true); 237 | sv = fd.getSingularValues(true); 238 | for (int i = k; i > 1; --i) { 239 | final double val = Math.abs(m.getElement(k - i, i)); 240 | assertEquals(val, i + 1.0, 1e-6); 241 | assertEquals(sv[k - i], i + 1.0, 1e-10); 242 | } 243 | assertEquals(m.getElement(k, 1), 0.0); 244 | } 245 | 246 | @Test 247 | public void mergeIncompatibleSketches() { 248 | final int k = 5; 249 | final int d = 12; // should be > 2k 250 | final FrequentDirections fd1 = FrequentDirections.newInstance(k, d); 251 | 252 | final double[] input = new double[d]; 253 | input[0] = 1.0; 254 | fd1.update(input); 255 | 256 | // merge in smaller k 257 | FrequentDirections fd2 = FrequentDirections.newInstance(k - 1, d); 258 | fd2.update(input); 259 | try { 260 | fd1.update(fd2); 261 | fail(); 262 | } catch (final IllegalArgumentException e) { 263 | // expected 264 | } 265 | 266 | // mismatch in d 267 | fd2 = FrequentDirections.newInstance(k, d - 1); 268 | fd2.update(new double[d - 1]); 269 | try { 270 | fd1.update(fd2); 271 | fail(); 272 | } catch (final IllegalArgumentException e) { 273 | // expected 274 | } 275 | } 276 | 277 | @Test 278 | public void checkSerialization() { 279 | final int k = 7; 280 | final int d = 20; 281 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 282 | 283 | byte[] sketchBytes = fd.toByteArray(); 284 | assertEquals(sketchBytes.length, 285 | MatrixFamily.FREQUENTDIRECTIONS.getMinPreLongs() * Long.BYTES); 286 | Memory mem = Memory.wrap(sketchBytes); 287 | FrequentDirections rebuilt = FrequentDirections.heapify(mem); 288 | assertTrue(rebuilt.isEmpty()); 289 | assertEquals(rebuilt.getD(), fd.getD()); 290 | assertEquals(rebuilt.getK(), fd.getK()); 291 | 292 | // creates matrix with increasing values along diagonal 293 | // k rows, so shouldn't compress yet 294 | final double[] input = new double[d]; 295 | for (int i = 0; i < k; ++i) { 296 | if (i > 0) { 297 | input[i - 1] = 0.0; 298 | } 299 | //input[i] = (2 * k) - (i * 1.0); 300 | input[i] = i * 1.0; 301 | fd.update(input); 302 | } 303 | sketchBytes = fd.toByteArray(); 304 | mem = Memory.wrap(sketchBytes); 305 | rebuilt = FrequentDirections.heapify(mem); 306 | assertEquals(rebuilt.getN(), fd.getN()); 307 | assertEquals(rebuilt.getD(), fd.getD()); 308 | assertEquals(rebuilt.getK(), fd.getK()); 309 | 310 | // add another k rows and serialize, compressing this time 311 | for (int i = k; i < ((2 * k) - 1); ++i) { 312 | input[i] = i * 1.0; 313 | fd.update(input); 314 | } 315 | assertEquals(fd.getNumRows(), (2 * k) - 1); 316 | sketchBytes = fd.toByteArray(); 317 | mem = Memory.wrap(sketchBytes); 318 | rebuilt = FrequentDirections.heapify(mem); 319 | assertEquals(rebuilt.getN(), fd.getN()); 320 | assertEquals(rebuilt.getNumRows(), fd.getNumRows()); 321 | 322 | println(PreambleUtil.preambleToString(mem)); 323 | } 324 | 325 | @Test 326 | public void checkCorruptedHeapify() { 327 | final int k = 50; 328 | final int d = 250; 329 | final FrequentDirections fd = FrequentDirections.newInstance(k, d); 330 | byte[] sketchBytes = fd.toByteArray(); 331 | WritableMemory mem = WritableMemory.wrap(sketchBytes); 332 | 333 | final FrequentDirections rebuilt = FrequentDirections.heapify(mem); 334 | assertTrue(rebuilt.isEmpty()); 335 | println(PreambleUtil.preambleToString(mem)); 336 | 337 | // corrupt the serialization version 338 | mem.putByte(PreambleUtil.SER_VER_BYTE, (byte) 0); 339 | try { 340 | FrequentDirections.heapify(mem); 341 | } catch (final IllegalArgumentException e) { 342 | // expected 343 | } 344 | 345 | // corrupt the family ID, after grabbing fresh bytes 346 | sketchBytes = fd.toByteArray(); 347 | mem = WritableMemory.wrap(sketchBytes); 348 | mem.putByte(PreambleUtil.FAMILY_BYTE, (byte) 0); 349 | try { 350 | FrequentDirections.heapify(mem); 351 | } catch (final IllegalArgumentException e) { 352 | // expected 353 | } 354 | } 355 | 356 | @Test 357 | public void checkInsufficientMemory() { 358 | // no capacity 359 | byte[] bytes = new byte[0]; 360 | Memory mem = Memory.wrap(bytes); 361 | try { 362 | FrequentDirections.heapify(mem); 363 | } catch (final IllegalArgumentException e) { 364 | // expected 365 | } 366 | 367 | // capacity smaller than prelongs size 368 | final FrequentDirections fd = FrequentDirections.newInstance(10, 50); 369 | bytes = fd.toByteArray(); 370 | bytes = Arrays.copyOf(bytes, bytes.length - 1); 371 | mem = Memory.wrap(bytes); 372 | try { 373 | FrequentDirections.heapify(mem); 374 | } catch (final IllegalArgumentException e) { 375 | // expected 376 | } 377 | } 378 | 379 | /** 380 | * println the message 381 | * @param msg the message 382 | */ 383 | private void println(final String msg) { 384 | //System.out.println(msg); 385 | } 386 | } 387 | -------------------------------------------------------------------------------- /src/test/java/org/apache/datasketches/vector/decomposition/MatrixOpsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.decomposition; 21 | 22 | import static org.testng.Assert.assertEquals; 23 | import static org.testng.Assert.fail; 24 | 25 | import org.testng.annotations.Test; 26 | 27 | import org.apache.datasketches.vector.matrix.Matrix; 28 | import org.apache.datasketches.vector.matrix.MatrixBuilder; 29 | import org.apache.datasketches.vector.matrix.MatrixType; 30 | 31 | @SuppressWarnings("javadoc") 32 | public class MatrixOpsTest { 33 | 34 | @Test 35 | public void compareSVDAccuracy() { 36 | final int d = 10; 37 | final int k = 6; 38 | final Matrix input = generateIncreasingEye(d, 2 * k); 39 | 40 | final MatrixOps moFull = MatrixOps.newInstance(input, SVDAlgo.FULL, k); 41 | final MatrixOps moSym = MatrixOps.newInstance(input, SVDAlgo.SYM, k); 42 | final MatrixOps moSISVD = MatrixOps.newInstance(input, SVDAlgo.SISVD, k); 43 | moSISVD.setNumSISVDIter(50 * k); // intentionally run many extra iterations for tighter convegence 44 | 45 | // just singular values first 46 | moFull.svd(input, false); 47 | moSym.svd(input, false); 48 | moSISVD.svd(input, false); 49 | final double[] fullSv = moFull.getSingularValues(); 50 | compareSingularValues(fullSv, moSym.getSingularValues(), fullSv.length); 51 | compareSingularValues(fullSv, moSISVD.getSingularValues(), k); // SISVD only produces k values 52 | 53 | // now with vectors 54 | moFull.svd(input, true); 55 | moSym.svd(input, true); 56 | moSISVD.svd(input, true); 57 | // TODO: better comparison is vector-wise, testing that sign changes are consistent but that 58 | // requires non-zero elements 59 | final Matrix fullVt = moFull.getVt(); 60 | compareMatrixElementMagnitudes(fullVt, moSym.getVt(), (int) fullVt.getNumRows()); 61 | compareMatrixElementMagnitudes(fullVt, moSISVD.getVt(), k); // SISVD only produces k vectors 62 | 63 | // just to be sure 64 | compareMatrixElementMagnitudes(fullVt, moFull.getVt(input), (int) fullVt.getNumRows()); 65 | } 66 | 67 | @Test 68 | public void checkInvalidMatrixSize() { 69 | final int d = 10; 70 | final int k = 6; 71 | final Matrix A = generateIncreasingEye(d, 2 * k); 72 | final MatrixOps mo = MatrixOps.newInstance(A, SVDAlgo.FULL, k); 73 | 74 | Matrix B = generateIncreasingEye(d, (2 * k) + 1); 75 | try { 76 | mo.svd(B, true); 77 | fail(); 78 | } catch (final IllegalArgumentException e) { 79 | // expected 80 | } 81 | 82 | B = generateIncreasingEye(d - 1, 2 * k); 83 | try { 84 | mo.svd(B, false); 85 | fail(); 86 | } catch (final IllegalArgumentException e) { 87 | // expected 88 | } 89 | 90 | } 91 | 92 | private static void compareSingularValues(final double[] A, final double[] B, final int n) { 93 | assertEquals(A.length, B.length); 94 | 95 | for (int i = 0; i < n; ++i) { 96 | assertEquals(A[i], B[i], 1e-6); 97 | } 98 | } 99 | 100 | 101 | private static void compareMatrixElementMagnitudes(final Matrix A, final Matrix B, final int n) { 102 | assertEquals(A.getNumColumns(), B.getNumColumns()); 103 | assertEquals(A.getNumRows(), B.getNumRows()); 104 | 105 | for (int i = 0; i < n; ++i) { 106 | for (int j = 0; j < A.getNumColumns(); ++j) { 107 | assertEquals(Math.abs(A.getElement(i, j)), Math.abs(B.getElement(i, j)), 1e-6); 108 | } 109 | } 110 | } 111 | 112 | /** 113 | * Creates a scaled I matrix, where the diagonal consists of increasing integers, 114 | * starting with 1.0. 115 | * @param nRows number of rows 116 | * @param nCols number of columns 117 | * @return PrimitiveDenseStore, suitable for direct use or wrapping 118 | */ 119 | private static Matrix generateIncreasingEye(final int nRows, final int nCols) { 120 | final Matrix m = new MatrixBuilder().setType(MatrixType.OJALGO).build(nRows, nCols); 121 | for (int i = 0; (i < nRows) && (i < nCols); ++i) { 122 | m.setElement(i, i, 1.0 + i); 123 | } 124 | return m; 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /src/test/java/org/apache/datasketches/vector/matrix/MatrixBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.testng.Assert.assertEquals; 23 | import static org.testng.Assert.assertNotNull; 24 | 25 | import org.testng.annotations.Test; 26 | 27 | @SuppressWarnings("javadoc") 28 | public class MatrixBuilderTest { 29 | @Test 30 | public void checkBuild() { 31 | final MatrixBuilder builder = new MatrixBuilder(); 32 | assertEquals(builder.getBackingType(), MatrixType.OJALGO); // default type 33 | 34 | Matrix m = builder.build(128, 512); 35 | assertNotNull(m); 36 | 37 | m = builder.build(128, 512); 38 | assertNotNull(m); 39 | } 40 | 41 | @Test 42 | public void checkSetType() { 43 | final MatrixBuilder builder = new MatrixBuilder(); 44 | final MatrixType type = builder.getBackingType(); 45 | assertEquals(type, MatrixType.OJALGO); // default type 46 | assertEquals(type.getId(), MatrixType.OJALGO.getId()); 47 | assertEquals(type.getName(), MatrixType.OJALGO.getName()); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/org/apache/datasketches/vector/matrix/MatrixImplOjAlgoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.testng.Assert.assertEquals; 23 | import static org.testng.Assert.fail; 24 | 25 | import org.ojalgo.matrix.store.Primitive64Store; 26 | import org.testng.annotations.Test; 27 | 28 | import org.apache.datasketches.memory.Memory; 29 | import org.apache.datasketches.memory.WritableMemory; 30 | 31 | @SuppressWarnings("javadoc") 32 | public class MatrixImplOjAlgoTest { 33 | 34 | @Test 35 | public void checkInstantiation() { 36 | final int nRows = 10; 37 | final int nCols = 15; 38 | final Matrix m = MatrixImplOjAlgo.newInstance(nRows, nCols); 39 | assertEquals(m.getNumRows(), nRows); 40 | assertEquals(m.getNumColumns(), nCols); 41 | 42 | final Primitive64Store pds = (Primitive64Store) m.getRawObject(); 43 | assertEquals(pds.countRows(), nRows); 44 | assertEquals(pds.countColumns(), nCols); 45 | 46 | final Matrix wrapped = Matrix.wrap(pds); 47 | MatrixTest.checkMatrixEquality(wrapped, m); 48 | assertEquals(wrapped.getRawObject(), pds); 49 | } 50 | 51 | @Test 52 | public void updateAndQueryValues() { 53 | final int nRows = 5; 54 | final int nCols = 5; 55 | final Matrix m = generateIncreasingEye(nRows, nCols); // tests setElement() in method 56 | 57 | for (int i = 0; i < nRows; ++i) { 58 | for (int j = 0; j < nCols; ++j) { 59 | final double val = m.getElement(i, j); 60 | if (i == j) { 61 | assertEquals(val, i + 1.0); 62 | } else { 63 | assertEquals(val, 0.0); 64 | } 65 | } 66 | } 67 | } 68 | 69 | @Test 70 | public void checkStandardSerialization() { 71 | final int nRows = 3; 72 | final int nCols = 7; 73 | final Matrix m = generateIncreasingEye(nRows, nCols); 74 | 75 | final byte[] mtxBytes = m.toByteArray(); 76 | assertEquals(mtxBytes.length, m.getSizeBytes()); 77 | 78 | final Memory mem = Memory.wrap(mtxBytes); 79 | final Matrix tgt = MatrixImplOjAlgo.heapifyInstance(mem); 80 | MatrixTest.checkMatrixEquality(tgt, m); 81 | } 82 | 83 | @Test 84 | public void checkCompactSerialization() { 85 | final int nRows = 4; 86 | final int nCols = 7; 87 | final Matrix m = generateIncreasingEye(nRows, nCols); 88 | 89 | byte[] mtxBytes = m.toCompactByteArray(nRows - 1, 7); 90 | assertEquals(mtxBytes.length, m.getCompactSizeBytes(nRows - 1, 7)); 91 | 92 | Memory mem = Memory.wrap(mtxBytes); 93 | Matrix tgt = MatrixImplOjAlgo.heapifyInstance(mem); 94 | for (int c = 0; c < nCols; ++c) { 95 | for (int r = 0; r < (nRows - 1); ++r) { 96 | assertEquals(tgt.getElement(r, c), m.getElement(r, c)); // equal here 97 | } 98 | // assuming nRows - 1 so check only the last row as being 0 99 | assertEquals(tgt.getElement(nRows - 1, c), 0.0); 100 | } 101 | 102 | // test without compacting 103 | mtxBytes = m.toCompactByteArray(nRows, nCols); 104 | assertEquals(mtxBytes.length, m.getSizeBytes()); 105 | mem = Memory.wrap(mtxBytes); 106 | tgt = MatrixImplOjAlgo.heapifyInstance(mem); 107 | MatrixTest.checkMatrixEquality(tgt, m); 108 | } 109 | 110 | @Test 111 | public void matrixRowOperations() { 112 | final int nRows = 7; 113 | final int nCols = 5; 114 | final Matrix m = generateIncreasingEye(nRows, nCols); 115 | 116 | final int tgtCol = 2; 117 | final double[] v = m.getRow(tgtCol); // diagonal matrix, so this works ok 118 | for (int i = 0; i < v.length; ++i) { 119 | assertEquals(v[i], (i == tgtCol ? i + 1.0 : 0.0)); 120 | } 121 | 122 | assertEquals(m.getElement(6, tgtCol), 0.0); 123 | m.setRow(6, v); 124 | assertEquals(m.getElement(6, tgtCol), tgtCol + 1.0); 125 | } 126 | 127 | @Test 128 | public void matrixColumnOperations() { 129 | final int nRows = 9; 130 | final int nCols = 4; 131 | final Matrix m = generateIncreasingEye(nRows, nCols); 132 | 133 | final int tgtRow = 3; 134 | final double[] v = m.getColumn(tgtRow); // diagonal matrix, so this works ok 135 | for (int i = 0; i < v.length; ++i) { 136 | assertEquals(v[i], (i == tgtRow ? i + 1.0 : 0.0)); 137 | } 138 | 139 | assertEquals(m.getElement(tgtRow, 0), 0.0); 140 | m.setColumn(0, v); 141 | assertEquals(m.getElement(tgtRow, 0), tgtRow + 1.0); 142 | } 143 | 144 | @Test 145 | public void invalidRowColumnOperations() { 146 | final int nRows = 9; 147 | final int nCols = 4; 148 | final Matrix m = generateIncreasingEye(nRows, nCols); 149 | 150 | final double[] shortRow = new double[nCols - 2]; 151 | try { 152 | m.setRow(1, shortRow); 153 | fail(); 154 | } catch (final IllegalArgumentException e) { 155 | // expected 156 | } 157 | 158 | final double[] longColumn = new double[nRows + 2]; 159 | try { 160 | m.setColumn(1, longColumn); 161 | fail(); 162 | } catch (final IllegalArgumentException e) { 163 | // expected 164 | } 165 | } 166 | 167 | @Test 168 | public void invalidSerVer() { 169 | final int nRows = 3; 170 | final int nCols = 3; 171 | final Matrix m = generateIncreasingEye(nRows, nCols); 172 | final byte[] sketchBytes = m.toByteArray(); 173 | final WritableMemory mem = WritableMemory.wrap(sketchBytes); 174 | MatrixPreambleUtil.insertSerVer(mem.getArray(), mem.getCumulativeOffset(0L), 0); 175 | 176 | try { 177 | MatrixImplOjAlgo.heapifyInstance(mem); 178 | fail(); 179 | } catch (final IllegalArgumentException e) { 180 | // expected 181 | } 182 | } 183 | 184 | @Test 185 | public void invalidFamily() { 186 | final int nRows = 3; 187 | final int nCols = 3; 188 | final Matrix m = generateIncreasingEye(nRows, nCols); 189 | final byte[] sketchBytes = m.toByteArray(); 190 | final WritableMemory mem = WritableMemory.wrap(sketchBytes); 191 | MatrixPreambleUtil.insertFamilyID(mem.getArray(), mem.getCumulativeOffset(0L), 0); 192 | 193 | try { 194 | MatrixImplOjAlgo.heapifyInstance(mem); 195 | fail(); 196 | } catch (final IllegalArgumentException e) { 197 | // expected 198 | } 199 | } 200 | 201 | @Test 202 | public void insufficientMemoryCapacity() { 203 | final byte[] bytes = new byte[6]; 204 | final Memory mem = Memory.wrap(bytes); 205 | try { 206 | MatrixImplOjAlgo.heapifyInstance(mem); 207 | fail(); 208 | } catch (final IllegalArgumentException e) { 209 | // expected 210 | } 211 | } 212 | 213 | /** 214 | * Creates a scaled I matrix, where the diagonal consists of increasing integers, 215 | * starting with 1.0. 216 | * @param nRows number of rows 217 | * @param nCols number of columns 218 | * @return Primitive64Store, suitable for direct use or wrapping 219 | */ 220 | private static Matrix generateIncreasingEye(final int nRows, final int nCols) { 221 | final Matrix m = MatrixImplOjAlgo.newInstance(nRows, nCols); 222 | for (int i = 0; (i < nRows) && (i < nCols); ++i) { 223 | m.setElement(i, i, 1.0 + i); 224 | } 225 | return m; 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /src/test/java/org/apache/datasketches/vector/matrix/MatrixTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.datasketches.vector.matrix; 21 | 22 | import static org.testng.Assert.assertEquals; 23 | import static org.testng.Assert.assertNotNull; 24 | import static org.testng.Assert.assertNull; 25 | import static org.testng.Assert.assertTrue; 26 | import static org.testng.Assert.fail; 27 | 28 | import org.testng.annotations.Test; 29 | 30 | import org.apache.datasketches.memory.Memory; 31 | import org.apache.datasketches.memory.WritableMemory; 32 | import org.apache.datasketches.vector.MatrixFamily; 33 | 34 | @SuppressWarnings("javadoc") 35 | public class MatrixTest { 36 | 37 | @Test 38 | public void checkHeapify() { 39 | final Matrix m = Matrix.builder().setType(MatrixType.OJALGO).build(3, 3); 40 | final byte[] bytes = m.toByteArray(); 41 | final Memory mem = Memory.wrap(bytes); 42 | println(MatrixPreambleUtil.preambleToString(mem)); 43 | 44 | final Matrix tgt = Matrix.heapify(mem, MatrixType.OJALGO); 45 | assertTrue(tgt instanceof MatrixImplOjAlgo); 46 | checkMatrixEquality(m, tgt); 47 | } 48 | 49 | @Test 50 | public void checkWrap() { 51 | assertNull(Matrix.wrap(null)); 52 | 53 | final Matrix src = Matrix.builder().setType(MatrixType.OJALGO).build(3, 3); 54 | final Object obj = src.getRawObject(); 55 | final Matrix tgt = Matrix.wrap(obj); 56 | assertTrue(tgt instanceof MatrixImplOjAlgo); 57 | checkMatrixEquality(src, tgt); 58 | 59 | try { 60 | final Object notAMatrix = 1.0; 61 | Matrix.wrap(notAMatrix); 62 | fail(); 63 | } catch (final IllegalArgumentException e) { 64 | // expected 65 | } 66 | 67 | assertNotNull(src.toString()); 68 | } 69 | 70 | @Test 71 | public void checkSize() { 72 | final int nRow = 7; 73 | final int nCol = 3; 74 | final Matrix m = Matrix.builder().build(nRow, nCol); 75 | 76 | int expectedSize = (MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES) 77 | + (nRow * nCol * Double.BYTES); 78 | assertEquals(m.getSizeBytes(), expectedSize); 79 | 80 | // this should redirect call to getSizeBytes() 81 | assertEquals(m.getCompactSizeBytes(nRow, nCol), expectedSize); 82 | 83 | // degenerate cases 84 | expectedSize = (MatrixFamily.MATRIX.getMinPreLongs() * Long.BYTES); 85 | assertEquals(m.getCompactSizeBytes(0, nCol), expectedSize); 86 | assertEquals(m.getCompactSizeBytes(nRow, 0), expectedSize); 87 | 88 | // matrix subsets 89 | expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES) 90 | + ((5 * 3) * Double.BYTES); 91 | assertEquals(m.getCompactSizeBytes(5, 3), expectedSize); 92 | 93 | expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES) 94 | + ((7 * 2) * Double.BYTES); 95 | assertEquals(m.getCompactSizeBytes(7, 2), expectedSize); 96 | 97 | expectedSize = (MatrixFamily.MATRIX.getMaxPreLongs() * Long.BYTES) 98 | + ((2 * 2) * Double.BYTES); 99 | assertEquals(m.getCompactSizeBytes(2, 2), expectedSize); 100 | } 101 | 102 | @Test 103 | public void invalidSerVer() { 104 | final int nRows = 50; 105 | final int nCols = 75; 106 | final MatrixBuilder mb = new MatrixBuilder(); 107 | final Matrix m = mb.build(nRows, nCols); 108 | final byte[] sketchBytes = m.toByteArray(); 109 | final WritableMemory mem = WritableMemory.wrap(sketchBytes); 110 | MatrixPreambleUtil.insertSerVer(mem.getArray(), mem.getCumulativeOffset(0L), 0); 111 | 112 | try { 113 | MatrixPreambleUtil.preambleToString(mem); 114 | fail(); 115 | } catch (final IllegalArgumentException e) { 116 | // expected 117 | } 118 | } 119 | 120 | @Test 121 | public void invalidFamily() { 122 | final int nRows = 3; 123 | final int nCols = 3; 124 | final MatrixBuilder mb = new MatrixBuilder(); 125 | final Matrix m = mb.build(nRows, nCols); 126 | final byte[] sketchBytes = m.toByteArray(); 127 | final WritableMemory mem = WritableMemory.wrap(sketchBytes); 128 | MatrixPreambleUtil.insertFamilyID(mem.getArray(), mem.getCumulativeOffset(0L), 0); 129 | 130 | try { 131 | MatrixPreambleUtil.preambleToString(mem); 132 | fail(); 133 | } catch (final IllegalArgumentException e) { 134 | // expected 135 | } 136 | } 137 | 138 | @Test 139 | public void checkInsufficientMemoryCapacity() { 140 | final byte[] bytes = new byte[6]; 141 | final Memory mem = Memory.wrap(bytes); 142 | try { 143 | MatrixPreambleUtil.preambleToString(mem); 144 | fail(); 145 | } catch (final IllegalArgumentException e) { 146 | // expected 147 | } 148 | } 149 | 150 | @Test 151 | public void checkCompactPreamble() { 152 | final int nRows = 4; 153 | final int nCols = 7; 154 | final MatrixBuilder mb = new MatrixBuilder(); 155 | final Matrix m = mb.build(nRows, nCols); 156 | 157 | final Memory mem = Memory.wrap(m.toCompactByteArray(nRows - 1, 7)); 158 | MatrixPreambleUtil.preambleToString(mem); 159 | } 160 | 161 | static void checkMatrixEquality(final Matrix m1, final Matrix m2) { 162 | assertEquals(m1.numRows_, m2.numRows_); 163 | assertEquals(m1.numCols_, m2.numCols_); 164 | 165 | for (int i = 0; i < m1.numRows_; ++i) { 166 | for (int j = 0; j < m1.numCols_; ++j) { 167 | assertEquals(m1.getElement(i, j), m2.getElement(i, j), 168 | "Mismatch at (" + i + ", " + j + ")"); 169 | } 170 | } 171 | } 172 | 173 | /** 174 | * println the message 175 | * @param msg the message 176 | */ 177 | static void println(final String msg) { 178 | //System.out.println(msg); 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /tools/FindBugsExcludeFilter.xml: -------------------------------------------------------------------------------- 1 | 19 | 20 |