├── .gitignore ├── Makefile ├── README.md ├── examples ├── .gitignore ├── CFRecommender.java ├── cf_recommender.c └── smatrix_example.c └── src ├── Makefile ├── Makefile.in ├── java ├── Makefile ├── com │ └── paulasmuth │ │ └── libsmatrix │ │ └── SparseMatrix.java ├── pom.xml └── test │ └── TestSparseMatrix.java ├── ruby ├── .gitignore ├── Makefile ├── extconf.rb ├── libsmatrix.gemspec └── libsmatrix.rb ├── smatrix.c ├── smatrix.h ├── smatrix_benchmark.c ├── smatrix_jni.c ├── smatrix_jni.h ├── smatrix_private.h ├── smatrix_ruby.c └── smatrix_ruby.h /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | src/config.h 3 | src/smatrix_benchmark 4 | *.o 5 | *.so 6 | *.a 7 | *.gem 8 | *.bundle 9 | *.dylib 10 | *.class 11 | *.smx 12 | .DS_Store 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This file is part of the "libsmatrix" project 2 | # (c) 2011-2013 Paul Asmuth 3 | # 4 | # Licensed under the MIT License (the "License"); you may not use this 5 | # file except in compliance with the License. You may obtain a copy of 6 | # the License at: http://opensource.org/licenses/MIT 7 | 8 | include src/Makefile.in 9 | 10 | SHELL = /bin/sh 11 | CC = clang 12 | CFLAGS_ = $(CFLAGS) -Wall -Wextra -O3 -march=native -mtune=native -D NDEBUG -fPIC 13 | LDFLAGS = -lpthread -lm -lruby 14 | PREFIX = $(DESTDIR)/usr/local 15 | LIBDIR = $(PREFIX)/lib 16 | UNAME = $(shell uname) 17 | SOURCES = src/smatrix.c src/smatrix_jni.c src/smatrix_ruby.c 18 | 19 | all: src/smatrix.$(LIBEXT) 20 | 21 | src/smatrix.$(LIBEXT): 22 | cd src && make 23 | 24 | install: 25 | cp src/smatrix.$(LIBEXT) $(LIBDIR) 26 | 27 | clean: 28 | find . -name "*.o" -o -name "*.a" -o -name "*.class" -o -name "*.so" -o -name "*.dylib" -o -name "*.bundle" | xargs rm 29 | rm -rf src/java/target src/config.h src/smatrix_benchmark *.gem 30 | 31 | ruby: 32 | cd src/ruby && ruby extconf.rb 33 | cd src/ruby && make 34 | 35 | publish_ruby: 36 | gem build src/ruby/libsmatrix.gemspec 37 | mv *.gem src/ruby/ 38 | 39 | java: 40 | cd src/java && make 41 | 42 | publish_java: java 43 | cd src/java && mvn deploy 44 | 45 | benchmark: src/smatrix_benchmark 46 | src/smatrix_benchmark full 47 | 48 | src/smatrix_benchmark: 49 | cd src && make smatrix_benchmark 50 | 51 | test: 52 | cd src/java && make test 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | libsmatrix 2 | ========== 3 | 4 | A thread-safe two dimensional sparse matrix data structure with C, Java and Ruby bindings. 5 | It was created to make loading and accessing medium sized (10GB+) matrices in boxed languages 6 | like Java/Scala or Ruby easier. 7 | 8 | While the chosen internal storage format (nested hashmaps) is neither the most memory-efficient 9 | nor extremely fast in terms of access/insert time it seems to be a good tradeoff between these 10 | two goals. 11 | 12 | A libsmatrix sparse matrix features two modes of operation; a memory-only mode in which all data 13 | is kept in main memory and a mode in which the data is stored on disk and only a pool of recently 14 | used rows is kept in memory. In this mode the data is persisted across program restarts. It also 15 | allows you to handle datasets larger than your available main memory. 16 | 17 | #### Documentation 18 | 19 | + [Getting Started](#getting-started) 20 | + [C API](#c-api) 21 | + [Java/Scala API](#fnord) 22 | + [Ruby API](#ruby-api) 23 | + [Internals](#internals) 24 | + [Benchmarks](#benchmarks) 25 | + [Examples](#examples) 26 | + [License](#license) 27 | 28 | 29 | Getting Started (Building) 30 | -------------------------- 31 | 32 | There are multiple ways to install libsmatrix: 33 | 34 | ### Compile from source 35 | 36 | This will produce a single shared object "smatrix.so" file that exports all calls documented 37 | in "C API". 38 | 39 | $ make 40 | $ make install 41 | 42 | To run the tests/benchmarks (optional, requires java and ruby) 43 | 44 | $ make test 45 | $ make benchmark 46 | 47 | To build the MRI ruby and Java JNI bindings (optional), run: 48 | 49 | $ make ruby 50 | $ make java 51 | 52 | This will produce the respective shared objects and bundles in: 53 | 54 | src/ruby/smatrix_ruby.so 55 | src/ruby/smatrix_X.X.X.gem 56 | 57 | src/java/smatrix_java.so 58 | src/java/target/libsmatrix-X.X-SNAPSHOT.jar 59 | 60 | ### Import artifact via Maven/sbt (java/scala) 61 | 62 | Currently the maven artifact only contains the binding glue code and doesn't actually build 63 | the native shared object. You need to compile & install "libsmatrix.so" yourself on the target 64 | host, otherwise you'll get a "UnsatisfiedLinkError". 65 | 66 | Import artifact via sbt: 67 | 68 | resolvers += "sbt-libsmatrix-repo" at "https://raw.github.com/paulasmuth/libsmatrix/mvn-repo/" 69 | 70 | libraryDependencies += "com.paulasmuth.libsmatrix" % "libsmatrix" % "0.2-SNAPSHOT" 71 | 72 | Import artifact via Maven2 (put this into your pom.xml): 73 | 74 | 75 | libsmatrix-mvn-repo 76 | https://raw.github.com/paulsmuth/libsmatrix/mvn-repo/ 77 | 78 | true 79 | always 80 | 81 | 82 | 83 | To publish the maven artifact from source, check out libsmatrix and run this: 84 | 85 | $ make publish_java 86 | 87 | 88 | ### Import gem via rubygems (ruby only) 89 | 90 | This will install the ruby bindings and compile the native shared object: 91 | 92 | $ gem install libsmatrix 93 | 94 | To use libsmatrix in your project, require it like this: 95 | 96 | require "libsmatrix" 97 | 98 | To build and publish the ruby gem run: 99 | 100 | $ make publish_ruby 101 | 102 | C API 103 | ----- 104 | 105 | Open a smatrix (if filename is NULL, use in memory only mode; otherwise open or create file) 106 | 107 | smatrix_t* smatrix_open(const char* fname); 108 | 109 | Close a smatrix: 110 | 111 | void smatrix_close(smatrix_t* self); 112 | 113 | Get, Set, Increment, Decrement a (x,y) position. _All of the methods are threadsafe_ 114 | 115 | uint32_t smatrix_get(smatrix_t* self, uint32_t x, uint32_t y); 116 | uint32_t smatrix_set(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 117 | uint32_t smatrix_incr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 118 | uint32_t smatrix_decr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 119 | 120 | Get a whole "row" of the matrix by row coordinate x. _All of the methods are threadsafe_ 121 | 122 | uint32_t smatrix_rowlen(smatrix_t* self, uint32_t x); 123 | uint32_t smatrix_getrow(smatrix_t* self, uint32_t x, uint32_t* ret, size_t ret_len); 124 | 125 | 126 | Java / Scala API 127 | ---------------- 128 | 129 | here be dragons 130 | 131 | 132 | Ruby API 133 | -------- 134 | 135 | [![Gem Version](https://badge.fury.io/rb/libsmatrix.png)](http://badge.fury.io/rb/libsmatrix) 136 | 137 | Install and require the gem: 138 | 139 | $ install gem libsmatrix 140 | $ require 'libsmatrix' 141 | 142 | 143 | Create a new smatrix instance: 144 | 145 | $ smatrix = SparseMatrix.new("/path/to/smatrix.smx") 146 | 147 | Get, Set, Increment, Decrement a (x,y) position 148 | 149 | $ smatrix.set(x, y, 5) 150 | => 5 151 | $ smatrix.get(x, y) 152 | => 5 153 | $ smatrix.incr(x, y, 1) 154 | => 6 155 | $ smatrix.decr(x, y, 1) 156 | => 5 157 | 158 | Close and free the matrix (data is persisted to disk): 159 | 160 | $ smatrix = nil 161 | 162 | 163 | Benchmarks 164 | ---------- 165 | 166 | **No big-data disclaimer:** We are using this code to run a Collaborative Filtering 167 | recommendation engine for one of Germany's largest ecommerce sites. It is tested on "small-data" 168 | datasets with up to 40GB per matrix (1.5 billion values in 13 million rows). If your data is 169 | actually much bigger (measured in terrabytes, not gigabytes) this library is not for you. 170 | 171 | here be dragons 172 | 173 | 174 | Examples 175 | ------- 176 | 177 | + There is a simple example in src/smatrix_example.c 178 | + There is a simple Collaborative Filtering based recommendation engine in src/smatrix_example_recommender.c 179 | 180 | 181 | License 182 | ------- 183 | 184 | Copyright (c) 2011 Paul Asmuth 185 | 186 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to use, copy and modify copies of the Software, subject to the following conditions: 187 | 188 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 189 | 190 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 191 | -------------------------------------------------------------------------------- /examples/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.class 3 | -------------------------------------------------------------------------------- /examples/CFRecommender.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | import com.paulasmuth.libsmatrix.SparseMatrix; 10 | 11 | /** 12 | * Compile & run this example: 13 | * 14 | * $ javac CFRecommender.java && java CFRecommeder 15 | * 16 | */ 17 | class CFRecommender { 18 | 19 | public static void main(String[] opts) { 20 | SparseMatrix smx = new SparseMatrix(); 21 | smx.test(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /examples/cf_recommender.c: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "smatrix.h" 14 | 15 | smatrix_t* my_smatrix; 16 | 17 | // libsmatrix example: simple CF based recommendation engine 18 | int main(int argc, char **argv) { 19 | my_smatrix = smatrix_open(NULL); 20 | 21 | // one preference set = list of items in one session 22 | // e.g. list of viewed items by the same user 23 | // e.g. list of bought items in the same checkout 24 | uint32_t input_ids[5] = {12,52,63,76,43}; 25 | import_preference_set(input_ids, 5); 26 | 27 | // generate recommendations (similar items) for item #76 28 | void neighbors_for_item(76); 29 | 30 | smatrix_close(my_smatrix); 31 | return 0; 32 | } 33 | 34 | // train / add a preference set (list of items in one session) 35 | void import_preference_set(uint32_t* ids, uint32_t num_ids) { 36 | uint32_t i, n; 37 | 38 | for (n = 0; n < num_ids; n++) { 39 | smatrix_incr(my_smatrix, ids[n], 0, 1); 40 | 41 | for (i = 0; i < pset->len; i++) { 42 | if (i != n) { 43 | smatrix_incr(my_smatrix, ids[n], ids[i], 1); 44 | } 45 | } 46 | } 47 | } 48 | 49 | // get recommendations for item with id "item_id" 50 | void neighbors_for_item(uint32_t item_id) 51 | uint32_t neighbors, *row, total; 52 | 53 | total = smatrix_get(my_smatrix, item_id, 0); 54 | neighbors = smatrix_getrow(my_smatrix, item_id, row, 8192); 55 | 56 | for (pos = 0; pos < neighbors; pos++) { 57 | uint32_t cur_id = row[pos * 2]; 58 | 59 | printf("found neighbor for item %u: item %u with distance %f\n", 60 | item_id, cf_cosine(smatrix, cur_id, row[pos * 2 + 1], total)); 61 | } 62 | 63 | free(row); 64 | } 65 | 66 | // calculates the cosine vector distance between two items 67 | double cf_cosine(smatrix_t* smatrix, uint32_t b_id, uint32_t cc_count, uint32_t a_total) { 68 | uint32_t b_total; 69 | double num, den; 70 | 71 | b_total = smatrix_get(smatrix, b_id, 0); 72 | 73 | if (b_total == 0) 74 | b_total = 1; 75 | 76 | num = cc_count; 77 | den = sqrt((double) a_total) * sqrt((double) b_total); 78 | 79 | if (den == 0.0) 80 | return 0.0; 81 | 82 | if (num > den) 83 | return 0.0; 84 | 85 | return (num / den); 86 | } 87 | 88 | -------------------------------------------------------------------------------- /examples/smatrix_example.c: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "smatrix.h" 14 | 15 | smatrix_t* db; 16 | 17 | void* test(void* fnord) { 18 | uint64_t i, n, m; 19 | 20 | for (m = 0; m < 100; m++) { 21 | for (n = 1; n < 30; n++) { 22 | for (i = 1; i < 50; i++) { 23 | smatrix_incr(db, n, i, 1); 24 | } 25 | } 26 | } 27 | 28 | return NULL; 29 | } 30 | 31 | int main(int argc, char **argv) { 32 | int i,n,m,l,x=0, num_threads = 4; 33 | pthread_t threads[num_threads]; 34 | 35 | printf("\nloading\n"); 36 | db = smatrix_open("/var/tmp/reco.db"); 37 | //db = smatrix_open(NULL); // in-memory only mode 38 | 39 | if (db == NULL) 40 | abort(); 41 | 42 | printf("\nstarting\n"); 43 | 44 | for (n = 0; n < num_threads; n++) 45 | pthread_create(&threads[n], NULL, test, NULL); 46 | 47 | for (n = 0; n < num_threads; n++) 48 | pthread_join(threads[n], NULL); 49 | 50 | printf("\ndone\n"); 51 | 52 | for (n = 1; n < 30; n++) { 53 | for (i = 1; i < 50; i++) { 54 | printf("(%u,%u) => %u, ", n, i, smatrix_get(db, n, i)); 55 | if (x++ % 5 == 0) printf("\n"); 56 | } 57 | } 58 | 59 | printf("rowlen: %u\n", l = smatrix_rowlen(db, 23)); 60 | size_t bytes = sizeof(uint32_t) * l * 2; 61 | uint32_t* data = malloc(bytes); 62 | 63 | l = smatrix_getrow(db, 23, data, bytes); 64 | 65 | for (i = 0; i < l; i++) { 66 | printf("%u => %u, ", data[i * 2], data[i * 2 + 1]); 67 | } 68 | 69 | printf("\n"); 70 | 71 | smatrix_close(db); 72 | printf("in use at exit: %lu\n", db->mem); 73 | 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | # This file is part of the "libsmatrix" project 2 | # (c) 2011-2013 Paul Asmuth 3 | # 4 | # Licensed under the MIT License (the "License"); you may not use this 5 | # file except in compliance with the License. You may obtain a copy of 6 | # the License at: http://opensource.org/licenses/MIT 7 | 8 | include Makefile.in 9 | 10 | TARGET = smatrix.$(LIBEXT) 11 | TARGET_STATIC = smatrix-static.a 12 | 13 | all: $(TARGET) $(TARGET_STATIC) 14 | 15 | $(TARGET): smatrix.o 16 | $(CC) $(LIBFLAGS) smatrix.o -o $(TARGET) $(LDFLAGS) 17 | 18 | $(TARGET_STATIC): smatrix.o 19 | ar cr $(TARGET_STATIC) smatrix.o 20 | 21 | smatrix.o: config.h smatrix.c smatrix.h smatrix_private.h 22 | $(CC) -c $(CFLAGS) smatrix.c -o smatrix.o 23 | 24 | config.h: 25 | touch config.h 26 | 27 | smatrix_jni.h: 28 | javac com/paulasmuth/libsmatrix/SparseMatrix.java 29 | javah -o smatrix_jni.h -classpath . com.paulasmuth.libsmatrix.SparseMatrix 30 | 31 | smatrix_benchmark: smatrix.o smatrix_benchmark.c 32 | $(CC) $(CFLAGS) smatrix_benchmark.c smatrix.o -o smatrix_benchmark $(LDFLAGS) 33 | -------------------------------------------------------------------------------- /src/Makefile.in: -------------------------------------------------------------------------------- 1 | # This file is part of the "libsmatrix" project 2 | # (c) 2011-2013 Paul Asmuth 3 | # 4 | # Licensed under the MIT License (the "License"); you may not use this 5 | # file except in compliance with the License. You may obtain a copy of 6 | # the License at: http://opensource.org/licenses/MIT 7 | 8 | UNAME = $(shell uname) 9 | SHELL = /bin/sh 10 | CC = clang 11 | CFLAGS ?= -Wall -Wextra -O3 -march=native -mtune=native -D NDEBUG -fPIC 12 | LDFLAGS = -lpthread -lm 13 | PREFIX = $(DESTDIR)/usr/local 14 | LIBDIR = $(PREFIX)/lib 15 | 16 | LIBFLAGS = -shared 17 | LIBEXT = so 18 | 19 | ifeq ($(UNAME), Darwin) 20 | LIBFLAGS = -dynamic -bundle 21 | LIBEXT = bundle 22 | endif 23 | -------------------------------------------------------------------------------- /src/java/Makefile: -------------------------------------------------------------------------------- 1 | # This file is part of the "libsmatrix" project 2 | # (c) 2011-2013 Paul Asmuth 3 | # 4 | # Licensed under the MIT License (the "License"); you may not use this 5 | # file except in compliance with the License. You may obtain a copy of 6 | # the License at: http://opensource.org/licenses/MIT 7 | 8 | include ../Makefile.in 9 | 10 | TARGET = smatrix_java.$(LIBEXT) 11 | 12 | ifeq ($(UNAME), Darwin) 13 | JNI_FLAGS = -I /System/Library/Frameworks/JavaVM.framework/Headers -I /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Headers 14 | endif 15 | ifeq ($(UNAME), Linux) 16 | JNI_FLAGS = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux 17 | endif 18 | 19 | all: $(TARGET) 20 | mvn package 21 | 22 | $(TARGET): ../smatrix.o ../smatrix_jni.c ../smatrix_jni.h 23 | $(CC) $(JNI_FLAGS) $(LIBFLAGS) $(LDFLAGS) ../smatrix_jni.c ../smatrix.o -o $(TARGET) 24 | ifeq ($(UNAME), Darwin) 25 | ln -s $(TARGET) smatrix_java.so 26 | endif 27 | 28 | ../smatrix.o: 29 | cd .. && make 30 | 31 | test: $(TARGET) 32 | javac -classpath . test/TestSparseMatrix.java 33 | java -Djava.library.path=./ -classpath .:./test TestSparseMatrix 34 | 35 | .PHONY: test 36 | -------------------------------------------------------------------------------- /src/java/com/paulasmuth/libsmatrix/SparseMatrix.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | package com.paulasmuth.libsmatrix; 10 | import java.io.File; 11 | import java.util.SortedMap; 12 | import java.util.concurrent.ConcurrentSkipListMap; 13 | 14 | /** 15 | * A libsmatrix sparse matrix 16 | */ 17 | public class SparseMatrix { 18 | private static String library_path = null; 19 | private String filename = null; 20 | private long ptr; 21 | 22 | /** 23 | * Create a new sparse matrix that will be stored in main memory only. 24 | * 25 | * @param file_path path to the file or null 26 | * @return a new SparseMatrix 27 | */ 28 | public SparseMatrix() { 29 | SparseMatrix.loadLibrary(); 30 | init(null); 31 | } 32 | 33 | /** 34 | * Create a new sparse matrix that will be persisted to disk. A cache of the 35 | * data is still kept in main memory (the cache size can be adjusted by calling 36 | * setCacheSize on the instance). 37 | * 38 | * If the file pointed to by filename exists it will be opened, otherwise a new 39 | * file will be created. 40 | * 41 | * @param file_path path to the file 42 | * @return a new SparseMatrix 43 | */ 44 | public SparseMatrix(String file_path) { 45 | SparseMatrix.loadLibrary(); 46 | filename = file_path; 47 | init(file_path); 48 | } 49 | 50 | /** 51 | * Explcitly set the path to the native shared object (libsmatrix.so). 52 | * 53 | * @param the path to the native shared object 54 | */ 55 | public static void setLibraryPath(String path) { 56 | library_path = path; 57 | loadLibrary(); 58 | } 59 | 60 | /** 61 | * HERE BE DRAGONS 62 | */ 63 | public String getFilename() { 64 | return filename; 65 | } 66 | 67 | /** 68 | * HERE BE DRAGONS 69 | */ 70 | public native int get(int x, int y); 71 | 72 | /** 73 | * HERE BE DRAGONS 74 | */ 75 | public native void set(int x, int y, int val); 76 | 77 | /** 78 | * HERE BE DRAGONS 79 | */ 80 | public native void incr(int x, int y, int val); 81 | 82 | /** 83 | * HERE BE DRAGONS 84 | */ 85 | public native void decr(int x, int y, int val); 86 | 87 | /** 88 | * HERE BE DRAGONS 89 | */ 90 | public native int getRowLength(int x); 91 | 92 | /** 93 | * HERE BE DRAGONS 94 | */ 95 | public SortedMap getRow(int x) { 96 | return getRow(x, 0); 97 | } 98 | 99 | /** 100 | * HERE BE DRAGONS 101 | */ 102 | public SortedMap getRow(int x, int maxlen) { 103 | SortedMap map = new ConcurrentSkipListMap() { 104 | public void putIntTuple(int k, int v) { 105 | this.put(k, v); 106 | } 107 | }; 108 | 109 | getRowNative(x, map, maxlen); 110 | 111 | return map; 112 | } 113 | 114 | /** 115 | * Close this matrix. Calling any other method on the instance after it was 116 | * closed will throw an exception. 117 | */ 118 | public native void close(); 119 | 120 | /** 121 | * HERE BE DRAGONS 122 | */ 123 | private native void getRowNative(int x, SortedMap map, int maxlen); 124 | 125 | /** 126 | * Load the native shared object (libsmatrix.so) 127 | */ 128 | private static void loadLibrary() { 129 | if (library_path != null) { 130 | File libfile = new File(library_path); 131 | 132 | if (libfile.exists()) { 133 | System.load(libfile.getAbsolutePath()); 134 | return; 135 | } 136 | } 137 | 138 | System.loadLibrary("smatrix"); 139 | } 140 | 141 | /** 142 | * Initialize a new native matrix 143 | */ 144 | private native void init(String file_path); 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/java/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | libsmatrix sparse matrix 6 | http://github.com/paulasmuth/libsmatrix 7 | com.paulasmuth.libsmatrix 8 | libsmatrix 9 | 0.3-SNAPSHOT 10 | jar 11 | 12 | 13 | src/java 14 | 15 | 16 | 17 | com.github.github 18 | site-maven-plugin 19 | 0.8 20 | 33 | 34 | Maven artifacts for ${project.version} 35 | true 36 | ${project.build.directory}/mvn-repo 37 | refs/heads/mvn-repo 38 | **/* 39 | libsmatrix 40 | paulasmuth 41 | 42 | 43 | 44 | 45 | site 46 | 47 | deploy 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | internal.repo 57 | Temporary Staging Repository 58 | file://${project.build.directory}/mvn-repo 59 | false 60 | 61 | 62 | 63 | 64 | 65 | github 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /src/java/test/TestSparseMatrix.java: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | import java.util.LinkedList; 10 | import java.util.Iterator; 11 | import java.util.SortedMap; 12 | import com.paulasmuth.libsmatrix.SparseMatrix; 13 | 14 | interface TestCase { 15 | public String getName(); 16 | public boolean run(SparseMatrix smx); 17 | } 18 | 19 | class TestSparseMatrix { 20 | static LinkedList testCases = new LinkedList(); 21 | 22 | static { testCases.add(new TestCase() { 23 | public String getName() { 24 | return "simple set/get"; 25 | } 26 | public boolean run(SparseMatrix smx) { 27 | smx.set(42,23,17); 28 | return smx.get(42,23) == 17; 29 | } 30 | }); } 31 | 32 | static { testCases.add(new TestCase() { 33 | public String getName() { 34 | return "simple increment 1"; 35 | } 36 | public boolean run(SparseMatrix smx) { 37 | smx.set(4231,2634,0); 38 | smx.incr(4231,2634,1); 39 | return smx.get(4231,2634) == 1; 40 | } 41 | }); } 42 | 43 | static { testCases.add(new TestCase() { 44 | public String getName() { 45 | return "simple increment 5"; 46 | } 47 | public boolean run(SparseMatrix smx) { 48 | smx.set(1231,2634,0); 49 | smx.incr(1231,2634,1); 50 | smx.incr(1231,2634,5); 51 | return smx.get(1231,2634) == 6; 52 | } 53 | }); } 54 | 55 | static { testCases.add(new TestCase() { 56 | public String getName() { 57 | return "1 million increments + 1 million gets"; 58 | } 59 | public boolean run(SparseMatrix smx) { 60 | int v = 34; 61 | int i; 62 | int n; 63 | 64 | for (n = 0; n < 1000; n++) { 65 | for (i = 0; i < 1000; i++) { 66 | smx.set(i, n, v); 67 | } 68 | } 69 | 70 | for (n = 0; n < 1000; n++) { 71 | for (i = 0; i < 1000; i++) { 72 | if ((smx.get(i, n) != v)) { 73 | return false; 74 | } 75 | } 76 | } 77 | 78 | return true; 79 | } 80 | }); } 81 | 82 | static { testCases.add(new TestCase() { 83 | public String getName() { 84 | return "1000 increments + getRowLength()"; 85 | } 86 | public boolean run(SparseMatrix smx) { 87 | int i = 0; 88 | int n = 42; 89 | 90 | for (i = 0; i < 1000; i++) { 91 | smx.incr(i, n, 1); 92 | } 93 | 94 | return smx.getRowLength(n) == 1000; 95 | } 96 | }); } 97 | 98 | static { testCases.add(new TestCase() { 99 | public String getName() { 100 | return "1000 increments + getRow()"; 101 | } 102 | public boolean run(SparseMatrix smx) { 103 | int i = 0; 104 | int n = 85; 105 | 106 | for (i = 0; i < 1000; i++) { 107 | smx.incr(i, n, 1); 108 | } 109 | 110 | SortedMap row = smx.getRow(n); 111 | 112 | return row.size() == 1000; 113 | } 114 | }); } 115 | 116 | static { testCases.add(new TestCase() { 117 | public String getName() { 118 | return "1000 increments + getRow() with maxlen"; 119 | } 120 | public boolean run(SparseMatrix smx) { 121 | int i = 0; 122 | int n = 83; 123 | 124 | for (i = 0; i < 1000; i++) { 125 | smx.incr(i, n, 1); 126 | } 127 | 128 | SortedMap row = smx.getRow(n, 230); 129 | return row.size() == 230; 130 | } 131 | }); } 132 | 133 | static { testCases.add(new TestCase() { 134 | public String getName() { 135 | return "1 million increments; close; 1 million gets"; 136 | } 137 | public boolean run(SparseMatrix smx1) { 138 | if (smx1.getFilename() == null) { 139 | return true; 140 | } 141 | 142 | int v = 123; 143 | int i; 144 | int n; 145 | 146 | for (n = 0; n < 1000; n++) { 147 | for (i = 0; i < 1000; i++) { 148 | smx1.set(i, n, v); 149 | } 150 | } 151 | 152 | SparseMatrix smx2 = new SparseMatrix("/tmp/fnord.smx"); 153 | 154 | for (n = 0; n < 1000; n++) { 155 | for (i = 0; i < 1000; i++) { 156 | if ((smx2.get(i, n) != v)) { 157 | smx2.close(); 158 | return false; 159 | } 160 | } 161 | } 162 | 163 | smx2.close(); 164 | return true; 165 | } 166 | }); } 167 | 168 | public static void main(String[] opts) { 169 | boolean success = true; 170 | SparseMatrix.setLibraryPath("./smatrix_java.so"); 171 | 172 | SparseMatrix smx1 = new SparseMatrix(); 173 | success &= run_tests(smx1); 174 | smx1.close(); 175 | 176 | SparseMatrix smx2 = new SparseMatrix("/tmp/fnord.smx"); 177 | success &= run_tests(smx2); 178 | smx2.close(); 179 | 180 | if (success) { 181 | System.out.println("\033[1;32mAll tests finished successfully :)\033[0m"); 182 | System.exit(0); 183 | } else { 184 | System.out.println("\033[1;31mTests failed :(\033[0m"); 185 | System.exit(1); 186 | } 187 | } 188 | 189 | public static boolean run_tests(SparseMatrix smx) { 190 | boolean success = true; 191 | Iterator iter = testCases.iterator(); 192 | 193 | while (iter.hasNext()) { 194 | TestCase testcase = (TestCase) iter.next(); 195 | 196 | if (testcase.run(smx)) { 197 | System.out.println("\033[1;32m[SUCCESS] " + testcase.getName() + "\033[0m"); 198 | } else { 199 | System.out.println("\033[1;31m[FAILED] " + testcase.getName() + "\033[0m"); 200 | success = false; 201 | } 202 | } 203 | 204 | return success; 205 | } 206 | 207 | } 208 | -------------------------------------------------------------------------------- /src/ruby/.gitignore: -------------------------------------------------------------------------------- 1 | Makefile.in 2 | -------------------------------------------------------------------------------- /src/ruby/Makefile: -------------------------------------------------------------------------------- 1 | # This file is part of the "libsmatrix" project 2 | # (c) 2011-2013 Paul Asmuth 3 | # 4 | # Licensed under the MIT License (the "License"); you may not use this 5 | # file except in compliance with the License. You may obtain a copy of 6 | # the License at: http://opensource.org/licenses/MIT 7 | 8 | include ../Makefile.in 9 | include Makefile.in 10 | 11 | TARGET = smatrix_ruby.$(LIBEXT) 12 | 13 | all: $(TARGET) 14 | 15 | ../smatrix.o: 16 | cd .. && make 17 | 18 | $(TARGET): ../smatrix.o ../smatrix_ruby.c ../smatrix_ruby.h 19 | $(CC) -L$(RUBY_LIB) -I$(RUBY_INCLUDE_ARCH) -I$(RUBY_INCLUDE) $(LIBFLAGS) -Wl,-undefined,dynamic_lookup -Wl,-multiply_defined,suppress $(LDFLAGS) ../smatrix_ruby.c ../smatrix.o -o $(TARGET) 20 | 21 | install: $(TARGET) 22 | -------------------------------------------------------------------------------- /src/ruby/extconf.rb: -------------------------------------------------------------------------------- 1 | require "mkmf" 2 | 3 | mkmf_includes = < 0 ? $LIBRUBYARG_SHARED : $LIBRUBYARG_STATIC} 11 | EOF 12 | 13 | File.open(::File.expand_path("../Makefile.in", __FILE__), "w+") do |f| 14 | f.write(mkmf_includes) 15 | end 16 | 17 | $makefile_created = true 18 | 19 | -------------------------------------------------------------------------------- /src/ruby/libsmatrix.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | 3 | Gem::Specification.new do |s| 4 | s.name = "libsmatrix" 5 | s.version = "0.0.1" 6 | s.date = Date.today.to_s 7 | s.platform = Gem::Platform::RUBY 8 | s.authors = ["Paul Asmuth", "Amir Friedman"] 9 | s.email = ["paul@paulasmuth.com", "amirf@null.co.il"] 10 | s.homepage = "http://github.com/paulasmuth/libsmatrix" 11 | s.summary = %q{A thread-safe two dimensional sparse matrix data structure with C, Java and Ruby bindings.} 12 | s.description = %q{A thread-safe two dimensional sparse matrix data structure with C, Java and Ruby bindings. It was created to make loading and accessing medium sized (10GB+) matrices in boxed languages like Java/Scala or Ruby easier.} 13 | s.licenses = ["MIT"] 14 | s.extensions = ['src/ruby/extconf.rb'] 15 | s.files = `git ls-files`.split("\n") - [".gitignore", ".rspec", ".travis.yml"] 16 | s.test_files = `git ls-files -- spec/*`.split("\n") 17 | s.require_paths = ["src/ruby/"] 18 | 19 | s.add_development_dependency "rspec", "~> 2.8.0" 20 | end 21 | -------------------------------------------------------------------------------- /src/ruby/libsmatrix.rb: -------------------------------------------------------------------------------- 1 | bundle_file = ::File.expand_path("../smatrix_ruby.bundle", __FILE__) 2 | require bundle_file if ::File.exist? bundle_file 3 | 4 | bundle_file = ::File.expand_path("../smatrix_ruby.so", __FILE__) 5 | require bundle_file if ::File.exist? bundle_file 6 | 7 | -------------------------------------------------------------------------------- /src/smatrix.c: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "smatrix.h" 19 | #include "smatrix_private.h" 20 | 21 | // TODO 22 | // + make ioqueue fifo 23 | // + smatrix_gc() 24 | // + ftruncate in larger blocks 25 | // + aquire lock on file to prevent concurrent access 26 | // + check correct endianess on file open 27 | // + proper error handling / return codes for smatrix_open 28 | // + file free list 29 | 30 | /* 31 | 32 | libsmatrix file format (augmented BNF): 33 | --------------------------------------- 34 | 35 | FILE ::= FILE_HEADER ; header size is 512 bytes 36 | FILE_BODY 37 | 38 | FILE_HEADER ::= <8 Bytes 0x17> ; uint64_t, magic number 39 | CMAP_HEAD_FPOS ; uint64_t 40 | <496 Bytes 0x0> ; padding to 512 bytes 41 | 42 | FILE_BODY ::= *( CMAP_BLOCK | RMAP_BLOCK ) 43 | 44 | CMAP_BLOCK ::= CMAP_BLOCK_SIZE ; uint64_t 45 | CMAP_BLOCK_NEXT ; uint64_t, file offset 46 | *( CMAP_ENTRY ) ; 12 bytes each 47 | 48 | CMAP_ENTRY ::= CMAP_ENTRY_KEY ; uint32_t 49 | CMAP_ENTRY_VALUE ; uint64_t 50 | 51 | CMAP_ENTRY_KEY ::= ; key / first dimension 52 | CMAP_ENTRY_VALUE ::= ; file offset of the RMAP_BLOCK 53 | CMAP_HEAD_FPOS ::= ; file offset of the first CMAP_BLOCK 54 | CMAP_BLOCK_SIZE ::= ; number of entries in this block 55 | CMAP_BLOCK_NEXT ::= ; file offset of the next block or 0 56 | 57 | RMAP_BLOCK ::= <8 Bytes 0x23> ; uint64_t, magic number 58 | RMAP_BLOCK_SIZE ; uint64_t 59 | *( RMAP_SLOT ) ; 8 bytes each 60 | 61 | RMAP_SLOT ::= RMAP_ENTRY ; used hashmap slot 62 | | RMAP_SLOT_UNUSED ; unused hashmap slot 63 | 64 | RMAP_ENTRY ::= RMAP_ENTRY_KEY ; uint32_t 65 | RMAP_ENTRY_VALUE ; uint32_t 66 | 67 | RMAP_SLOT_UNUSED ::= <8 Bytes 0x0> ; empty slot 68 | RMAP_ENTRY_KEY ::= ; key / second dimension 69 | RMAP_ENTRY_VALUE ::= ; value 70 | RMAP_BLOCK_SIZE ::= ; number of slots in this block 71 | 72 | */ 73 | 74 | smatrix_t* smatrix_open(const char* fname) { 75 | smatrix_t* self = calloc(1, sizeof(smatrix_t)); 76 | 77 | if (self == NULL) 78 | return NULL; 79 | 80 | self->ioqueue = NULL; 81 | self->lock.count = 0; 82 | self->lock.mutex = 0; 83 | self->shutdown = 0; 84 | 85 | if (!fname) { 86 | smatrix_cmap_init(self); 87 | return self; 88 | } 89 | 90 | self->fd = open(fname, O_RDWR | O_CREAT, 00600); 91 | 92 | if (self->fd == -1) { 93 | perror("cannot open file"); 94 | free(self); 95 | return NULL; 96 | } 97 | 98 | self->fpos = lseek(self->fd, 0, SEEK_END); 99 | 100 | if (self->fpos == 0) { 101 | smatrix_fcreate(self); 102 | } else { 103 | smatrix_fload(self); 104 | } 105 | 106 | if (pthread_create(&self->iothread, NULL, &smatrix_io, self)) { 107 | smatrix_error("can't start the IO thread"); 108 | } 109 | 110 | return self; 111 | } 112 | 113 | void smatrix_close(smatrix_t* self) { 114 | void* retval; 115 | uint64_t pos; 116 | 117 | self->shutdown = 1; 118 | pthread_join(self->iothread, &retval); 119 | 120 | for (pos = 0; pos < self->cmap.size; pos++) { 121 | if (self->cmap.data[pos].flags & SMATRIX_CMAP_SLOT_USED) { 122 | smatrix_rmap_free(self, self->cmap.data[pos].rmap); 123 | } 124 | } 125 | 126 | smatrix_cmap_free(self, &self->cmap); 127 | 128 | if (self->fd) { 129 | close(self->fd); 130 | } 131 | 132 | free(self); 133 | } 134 | 135 | uint64_t smatrix_falloc(smatrix_t* self, uint64_t bytes) { 136 | smatrix_lock_getmutex(&self->lock); 137 | 138 | uint64_t old = self->fpos; 139 | uint64_t new = old + bytes; 140 | 141 | if (ftruncate(self->fd, new) == -1) { 142 | smatrix_error("truncate() failed"); 143 | } 144 | 145 | self->fpos = new; 146 | 147 | smatrix_lock_release(&self->lock); 148 | return old; 149 | } 150 | 151 | inline void* smatrix_malloc(smatrix_t* self, uint64_t bytes) { 152 | __sync_add_and_fetch(&self->mem, bytes); 153 | 154 | void* ptr = malloc(bytes); 155 | 156 | if (ptr == NULL) { 157 | smatrix_error("malloc() failed"); 158 | abort(); 159 | } 160 | 161 | return ptr; 162 | } 163 | 164 | inline void smatrix_mfree(smatrix_t* self, uint64_t bytes) { 165 | __sync_sub_and_fetch(&self->mem, bytes); 166 | } 167 | 168 | void smatrix_ffree(smatrix_t* self, uint64_t fpos, uint64_t bytes) { 169 | (void) self; 170 | (void) fpos; 171 | (void) bytes; 172 | } 173 | 174 | uint32_t smatrix_get(smatrix_t* self, uint32_t x, uint32_t y) { 175 | smatrix_ref_t ref; 176 | uint32_t retval = 0; 177 | 178 | smatrix_lookup(self, &ref, x, y, 0); 179 | 180 | if (ref.slot) 181 | retval = ref.slot->value; 182 | 183 | smatrix_decref(self, &ref); 184 | return retval; 185 | } 186 | 187 | // returns a whole row as an array of uint32_t's, odd slots contain indexes, even slots contain 188 | // values. example: [index, value, index, value...] 189 | uint32_t smatrix_getrow(smatrix_t* self, uint32_t x, uint32_t* ret, size_t ret_len) { 190 | smatrix_ref_t ref; 191 | uint32_t pos, num = 0; 192 | 193 | smatrix_lookup(self, &ref, x, 0, 0); 194 | 195 | if (ref.rmap) { 196 | for (pos = 0; pos < ref.rmap->size; pos++) { 197 | if (!ref.rmap->data[pos].key && !ref.rmap->data[pos].value) 198 | continue; 199 | 200 | ret[num * 2] = ref.rmap->data[pos].key; 201 | ret[num * 2 + 1] = ref.rmap->data[pos].value; 202 | 203 | if ((++num * 2 * sizeof(uint32_t)) >= ret_len) 204 | break; 205 | } 206 | } 207 | 208 | smatrix_decref(self, &ref); 209 | return num; 210 | } 211 | 212 | uint32_t smatrix_rowlen(smatrix_t* self, uint32_t x) { 213 | smatrix_ref_t ref; 214 | uint32_t len = 0; 215 | 216 | smatrix_lookup(self, &ref, x, 0, 0); 217 | 218 | if (ref.rmap) 219 | len = ref.rmap->used; 220 | 221 | smatrix_decref(self, &ref); 222 | return len; 223 | } 224 | 225 | uint32_t smatrix_set(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value) { 226 | smatrix_ref_t ref; 227 | uint32_t retval; 228 | 229 | smatrix_lookup(self, &ref, x, y, 1); 230 | retval = (ref.slot->value = value); 231 | smatrix_decref(self, &ref); 232 | 233 | return retval; 234 | } 235 | 236 | uint32_t smatrix_incr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value) { 237 | smatrix_ref_t ref; 238 | uint32_t retval; 239 | 240 | smatrix_lookup(self, &ref, x, y, 1); 241 | retval = (ref.slot->value += value); 242 | smatrix_decref(self, &ref); 243 | 244 | return retval; 245 | } 246 | 247 | uint32_t smatrix_decr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value) { 248 | smatrix_ref_t ref; 249 | uint32_t retval; 250 | 251 | smatrix_lookup(self, &ref, x, y, 1); 252 | retval = (ref.slot->value -= value); 253 | smatrix_decref(self, &ref); 254 | 255 | return retval; 256 | } 257 | 258 | void smatrix_lookup(smatrix_t* self, smatrix_ref_t* ref, uint32_t x, uint32_t y, int write) { 259 | int mutex = 0; 260 | smatrix_rmap_t* rmap; 261 | smatrix_rmap_slot_t* slot; 262 | 263 | ref->rmap = NULL; 264 | ref->slot = NULL; 265 | ref->write = write; 266 | 267 | rmap = smatrix_cmap_lookup(self, &self->cmap, x, write); 268 | 269 | if (rmap == NULL) { 270 | return; 271 | } 272 | 273 | if (write) { 274 | smatrix_lock_decref(&rmap->lock); 275 | smatrix_lock_getmutex(&rmap->lock); 276 | mutex = 1; 277 | } 278 | 279 | if (rmap->size == 0) { 280 | if (!mutex) { 281 | smatrix_lock_decref(&rmap->lock); 282 | smatrix_lock_getmutex(&rmap->lock); 283 | mutex = 1; 284 | } 285 | 286 | if (rmap->size == 0) { 287 | smatrix_rmap_load(self, rmap); 288 | } 289 | } 290 | 291 | ref->rmap = rmap; 292 | 293 | if (mutex && !write) { 294 | smatrix_lock_dropmutex(&rmap->lock); 295 | } 296 | 297 | slot = smatrix_rmap_probe(rmap, y); 298 | 299 | if (slot != NULL && slot->key == y) { 300 | ref->slot = slot; 301 | } else if (write) { 302 | ref->slot = smatrix_rmap_insert(self, rmap, y); 303 | } 304 | } 305 | 306 | void smatrix_decref(smatrix_t* self, smatrix_ref_t* ref) { 307 | if (!ref->rmap) { 308 | return; 309 | } 310 | 311 | if (ref->write) { 312 | if (self->fd) { 313 | // FIXPAUL: this will sync the whole rmap. if only one slot changed this is a lot of overhead... 314 | smatrix_rmap_sync_defer(self, ref->rmap); 315 | } 316 | 317 | smatrix_lock_release(&ref->rmap->lock); 318 | } else { 319 | smatrix_lock_decref(&ref->rmap->lock); 320 | } 321 | } 322 | 323 | void smatrix_rmap_init(smatrix_t* self, smatrix_rmap_t* rmap, uint32_t size) { 324 | if (size > 0) { 325 | size_t bytes = sizeof(smatrix_rmap_slot_t) * size; 326 | 327 | rmap->data = smatrix_malloc(self, bytes); 328 | memset(rmap->data, 0, bytes); 329 | } else { 330 | rmap->data = NULL; 331 | } 332 | 333 | rmap->size = size; 334 | rmap->used = 0; 335 | rmap->fpos = 0; 336 | rmap->flags = 0; 337 | rmap->lock.count = 0; 338 | rmap->lock.mutex = 0; 339 | } 340 | 341 | 342 | // you need to hold a write lock on rmap to call this function safely 343 | smatrix_rmap_slot_t* smatrix_rmap_insert(smatrix_t* self, smatrix_rmap_t* rmap, uint32_t key) { 344 | smatrix_rmap_slot_t* slot; 345 | 346 | if (rmap->used > rmap->size / 2) { 347 | smatrix_rmap_resize(self, rmap); 348 | } 349 | 350 | slot = smatrix_rmap_probe(rmap, key); 351 | assert(slot != NULL); 352 | 353 | if (!slot->key || slot->key != key) { 354 | rmap->used++; 355 | slot->key = key; 356 | slot->value = 0; 357 | } 358 | 359 | return slot; 360 | } 361 | 362 | // you need to hold a read or write lock on rmap to call this function safely 363 | smatrix_rmap_slot_t* smatrix_rmap_probe(smatrix_rmap_t* rmap, uint32_t key) { 364 | uint64_t n, pos; 365 | 366 | pos = key % rmap->size; 367 | 368 | // linear probing 369 | for (n = 0; n < rmap->size; n++) { 370 | if (rmap->data[pos].key == key) 371 | break; 372 | 373 | if (!rmap->data[pos].key && !rmap->data[pos].value) 374 | break; 375 | 376 | pos = (pos + 1) % rmap->size; 377 | } 378 | 379 | return &rmap->data[pos]; 380 | } 381 | 382 | // you need to hold a write lock on rmap in order to call this function safely 383 | void smatrix_rmap_resize(smatrix_t* self, smatrix_rmap_t* rmap) { 384 | uint64_t pos, bytes, old_size, new_size; 385 | smatrix_rmap_slot_t* slot; 386 | smatrix_rmap_t new; 387 | 388 | old_size = rmap->size; 389 | new_size = rmap->size * 2; 390 | bytes = sizeof(smatrix_rmap_slot_t) * new_size; 391 | 392 | new.size = new_size; 393 | new.used = 0; 394 | new.data = smatrix_malloc(self, bytes); 395 | memset(new.data, 0, bytes); 396 | 397 | for (pos = 0; pos < rmap->size; pos++) { 398 | if (!rmap->data[pos].key && !rmap->data[pos].value) 399 | continue; 400 | 401 | slot = smatrix_rmap_insert(self, &new, rmap->data[pos].key); 402 | slot->value = rmap->data[pos].value; 403 | } 404 | 405 | smatrix_mfree(self, sizeof(smatrix_rmap_slot_t) * old_size); 406 | free(rmap->data); 407 | 408 | rmap->data = new.data; 409 | rmap->size = new.size; 410 | rmap->used = new.used; 411 | 412 | if (self->fd) { 413 | rmap->flags |= SMATRIX_RMAP_FLAG_RESIZED; 414 | smatrix_rmap_sync_defer(self, rmap); 415 | } 416 | } 417 | 418 | inline void smatrix_rmap_sync_defer(smatrix_t* self, smatrix_rmap_t* rmap) { 419 | if ((rmap->flags & SMATRIX_RMAP_FLAG_DIRTY) > 0) { 420 | return; 421 | } 422 | 423 | rmap->flags |= SMATRIX_RMAP_FLAG_DIRTY; 424 | smatrix_ioqueue_add(self, rmap); 425 | } 426 | 427 | void smatrix_rmap_sync(smatrix_t* self, smatrix_rmap_t* rmap) { 428 | uint64_t bytes; 429 | 430 | if ((rmap->flags & SMATRIX_RMAP_FLAG_RESIZED) > 0) { 431 | // FIXPAUL can't ffree without knowing the old size.. just dividing by 2 seems too hacky 432 | //bytes = SMATRIX_RMAP_SLOT_SIZE * rmap->size + SMATRIX_RMAP_HEAD_SIZE; 433 | //smatrix_ffree(self, rmap->fpos, bytes); 434 | 435 | rmap->fpos = 0; 436 | } 437 | 438 | if (rmap->fpos == 0) { 439 | bytes = SMATRIX_RMAP_SLOT_SIZE * rmap->size + SMATRIX_RMAP_HEAD_SIZE; 440 | rmap->fpos = smatrix_falloc(self, bytes); 441 | 442 | smatrix_rmap_write_batch(self, rmap, 1); 443 | smatrix_cmap_write(self, rmap); 444 | } else { 445 | // FIXPAUL write only the actualy dirty slots! 446 | smatrix_rmap_write_batch(self, rmap, 1); 447 | } 448 | 449 | rmap->flags &= ~SMATRIX_RMAP_FLAG_DIRTY; 450 | rmap->flags &= ~SMATRIX_RMAP_FLAG_RESIZED; 451 | } 452 | 453 | // the caller of this must hold a read lock on rmap 454 | void smatrix_rmap_write_batch(smatrix_t* self, smatrix_rmap_t* rmap, int full) { 455 | uint64_t pos = 0, bytes, buf_pos, rmap_size = rmap->size; 456 | char *buf; 457 | 458 | if (full) { 459 | bytes = rmap->size * SMATRIX_RMAP_SLOT_SIZE; 460 | bytes += SMATRIX_RMAP_HEAD_SIZE; 461 | } else { 462 | bytes = SMATRIX_RMAP_HEAD_SIZE; 463 | } 464 | 465 | buf = smatrix_malloc(self, bytes); 466 | 467 | memset(buf, 0, bytes); 468 | memset(buf, 0x23, 8); 469 | memcpy(buf + 8, &rmap_size, 8); 470 | 471 | if (full) { 472 | buf_pos = SMATRIX_RMAP_HEAD_SIZE; 473 | 474 | for (pos = 0; pos < rmap->size; pos++) { 475 | memcpy(buf + buf_pos, &rmap->data[pos].key, 4); 476 | memcpy(buf + buf_pos + 4, &rmap->data[pos].value, 4); 477 | buf_pos += SMATRIX_RMAP_SLOT_SIZE; 478 | } 479 | } 480 | 481 | smatrix_write(self, rmap->fpos, buf, bytes); 482 | } 483 | 484 | void smatrix_rmap_write_slot(smatrix_t* self, smatrix_rmap_t* rmap, smatrix_rmap_slot_t* slot) { 485 | uint64_t rmap_pos, fpos; 486 | char* buf = smatrix_malloc(self, SMATRIX_RMAP_SLOT_SIZE); 487 | 488 | rmap_pos = slot - rmap->data; 489 | fpos = rmap_pos * SMATRIX_RMAP_SLOT_SIZE; 490 | fpos += rmap->fpos + SMATRIX_RMAP_HEAD_SIZE; 491 | 492 | memcpy(buf, &slot->key, 4); 493 | memcpy(buf + 4, &slot->value, 4); 494 | 495 | smatrix_write(self, fpos, buf, SMATRIX_RMAP_SLOT_SIZE); 496 | } 497 | 498 | // caller must hold writelock on rmap 499 | void smatrix_rmap_load(smatrix_t* self, smatrix_rmap_t* rmap) { 500 | uint64_t pos, read_bytes, mem_bytes, disk_bytes, rmap_size; 501 | unsigned char meta_buf[SMATRIX_RMAP_HEAD_SIZE] = {0}, *buf; 502 | 503 | if (rmap->flags & SMATRIX_RMAP_FLAG_LOADED) 504 | return; 505 | 506 | if (!rmap->size) { 507 | if (pread(self->fd, &meta_buf, SMATRIX_RMAP_HEAD_SIZE, rmap->fpos) != SMATRIX_RMAP_HEAD_SIZE) { 508 | smatrix_error("pread() failed (rmap_load). corrupt file?"); 509 | } 510 | 511 | if (memcmp(&meta_buf, &SMATRIX_RMAP_MAGIC, SMATRIX_RMAP_MAGIC_SIZE)) { 512 | smatrix_error("file is corrupt (rmap_load)"); 513 | } 514 | 515 | rmap_size = *((uint64_t *) &meta_buf[8]); 516 | rmap->size = rmap_size; 517 | assert(rmap->size > 0); 518 | } 519 | 520 | mem_bytes = rmap->size * sizeof(smatrix_rmap_slot_t); 521 | disk_bytes = rmap->size * SMATRIX_RMAP_SLOT_SIZE; 522 | rmap->used = 0; 523 | rmap->data = smatrix_malloc(self, mem_bytes); 524 | buf = smatrix_malloc(self, disk_bytes); 525 | 526 | memset(rmap->data, 0, mem_bytes); 527 | read_bytes = pread(self->fd, buf, disk_bytes, rmap->fpos + SMATRIX_RMAP_HEAD_SIZE); 528 | 529 | if (read_bytes != disk_bytes) { 530 | smatrix_error("read() failed (rmap_load)"); 531 | } 532 | 533 | for (pos = 0; pos < rmap->size; pos++) { 534 | memcpy(&rmap->data[pos].value, buf + pos * SMATRIX_RMAP_SLOT_SIZE + 4, 4); 535 | 536 | if (rmap->data[pos].value) { 537 | memcpy(&rmap->data[pos].key, buf + pos * SMATRIX_RMAP_SLOT_SIZE, 4); 538 | rmap->used++; 539 | } 540 | } 541 | 542 | rmap->flags = SMATRIX_RMAP_FLAG_LOADED; 543 | smatrix_mfree(self, disk_bytes); 544 | free(buf); 545 | } 546 | 547 | // caller must hold a write lock on rmap 548 | void smatrix_rmap_swap(smatrix_t* self, smatrix_rmap_t* rmap) { 549 | rmap->flags &= ~SMATRIX_RMAP_FLAG_LOADED; 550 | smatrix_mfree(self, sizeof(smatrix_rmap_slot_t) * rmap->size); 551 | free(rmap->data); 552 | } 553 | 554 | void smatrix_rmap_free(smatrix_t* self, smatrix_rmap_t* rmap) { 555 | if (rmap->data) { 556 | smatrix_mfree(self, sizeof(smatrix_rmap_slot_t) * rmap->size); 557 | free(rmap->data); 558 | } 559 | 560 | smatrix_mfree(self, sizeof(smatrix_rmap_t)); 561 | free(rmap); 562 | } 563 | 564 | void smatrix_fcreate(smatrix_t* self) { 565 | char buf[SMATRIX_META_SIZE]; 566 | smatrix_falloc(self, SMATRIX_META_SIZE); 567 | 568 | memset(&buf, 0, SMATRIX_META_SIZE); 569 | memset(&buf, 0x17, 8); 570 | pwrite(self->fd, &buf, SMATRIX_META_SIZE, 0); 571 | 572 | smatrix_cmap_init(self); 573 | smatrix_cmap_mkblock(self, &self->cmap); 574 | } 575 | 576 | void smatrix_fload(smatrix_t* self) { 577 | char buf[SMATRIX_META_SIZE]; 578 | uint64_t read, cmap_head_fpos; 579 | 580 | read = pread(self->fd, &buf, SMATRIX_META_SIZE, 0); 581 | 582 | if (read != SMATRIX_META_SIZE) { 583 | smatrix_error("invalid file header\n"); 584 | abort(); 585 | } 586 | 587 | if (buf[0] != 0x17 || buf[1] != 0x17) { 588 | smatrix_error("invalid file header\n"); 589 | abort(); 590 | } 591 | 592 | memcpy(&cmap_head_fpos, &buf[8], 8); 593 | 594 | smatrix_cmap_init(self); 595 | smatrix_cmap_load(self, cmap_head_fpos); 596 | } 597 | 598 | void smatrix_cmap_init(smatrix_t* self) { 599 | uint64_t bytes; 600 | 601 | self->cmap.size = SMATRIX_CMAP_INITIAL_SIZE; 602 | self->cmap.used = 0; 603 | self->cmap.lock.count = 0; 604 | self->cmap.lock.mutex = 0; 605 | self->cmap.block_fpos = 0; 606 | self->cmap.block_used = 0; 607 | self->cmap.block_size = 0; 608 | 609 | bytes = sizeof(smatrix_cmap_slot_t) * self->cmap.size; 610 | self->cmap.data = smatrix_malloc(self, bytes); 611 | memset(self->cmap.data, 0, bytes); 612 | } 613 | 614 | void smatrix_cmap_free(smatrix_t* self, smatrix_cmap_t* cmap) { 615 | uint64_t bytes = sizeof(smatrix_cmap_slot_t) * cmap->size; 616 | smatrix_mfree(self, bytes); 617 | free(cmap->data); 618 | } 619 | 620 | // caller must hold no locks on cmap! 621 | smatrix_rmap_t* smatrix_cmap_lookup(smatrix_t* self, smatrix_cmap_t* cmap, uint32_t key, int create) { 622 | smatrix_cmap_slot_t* slot; 623 | smatrix_rmap_t* rmap; 624 | 625 | smatrix_lock_incref(&cmap->lock); 626 | slot = smatrix_cmap_probe(cmap, key); 627 | 628 | if (slot && slot->key == key && (slot->flags & SMATRIX_CMAP_SLOT_USED) != 0) { 629 | rmap = slot->rmap; 630 | smatrix_lock_incref(&rmap->lock); 631 | smatrix_lock_decref(&cmap->lock); 632 | return rmap; 633 | } 634 | 635 | smatrix_lock_decref(&cmap->lock); 636 | 637 | if (!create) { 638 | return NULL; 639 | } 640 | 641 | rmap = smatrix_malloc(self, sizeof(smatrix_rmap_t)); 642 | smatrix_rmap_init(self, rmap, SMATRIX_RMAP_INITIAL_SIZE); 643 | rmap->key = key; 644 | 645 | smatrix_lock_getmutex(&cmap->lock); 646 | slot = smatrix_cmap_insert(self, cmap, key); 647 | 648 | if (slot->rmap) { 649 | smatrix_rmap_free(self, rmap); 650 | rmap = slot->rmap; 651 | 652 | smatrix_lock_incref(&rmap->lock); 653 | smatrix_lock_release(&cmap->lock); 654 | } else { 655 | slot->rmap = rmap; 656 | 657 | if (self->fd) { 658 | rmap->meta_fpos = smatrix_cmap_falloc(self, &self->cmap); 659 | } 660 | 661 | smatrix_lock_incref(&rmap->lock); 662 | smatrix_lock_release(&cmap->lock); 663 | 664 | if (self->fd) { 665 | smatrix_rmap_sync_defer(self, rmap); 666 | } 667 | } 668 | 669 | return rmap; 670 | } 671 | 672 | // caller must hold a read lock on cmap! 673 | smatrix_cmap_slot_t* smatrix_cmap_probe(smatrix_cmap_t* cmap, uint32_t key) { 674 | unsigned pos = key; 675 | smatrix_cmap_slot_t* slot; 676 | 677 | slot = cmap->data + (key % cmap->size); 678 | 679 | for (;;) { 680 | if ((slot->flags & SMATRIX_CMAP_SLOT_USED) == 0) { 681 | return slot; 682 | } 683 | 684 | if (slot->key == key) { 685 | return slot; 686 | } 687 | 688 | pos++; 689 | slot = cmap->data + (pos % cmap->size); 690 | } 691 | 692 | return slot; 693 | } 694 | 695 | smatrix_cmap_slot_t* smatrix_cmap_insert(smatrix_t* self, smatrix_cmap_t* cmap, uint32_t key) { 696 | smatrix_cmap_slot_t* slot; 697 | 698 | if (cmap->used * 4 >= cmap->size * 3) { 699 | smatrix_cmap_resize(self, cmap); 700 | } 701 | 702 | slot = smatrix_cmap_probe(cmap, key); 703 | assert(slot != NULL); 704 | 705 | if ((slot->flags & SMATRIX_CMAP_SLOT_USED) == 0 || slot->key != key) { 706 | cmap->used++; 707 | slot->key = key; 708 | slot->flags = SMATRIX_CMAP_SLOT_USED; 709 | slot->rmap = NULL; 710 | } 711 | 712 | return slot; 713 | } 714 | 715 | void smatrix_cmap_resize(smatrix_t* self, smatrix_cmap_t* cmap) { 716 | uint64_t new_bytes, pos; 717 | smatrix_cmap_slot_t *slot; 718 | smatrix_cmap_t new; 719 | 720 | new.used = 0; 721 | new.size = cmap->size * 2; 722 | new_bytes = sizeof(smatrix_cmap_slot_t) * new.size; 723 | new.data = smatrix_malloc(self, new_bytes); 724 | 725 | smatrix_mfree(self, sizeof(smatrix_cmap_slot_t) * cmap->size); 726 | memset(new.data, 0, new_bytes); 727 | 728 | for (pos = 0; pos < cmap->size; pos++) { 729 | if ((cmap->data[pos].flags & SMATRIX_CMAP_SLOT_USED) == 0) 730 | continue; 731 | 732 | slot = smatrix_cmap_insert(self, &new, cmap->data[pos].key); 733 | slot->rmap = cmap->data[pos].rmap; 734 | } 735 | 736 | free(cmap->data); 737 | 738 | cmap->data = new.data; 739 | cmap->size = new.size; 740 | cmap->used = new.used; 741 | } 742 | 743 | // caller must hold a write lock on cmap 744 | uint64_t smatrix_cmap_falloc(smatrix_t* self, smatrix_cmap_t* cmap) { 745 | uint64_t fpos; 746 | 747 | if (cmap->block_used >= cmap->block_size) { 748 | smatrix_cmap_mkblock(self, cmap); 749 | } 750 | 751 | fpos = cmap->block_fpos + SMATRIX_CMAP_HEAD_SIZE; 752 | fpos += cmap->block_used * SMATRIX_CMAP_SLOT_SIZE; 753 | 754 | cmap->block_used++; 755 | 756 | return fpos; 757 | } 758 | 759 | void smatrix_cmap_mkblock(smatrix_t* self, smatrix_cmap_t* cmap) { 760 | uint64_t bytes, meta_fpos; 761 | char* buf = smatrix_malloc(self, SMATRIX_CMAP_HEAD_SIZE); 762 | char* meta_buf = smatrix_malloc(self, 8); 763 | 764 | meta_fpos = cmap->block_fpos + 8; 765 | 766 | bytes = SMATRIX_CMAP_BLOCK_SIZE * SMATRIX_CMAP_SLOT_SIZE; 767 | bytes += SMATRIX_CMAP_HEAD_SIZE; 768 | 769 | cmap->block_fpos = smatrix_falloc(self, bytes); 770 | cmap->block_used = 0; 771 | cmap->block_size = SMATRIX_CMAP_BLOCK_SIZE; 772 | 773 | memcpy(meta_buf, &cmap->block_fpos, 8); 774 | memcpy(buf, &cmap->block_size, 8); 775 | memset(buf + 8, 0, 8); 776 | 777 | smatrix_write(self, cmap->block_fpos, buf, SMATRIX_CMAP_HEAD_SIZE); 778 | smatrix_write(self, meta_fpos, meta_buf, 8); 779 | } 780 | 781 | void smatrix_cmap_write(smatrix_t* self, smatrix_rmap_t* rmap) { 782 | char* buf = smatrix_malloc(self, SMATRIX_CMAP_SLOT_SIZE); 783 | 784 | memcpy(buf, &rmap->key, 4); 785 | memcpy(buf + 4, &rmap->fpos, 8); 786 | 787 | smatrix_write(self, rmap->meta_fpos, buf, SMATRIX_CMAP_SLOT_SIZE); 788 | } 789 | 790 | void smatrix_cmap_load(smatrix_t* self, uint64_t head_fpos) { 791 | smatrix_rmap_t* rmap; 792 | unsigned char meta_buf[SMATRIX_CMAP_HEAD_SIZE], *buf; 793 | ssize_t bytes, pos; 794 | uint64_t fpos, value; 795 | 796 | for (fpos = head_fpos; fpos;) { 797 | self->cmap.block_fpos = fpos; 798 | 799 | if (pread(self->fd, &meta_buf, SMATRIX_CMAP_HEAD_SIZE, fpos) != SMATRIX_CMAP_HEAD_SIZE) { 800 | smatrix_error("pread() failed (cmap_load). corrupt file?"); 801 | } 802 | 803 | fpos += SMATRIX_CMAP_HEAD_SIZE; 804 | bytes = *((uint64_t *) &meta_buf) * SMATRIX_CMAP_SLOT_SIZE; 805 | buf = smatrix_malloc(self, bytes); 806 | 807 | if (pread(self->fd, buf, bytes, fpos) != bytes) { 808 | smatrix_error("pread() failed (cmap_load). corrupt file?"); 809 | } 810 | 811 | for (pos = 0; pos < bytes; pos += SMATRIX_CMAP_SLOT_SIZE) { 812 | value = *((uint64_t *) &buf[pos + 4]); 813 | 814 | if (!value) 815 | break; 816 | 817 | rmap = smatrix_malloc(self, sizeof(smatrix_rmap_t)); 818 | smatrix_rmap_init(self, rmap, 0); 819 | rmap->key = *((uint32_t *) &buf[pos]); 820 | rmap->meta_fpos = fpos + pos; 821 | rmap->fpos = value; 822 | 823 | smatrix_cmap_insert(self, &self->cmap, rmap->key)->rmap = rmap; 824 | } 825 | 826 | smatrix_mfree(self, bytes); 827 | free(buf); 828 | fpos = *((uint64_t *) &meta_buf[8]); 829 | } 830 | } 831 | 832 | void smatrix_write(smatrix_t* self, uint64_t fpos, char* data, uint64_t bytes) { 833 | if (pwrite(self->fd, data, bytes, fpos) != (ssize_t) bytes) { 834 | smatrix_error("write() failed"); 835 | } 836 | 837 | free(data); 838 | smatrix_mfree(self, bytes); 839 | } 840 | 841 | // the caller of this function must have called smatrix_lock_incref before 842 | // returns 0 for success, 1 for failure 843 | void smatrix_lock_getmutex(smatrix_lock_t* lock) { 844 | assert(lock->count > 0); 845 | 846 | for (;;) { 847 | if (__sync_bool_compare_and_swap(&lock->mutex, 0, 1)) { 848 | break; 849 | } 850 | 851 | while (lock->mutex != 0) { 852 | asm("pause"); 853 | } 854 | } 855 | 856 | while (lock->count > 0) { 857 | asm("pause"); 858 | } 859 | } 860 | 861 | void smatrix_lock_dropmutex(smatrix_lock_t* lock) { 862 | assert(lock->count == 0); 863 | asm("lock incw (%0)" : : "c" (&lock->count)); 864 | lock->mutex = 0; 865 | } 866 | 867 | void smatrix_lock_release(smatrix_lock_t* lock) { 868 | lock->mutex = 0; 869 | } 870 | 871 | inline void smatrix_lock_incref(smatrix_lock_t* lock) { 872 | for (;;) { 873 | asm("lock incw (%0)" : : "c" (&lock->count)); 874 | 875 | if (lock->mutex == 0) { 876 | return; 877 | } 878 | 879 | asm("lock decw (%0)" : : "c" (&lock->count)); 880 | 881 | while (lock->mutex != 0) { 882 | asm("pause"); 883 | } 884 | } 885 | } 886 | 887 | inline void smatrix_lock_decref(smatrix_lock_t* lock) { 888 | asm("lock decw (%0)" : : "c" (&lock->count)); 889 | } 890 | 891 | void smatrix_error(const char* msg) { 892 | printf("libsmatrix error: %s", msg); 893 | abort(); 894 | } 895 | 896 | void smatrix_ioqueue_add(smatrix_t* self, smatrix_rmap_t* rmap) { 897 | smatrix_ref_t* ref; 898 | 899 | ref = smatrix_malloc(self, sizeof(smatrix_ref_t)); 900 | ref->rmap = rmap; 901 | 902 | smatrix_lock_getmutex(&self->lock); 903 | 904 | ref->next = self->ioqueue; 905 | self->ioqueue = ref; 906 | 907 | smatrix_lock_release(&self->lock); 908 | } 909 | 910 | smatrix_rmap_t* smatrix_ioqueue_pop(smatrix_t* self) { 911 | smatrix_ref_t* ref; 912 | smatrix_rmap_t* rmap; 913 | 914 | smatrix_lock_getmutex(&self->lock); 915 | 916 | ref = self->ioqueue; 917 | 918 | if (ref == NULL) { 919 | smatrix_lock_release(&self->lock); 920 | return NULL; 921 | } 922 | 923 | self->ioqueue = ref->next; 924 | smatrix_lock_release(&self->lock); 925 | 926 | rmap = ref->rmap; 927 | 928 | free(ref); 929 | smatrix_mfree(self, sizeof(smatrix_ref_t)); 930 | 931 | return rmap; 932 | } 933 | 934 | void* smatrix_io(void* self_) { 935 | smatrix_t* self = self_; 936 | smatrix_rmap_t* rmap; 937 | 938 | for (;;) { 939 | rmap = smatrix_ioqueue_pop(self); 940 | 941 | if (rmap == NULL) { 942 | if (self->shutdown) { 943 | break; 944 | } else { 945 | usleep(100000); 946 | continue; 947 | } 948 | } 949 | 950 | smatrix_lock_getmutex(&rmap->lock); 951 | 952 | if ((rmap->flags & SMATRIX_RMAP_FLAG_DIRTY) > 0) { 953 | smatrix_rmap_sync(self, rmap); 954 | } 955 | 956 | smatrix_lock_release(&rmap->lock); 957 | } 958 | 959 | return NULL; 960 | } 961 | -------------------------------------------------------------------------------- /src/smatrix.h: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #ifndef SMATRIX_H 13 | #define SMATRIX_H 14 | 15 | #define SMATRIX_META_SIZE 512 16 | #define SMATRIX_RMAP_FLAG_LOADED 4 17 | #define SMATRIX_RMAP_FLAG_DIRTY 8 18 | #define SMATRIX_RMAP_FLAG_RESIZED 16 19 | #define SMATRIX_RMAP_MAGIC "\x23\x23\x23\x23\x23\x23\x23\x23" 20 | #define SMATRIX_RMAP_MAGIC_SIZE 8 21 | #define SMATRIX_RMAP_INITIAL_SIZE 16 22 | #define SMATRIX_RMAP_SLOT_SIZE 8 23 | #define SMATRIX_RMAP_HEAD_SIZE 16 24 | #define SMATRIX_CMAP_INITIAL_SIZE 65536 25 | #define SMATRIX_CMAP_SLOT_SIZE 12 26 | #define SMATRIX_CMAP_HEAD_SIZE 16 27 | #define SMATRIX_CMAP_BLOCK_SIZE 4194304 28 | #define SMATRIX_CMAP_SLOT_USED 1 29 | 30 | typedef struct { 31 | volatile uint16_t count; 32 | volatile uint16_t mutex; 33 | } smatrix_lock_t; 34 | 35 | typedef struct { 36 | uint32_t key; 37 | uint32_t value; 38 | } smatrix_rmap_slot_t; 39 | 40 | typedef struct { 41 | uint64_t fpos; 42 | uint64_t meta_fpos; 43 | uint32_t size; 44 | uint32_t used; 45 | uint32_t key; 46 | uint32_t flags; 47 | smatrix_rmap_slot_t* data; 48 | smatrix_lock_t lock; 49 | } smatrix_rmap_t; 50 | 51 | typedef struct { 52 | uint32_t flags; 53 | uint32_t key; 54 | smatrix_rmap_t* rmap; 55 | } smatrix_cmap_slot_t; 56 | 57 | typedef struct { 58 | uint64_t size; 59 | uint64_t used; 60 | uint64_t block_fpos; 61 | uint64_t block_used; 62 | uint64_t block_size; 63 | smatrix_cmap_slot_t* data; 64 | smatrix_lock_t lock; 65 | } smatrix_cmap_t; 66 | 67 | typedef struct smatrix_ref_s smatrix_ref_t; 68 | 69 | struct smatrix_ref_s { 70 | int write; 71 | smatrix_rmap_t* rmap; 72 | smatrix_rmap_slot_t* slot; 73 | smatrix_ref_t* next; 74 | }; 75 | 76 | typedef struct { 77 | int fd; 78 | int shutdown; 79 | uint64_t fpos; 80 | uint64_t mem; 81 | smatrix_ref_t* ioqueue; 82 | pthread_t iothread; 83 | smatrix_cmap_t cmap; 84 | smatrix_lock_t lock; 85 | } smatrix_t; 86 | 87 | smatrix_t* smatrix_open(const char* fname); 88 | uint32_t smatrix_get(smatrix_t* self, uint32_t x, uint32_t y); 89 | uint32_t smatrix_set(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 90 | uint32_t smatrix_incr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 91 | uint32_t smatrix_decr(smatrix_t* self, uint32_t x, uint32_t y, uint32_t value); 92 | uint32_t smatrix_rowlen(smatrix_t* self, uint32_t x); 93 | uint32_t smatrix_getrow(smatrix_t* self, uint32_t x, uint32_t* ret, size_t ret_len); 94 | void smatrix_close(smatrix_t* self); 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/smatrix_benchmark.c: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "smatrix.h" 20 | 21 | typedef struct { 22 | smatrix_t* smx; 23 | int threadn; 24 | int user1; 25 | } args_t; 26 | 27 | smatrix_t* smx_mem; 28 | 29 | void* benchmark_incr_mixed(void* args_) { 30 | args_t* args = (args_t*) args_; 31 | int i, n, r, o; 32 | 33 | o = 42 + args->threadn; 34 | smatrix_t* smx = args->smx; 35 | 36 | for (r = 0; r < args->user1; r++) { 37 | for (n = 0; n < 23; n++) { 38 | for (i = 0; i < 22; i++) { 39 | smatrix_incr(smx, n + o, i + o, 1); 40 | smatrix_incr(smx, i + o, n + o, 1); 41 | } 42 | } 43 | } 44 | 45 | return NULL; 46 | } 47 | 48 | void* benchmark_get_mixed(void* args_) { 49 | args_t* args = (args_t*) args_; 50 | int i, n, r, o; 51 | 52 | o = 42 + args->threadn; 53 | smatrix_t* smx = args->smx; 54 | 55 | for (r = 0; r < args->user1; r++) { 56 | for (n = 0; n < 23; n++) { 57 | for (i = 0; i < 22; i++) { 58 | smatrix_get(smx, n + o, i + o); 59 | smatrix_get(smx, i + o, n + o); 60 | } 61 | } 62 | } 63 | 64 | return NULL; 65 | } 66 | 67 | void* benchmark_incr_independent(void* args_) { 68 | args_t* args = (args_t*) args_; 69 | int i, r, o; 70 | 71 | smatrix_t* smx = args->smx; 72 | o = 123 + args->threadn; 73 | 74 | for (r = 0; r < args->user1; r++) { 75 | for (i = 0; i < 1000; i++) { 76 | smatrix_incr(smx, o, 1, 1); 77 | } 78 | } 79 | 80 | return NULL; 81 | } 82 | 83 | void* benchmark_incr_compete(void* args_) { 84 | args_t* args = (args_t*) args_; 85 | int i, r; 86 | 87 | smatrix_t* smx = args->smx; 88 | 89 | for (r = 0; r < args->user1; r++) { 90 | for (i = 0; i < 1000; i++) { 91 | smatrix_incr(smx, 23, 1, 1); 92 | } 93 | } 94 | 95 | return NULL; 96 | } 97 | 98 | void measure(void* (*cb)(void*), int nthreads, smatrix_t* smx, int user1) { 99 | char str[20] = " "; 100 | int n; 101 | pthread_t* threads; 102 | double elapsed; 103 | struct timeval t0, t1; 104 | void* retval; 105 | 106 | args_t* args = malloc(sizeof(args_t) * nthreads); 107 | threads = malloc(sizeof(pthread_t) * nthreads); 108 | 109 | gettimeofday(&t0, NULL); 110 | 111 | for (n = 0; n < nthreads; n++) { 112 | args[n].smx = smx; 113 | args[n].threadn = n; 114 | args[n].user1 = user1; 115 | pthread_create(threads + n, NULL, cb, &args[n]); 116 | } 117 | 118 | for (n = 0; n < nthreads; n++) { 119 | pthread_join(threads[n], &retval); 120 | } 121 | 122 | gettimeofday(&t1, NULL); 123 | 124 | elapsed = (double) (t1.tv_sec - t0.tv_sec) * 1000; 125 | elapsed += (double) (t1.tv_usec - t0.tv_usec) / 1000; 126 | 127 | str[snprintf(str, 20, "%.1fms", elapsed)] = ' '; 128 | 129 | free(threads); 130 | free(args); 131 | printf("%s", str); 132 | } 133 | 134 | void print_header(const char* title) { 135 | printf("TEST: %s\n", title); 136 | printf("---------------------------------------------------------------\n"); 137 | printf("T=1 T=2 T=4 T=8 T=16 T=32 \n"); 138 | } 139 | 140 | void test_incr(smatrix_t* smx_mem) { 141 | int n, max = 10; 142 | 143 | for (n = 0; n < max; n++) { 144 | measure(&benchmark_incr_mixed, 1, smx_mem, 1024); 145 | measure(&benchmark_incr_mixed, 2, smx_mem, 512); 146 | measure(&benchmark_incr_mixed, 4, smx_mem, 256); 147 | measure(&benchmark_incr_mixed, 8, smx_mem, 128); 148 | measure(&benchmark_incr_mixed, 16, smx_mem, 64); 149 | measure(&benchmark_incr_mixed, 32, smx_mem, 32); 150 | 151 | if (n < max - 1) { 152 | printf("\n"); 153 | } 154 | } 155 | 156 | printf("\n\n"); 157 | } 158 | 159 | void test_get(smatrix_t* smx_mem) { 160 | int n, max = 10; 161 | 162 | for (n = 0; n < max; n++) { 163 | measure(&benchmark_get_mixed, 1, smx_mem, 1024); 164 | measure(&benchmark_get_mixed, 2, smx_mem, 512); 165 | measure(&benchmark_get_mixed, 4, smx_mem, 256); 166 | measure(&benchmark_get_mixed, 8, smx_mem, 128); 167 | measure(&benchmark_get_mixed, 16, smx_mem, 64); 168 | measure(&benchmark_get_mixed, 32, smx_mem, 32); 169 | 170 | if (n < max - 1) { 171 | printf("\n"); 172 | } 173 | } 174 | 175 | printf("\n\n"); 176 | } 177 | 178 | int main(int argc, char** argv) { 179 | int num = 1024, threads = 4; 180 | smatrix_t* smx = NULL; 181 | 182 | if (argc < 2) { 183 | printf("usage: smatrix_benchmark [test] [times] [threads] [file]\n\n"); 184 | printf(" Available Tests:\n"); 185 | printf(" full test all methods\n"); 186 | printf(" incr test the incr method\n"); 187 | printf(" get test the get method\n\n"); 188 | printf(" Examples:\n"); 189 | printf(" $ smatrix_benchmark incr 1024 4\n"); 190 | printf(" $ smatrix_benchmark incr 1024 4 /tmp/test.smx\n"); 191 | printf(" $ smatrix_benchmark get 10000 2 /tmp/text.smx\n"); 192 | printf(" $ smatrix_benchmark full\n\n"); 193 | return 1; 194 | } 195 | 196 | if (argc > 2) { 197 | num = atoi(argv[2]); 198 | } 199 | 200 | if (argc > 3) { 201 | threads = atoi(argv[3]); 202 | } 203 | 204 | if (argc > 4) { 205 | smx = smatrix_open(argv[4]); 206 | } else { 207 | smx = smatrix_open(NULL); 208 | } 209 | 210 | if (!strcmp(argv[1], "incr")) { 211 | printf("testing: %ik x incr @ %i threads: ", num, threads); 212 | measure(&benchmark_incr_mixed, threads, smx, num / threads); 213 | printf("\n"); 214 | goto exit; 215 | } 216 | 217 | if (!strcmp(argv[1], "get")) { 218 | printf("testing: %ik x get @ %i threads: ", num, threads); 219 | measure(&benchmark_get_mixed, threads, smx, num / threads); 220 | printf("\n"); 221 | goto exit; 222 | } 223 | 224 | printf("libsmatrix benchmark [date]\n\n"); 225 | 226 | print_header("1 million x incr (memory)"); 227 | test_incr(smx); 228 | 229 | print_header("1 million x get (memory)"); 230 | test_get(smx); 231 | 232 | exit: 233 | smatrix_close(smx); 234 | 235 | return 0; 236 | } 237 | -------------------------------------------------------------------------------- /src/smatrix_jni.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | #include 10 | #include "smatrix_jni.h" 11 | #include "smatrix.h" 12 | 13 | #define _JM(X) Java_com_paulasmuth_libsmatrix_SparseMatrix_##X 14 | #define ERR_PTRNOTFOUND "can't find native object. maybe close() was already called" 15 | 16 | void throw_exception(JNIEnv* env, const char* error) { 17 | jclass exception = (*env)->FindClass(env, "java/lang/IllegalArgumentException"); 18 | (*env)->ThrowNew(env, exception, error); 19 | } 20 | 21 | void set_ptr(JNIEnv* env, jobject self, void* ptr_) { 22 | jclass cls; 23 | jfieldID fid; 24 | long ptr = (long) ptr_; 25 | 26 | cls = (*env)->FindClass(env, "com/paulasmuth/libsmatrix/SparseMatrix"); 27 | fid = (*env)->GetFieldID(env, cls, "ptr", "J"); 28 | 29 | (*env)->SetLongField(env, self, fid, ptr); 30 | } 31 | 32 | int get_ptr(JNIEnv* env, jobject self, void** ptr) { 33 | jclass cls; 34 | jfieldID fid; 35 | jlong ptr_; 36 | 37 | cls = (*env)->FindClass(env, "com/paulasmuth/libsmatrix/SparseMatrix"); 38 | fid = (*env)->GetFieldID(env, cls, "ptr", "J"); 39 | ptr_ = (*env)->GetLongField(env, self, fid); 40 | 41 | if (ptr_ > 0) { 42 | *ptr = (void *) ptr_; 43 | return 0; 44 | } else { 45 | throw_exception(env, ERR_PTRNOTFOUND); 46 | 47 | return 1; 48 | } 49 | } 50 | 51 | JNIEXPORT void JNICALL _JM(init) (JNIEnv* env, jobject self, jstring file_) { 52 | void* ptr; 53 | char* file = NULL; 54 | 55 | if (file_ != NULL) { 56 | file = (char *) (*env)->GetStringUTFChars(env, file_, 0); 57 | } 58 | 59 | ptr = smatrix_open(file); 60 | 61 | if (ptr == NULL) { 62 | throw_exception(env, "smatrix_open() failed"); 63 | } else { 64 | set_ptr(env, self, ptr); 65 | } 66 | 67 | if (file != NULL) { 68 | (*env)->ReleaseStringUTFChars(env, file_, file); 69 | } 70 | } 71 | 72 | JNIEXPORT void JNICALL _JM(close) (JNIEnv* env, jobject self) { 73 | void* ptr = NULL; 74 | 75 | if (!get_ptr(env, self, &ptr)) { 76 | smatrix_close(ptr); 77 | set_ptr(env, self, NULL); 78 | } 79 | } 80 | 81 | JNIEXPORT jint _JM(get) (JNIEnv* env, jobject self, jint x, jint y) { 82 | void* ptr = NULL; 83 | 84 | if (get_ptr(env, self, &ptr)) { 85 | return 0; 86 | } else { 87 | return (jint) smatrix_get(ptr, x, y); 88 | } 89 | } 90 | 91 | JNIEXPORT void JNICALL _JM(set) (JNIEnv* env, jobject self, jint x, jint y, jint v) { 92 | void* ptr = NULL; 93 | 94 | if (!get_ptr(env, self, &ptr)) { 95 | smatrix_set(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v); 96 | } 97 | } 98 | 99 | JNIEXPORT void JNICALL _JM(incr) (JNIEnv* env, jobject self, jint x, jint y, jint v) { 100 | void* ptr = NULL; 101 | 102 | if (!get_ptr(env, self, &ptr)) { 103 | smatrix_incr(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v); 104 | } 105 | } 106 | 107 | JNIEXPORT void JNICALL _JM(decr) (JNIEnv* env, jobject self, jint x, jint y, jint v) { 108 | void* ptr = NULL; 109 | 110 | if (!get_ptr(env, self, &ptr)) { 111 | smatrix_decr(ptr, (uint32_t) x, (uint32_t) y, (uint32_t) v); 112 | } 113 | } 114 | 115 | JNIEXPORT void JNICALL _JM(getRowNative) (JNIEnv* env, jobject self, jint x, jobject map, jint maxlen) { 116 | jclass cls; 117 | jint i, len; 118 | jmethodID mid; 119 | uint32_t *data; 120 | void* ptr = NULL; 121 | size_t bytes; 122 | 123 | cls = (*env)->GetObjectClass(env, map); 124 | mid = (*env)->GetMethodID(env, cls, "putIntTuple", "(II)V"); 125 | 126 | if (mid == NULL || get_ptr(env, self, &ptr)) { 127 | return; 128 | } 129 | 130 | len = smatrix_rowlen(ptr, x); 131 | bytes = len * 8; 132 | data = malloc(bytes); 133 | 134 | if (data == NULL) { 135 | throw_exception(env, "malloc() failed"); 136 | return; 137 | } 138 | 139 | len = smatrix_getrow(ptr, (uint32_t) x, data, bytes); 140 | 141 | for (i = 0; i < len; i++) { 142 | if (maxlen > 0 && i >= maxlen) { 143 | break; 144 | } 145 | 146 | (*env)->CallVoidMethod(env, map, mid, data[i * 2], data[i * 2 + 1]); 147 | } 148 | 149 | free(data); 150 | } 151 | 152 | JNIEXPORT jint JNICALL _JM(getRowLength) (JNIEnv* env, jobject self, jint x) { 153 | void* ptr = NULL; 154 | 155 | if (get_ptr(env, self, &ptr)) { 156 | return 0; 157 | } else { 158 | return (jint) smatrix_rowlen(ptr, x); 159 | } 160 | } 161 | 162 | -------------------------------------------------------------------------------- /src/smatrix_jni.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class com_paulasmuth_libsmatrix_SparseMatrix */ 4 | 5 | #ifndef _Included_com_paulasmuth_libsmatrix_SparseMatrix 6 | #define _Included_com_paulasmuth_libsmatrix_SparseMatrix 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | /* 11 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 12 | * Method: get 13 | * Signature: (II)I 14 | */ 15 | JNIEXPORT jint JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_get 16 | (JNIEnv *, jobject, jint, jint); 17 | 18 | /* 19 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 20 | * Method: set 21 | * Signature: (III)V 22 | */ 23 | JNIEXPORT void JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_set 24 | (JNIEnv *, jobject, jint, jint, jint); 25 | 26 | /* 27 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 28 | * Method: incr 29 | * Signature: (III)V 30 | */ 31 | JNIEXPORT void JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_incr 32 | (JNIEnv *, jobject, jint, jint, jint); 33 | 34 | /* 35 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 36 | * Method: decr 37 | * Signature: (III)V 38 | */ 39 | JNIEXPORT void JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_decr 40 | (JNIEnv *, jobject, jint, jint, jint); 41 | 42 | /* 43 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 44 | * Method: getRow 45 | * Signature: (I)Ljava/util/SortedMap; 46 | */ 47 | JNIEXPORT jobject JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_getRow 48 | (JNIEnv *, jobject, jint); 49 | 50 | /* 51 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 52 | * Method: getRowLength 53 | * Signature: (I)I 54 | */ 55 | JNIEXPORT jint JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_getRowLength 56 | (JNIEnv *, jobject, jint); 57 | 58 | /* 59 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 60 | * Method: close 61 | * Signature: ()V 62 | */ 63 | JNIEXPORT void JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_close 64 | (JNIEnv *, jobject); 65 | 66 | /* 67 | * Class: com_paulasmuth_libsmatrix_SparseMatrix 68 | * Method: init 69 | * Signature: (Ljava/lang/String;)V 70 | */ 71 | JNIEXPORT void JNICALL Java_com_paulasmuth_libsmatrix_SparseMatrix_init 72 | (JNIEnv *, jobject, jstring); 73 | 74 | #ifdef __cplusplus 75 | } 76 | #endif 77 | #endif 78 | -------------------------------------------------------------------------------- /src/smatrix_private.h: -------------------------------------------------------------------------------- 1 | // This file is part of the "libsmatrix" project 2 | // (c) 2011-2013 Paul Asmuth 3 | // 4 | // Licensed under the MIT License (the "License"); you may not use this 5 | // file except in compliance with the License. You may obtain a copy of 6 | // the License at: http://opensource.org/licenses/MIT 7 | 8 | #ifndef SMATRIX_PRIVATE_H 9 | #define SMATRIX_PRIVATE_H 10 | 11 | void smatrix_fcreate(smatrix_t* self); 12 | void smatrix_fload(smatrix_t* self); 13 | void smatrix_lookup(smatrix_t* self, smatrix_ref_t* ref, uint32_t x, uint32_t y, int write); 14 | void smatrix_decref(smatrix_t* self, smatrix_ref_t* ref); 15 | void* smatrix_malloc(smatrix_t* self, uint64_t bytes); 16 | void smatrix_mfree(smatrix_t* self, uint64_t bytes); 17 | uint64_t smatrix_falloc(smatrix_t* self, uint64_t bytes); 18 | void smatrix_ffree(smatrix_t* self, uint64_t fpos, uint64_t bytes); 19 | void smatrix_write(smatrix_t* self, uint64_t fpos, char* data, uint64_t bytes); 20 | void smatrix_rmap_init(smatrix_t* self, smatrix_rmap_t* rmap, uint32_t size); 21 | smatrix_rmap_slot_t* smatrix_rmap_probe(smatrix_rmap_t* rmap, uint32_t key); 22 | smatrix_rmap_slot_t* smatrix_rmap_insert(smatrix_t* self, smatrix_rmap_t* rmap, uint32_t key); 23 | void smatrix_rmap_resize(smatrix_t* self, smatrix_rmap_t* rmap); 24 | void smatrix_rmap_load(smatrix_t* self, smatrix_rmap_t* rmap); 25 | void smatrix_rmap_write_batch(smatrix_t* self, smatrix_rmap_t* rmap, int full); 26 | void smatrix_rmap_write_slot(smatrix_t* self, smatrix_rmap_t* rmap, smatrix_rmap_slot_t* slot); 27 | void smatrix_rmap_swap(smatrix_t* self, smatrix_rmap_t* rmap); 28 | void smatrix_rmap_free(smatrix_t* self, smatrix_rmap_t* rmap); 29 | void smatrix_rmap_sync_defer(smatrix_t* self, smatrix_rmap_t* rmap); 30 | void smatrix_rmap_sync(smatrix_t* self, smatrix_rmap_t* rmap); 31 | void smatrix_cmap_init(smatrix_t* self); 32 | smatrix_rmap_t* smatrix_cmap_lookup(smatrix_t* self, smatrix_cmap_t* cmap, uint32_t key, int create); 33 | smatrix_cmap_slot_t* smatrix_cmap_probe(smatrix_cmap_t* cmap, uint32_t key); 34 | smatrix_cmap_slot_t* smatrix_cmap_insert(smatrix_t* self, smatrix_cmap_t* cmap, uint32_t key); 35 | void smatrix_cmap_resize(smatrix_t* self, smatrix_cmap_t* cmap); 36 | void smatrix_cmap_free(smatrix_t* self, smatrix_cmap_t* cmap); 37 | uint64_t smatrix_cmap_falloc(smatrix_t* self, smatrix_cmap_t* cmap); 38 | void smatrix_cmap_mkblock(smatrix_t* self, smatrix_cmap_t* cmap); 39 | void smatrix_cmap_write(smatrix_t* self, smatrix_rmap_t* rmap); 40 | void smatrix_cmap_load(smatrix_t* self, uint64_t head_fpos); 41 | void smatrix_lock_getmutex(smatrix_lock_t* lock); 42 | void smatrix_lock_dropmutex(smatrix_lock_t* lock); 43 | void smatrix_lock_release(smatrix_lock_t* lock); 44 | void smatrix_lock_incref(smatrix_lock_t* lock); 45 | void smatrix_lock_decref(smatrix_lock_t* lock); 46 | void smatrix_error(const char* msg); 47 | void smatrix_ioqueue_add(smatrix_t* self, smatrix_rmap_t* rmap); 48 | smatrix_rmap_t* smatrix_ioqueue_pop(smatrix_t* self); 49 | void* smatrix_io(void* self); 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/smatrix_ruby.c: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | #include 10 | #include 11 | #include 12 | #include "smatrix.h" 13 | #include "smatrix_ruby.h" 14 | 15 | void smatrix_rb_gethandle(VALUE self, smatrix_t** handle) { 16 | VALUE handle_wrapped = rb_iv_get(self, "@handle"); 17 | 18 | if (rb_type(handle_wrapped) != RUBY_T_DATA) { 19 | rb_raise(rb_eTypeError, "smatrix @handle is of the wrong type, something went horribly wrong :("); 20 | return; 21 | } 22 | 23 | Data_Get_Struct(handle_wrapped, smatrix_t, *handle); 24 | } 25 | 26 | VALUE smatrix_rb_initialize(VALUE self, VALUE filename) { 27 | smatrix_t* smatrix = NULL; 28 | VALUE smatrix_handle; 29 | 30 | switch (rb_type(filename)) { 31 | 32 | case RUBY_T_STRING: 33 | smatrix = smatrix_open(RSTRING_PTR(filename)); 34 | break; 35 | 36 | case RUBY_T_NIL: 37 | smatrix = smatrix_open(NULL); 38 | break; 39 | 40 | default: 41 | rb_raise(rb_eTypeError, "first argument (filename) must be nil or a string"); 42 | break; 43 | 44 | } 45 | 46 | if (smatrix) { 47 | smatrix_handle = Data_Wrap_Struct(rb_cObject, NULL, smatrix_rb_free, smatrix); 48 | rb_iv_set(self, "@handle", smatrix_handle); 49 | } 50 | 51 | return self; 52 | } 53 | 54 | VALUE smatrix_rb_get(VALUE self, VALUE x, VALUE y) { 55 | smatrix_t* smatrix = NULL; 56 | smatrix_rb_gethandle(self, &smatrix); 57 | 58 | if (!smatrix) { 59 | rb_raise(rb_eTypeError, "smatrix @handle is Nil, something went horribly wrong :("); 60 | return Qnil; 61 | } 62 | 63 | if (rb_type(x) != RUBY_T_FIXNUM) { 64 | rb_raise(rb_eTypeError, "first argument (x) must be a Fixnum"); 65 | return Qnil; 66 | } 67 | 68 | if (rb_type(y) != RUBY_T_FIXNUM) { 69 | rb_raise(rb_eTypeError, "second argument (y) must be a Fixnum"); 70 | return Qnil; 71 | } 72 | 73 | return INT2NUM(smatrix_get(smatrix, NUM2INT(x), NUM2INT(y))); 74 | } 75 | 76 | VALUE smatrix_rb_set(VALUE self, VALUE x, VALUE y, VALUE value) { 77 | smatrix_t* smatrix = NULL; 78 | smatrix_rb_gethandle(self, &smatrix); 79 | 80 | if (!smatrix) { 81 | rb_raise(rb_eTypeError, "smatrix @handle is Nil, something is very bad :'("); 82 | return Qnil; 83 | } 84 | 85 | if (rb_type(x) != RUBY_T_FIXNUM) { 86 | rb_raise(rb_eTypeError, "first argument (x) must be a Fixnum"); 87 | return Qnil; 88 | } 89 | 90 | if (rb_type(y) != RUBY_T_FIXNUM) { 91 | rb_raise(rb_eTypeError, "second argument (y) must be a Fixnum"); 92 | return Qnil; 93 | } 94 | 95 | if (rb_type(value) != RUBY_T_FIXNUM) { 96 | rb_raise(rb_eTypeError, "third argument must be a Fixnum"); 97 | return Qnil; 98 | } 99 | 100 | return INT2NUM(smatrix_set(smatrix, NUM2INT(x), NUM2INT(y), NUM2INT(value))); 101 | } 102 | 103 | VALUE smatrix_rb_incr(VALUE self, VALUE x, VALUE y, VALUE value) { 104 | smatrix_t* smatrix = NULL; 105 | smatrix_rb_gethandle(self, &smatrix); 106 | 107 | if (!smatrix) { 108 | rb_raise(rb_eTypeError, "smatrix @handle is Nil, something is very bad :'("); 109 | return Qnil; 110 | } 111 | 112 | if (rb_type(x) != RUBY_T_FIXNUM) { 113 | rb_raise(rb_eTypeError, "first argument (x) must be a Fixnum"); 114 | return Qnil; 115 | } 116 | 117 | if (rb_type(y) != RUBY_T_FIXNUM) { 118 | rb_raise(rb_eTypeError, "second argument (y) must be a Fixnum"); 119 | return Qnil; 120 | } 121 | 122 | if (rb_type(value) != RUBY_T_FIXNUM) { 123 | rb_raise(rb_eTypeError, "third argument must be a Fixnum"); 124 | return Qnil; 125 | } 126 | 127 | return INT2NUM(smatrix_incr(smatrix, NUM2INT(x), NUM2INT(y), NUM2INT(value))); 128 | } 129 | 130 | VALUE smatrix_rb_decr(VALUE self, VALUE x, VALUE y, VALUE value) { 131 | smatrix_t* smatrix = NULL; 132 | smatrix_rb_gethandle(self, &smatrix); 133 | 134 | if (!smatrix) { 135 | rb_raise(rb_eTypeError, "smatrix @handle is Nil, something is very bad :'("); 136 | return Qnil; 137 | } 138 | 139 | if (rb_type(x) != RUBY_T_FIXNUM) { 140 | rb_raise(rb_eTypeError, "first argument (x) must be a Fixnum"); 141 | return Qnil; 142 | } 143 | 144 | if (rb_type(y) != RUBY_T_FIXNUM) { 145 | rb_raise(rb_eTypeError, "second argument (y) must be a Fixnum"); 146 | return Qnil; 147 | } 148 | 149 | if (rb_type(value) != RUBY_T_FIXNUM) { 150 | rb_raise(rb_eTypeError, "third argument must be a Fixnum"); 151 | return Qnil; 152 | } 153 | 154 | return INT2NUM(smatrix_decr(smatrix, NUM2INT(x), NUM2INT(y), NUM2INT(value))); 155 | } 156 | 157 | void smatrix_rb_free(smatrix_t* smatrix) { 158 | if (!smatrix) { 159 | rb_raise(rb_eTypeError, "smatrix @handle is Nil, something is very bad :'("); 160 | return; 161 | } 162 | 163 | smatrix_close(smatrix); 164 | } 165 | 166 | void Init_smatrix() { 167 | VALUE klass = rb_define_class("SparseMatrix", rb_cObject); 168 | 169 | rb_define_method(klass, "initialize", smatrix_rb_initialize, 1); 170 | rb_define_method(klass, "get", smatrix_rb_get, 2); 171 | rb_define_method(klass, "set", smatrix_rb_set, 3); 172 | rb_define_method(klass, "incr", smatrix_rb_incr, 3); 173 | rb_define_method(klass, "decr", smatrix_rb_decr, 3); 174 | } 175 | 176 | void Init_smatrix_ruby() { 177 | Init_smatrix(); 178 | } 179 | -------------------------------------------------------------------------------- /src/smatrix_ruby.h: -------------------------------------------------------------------------------- 1 | /** 2 | * This file is part of the "libsmatrix" project 3 | * (c) 2011-2013 Paul Asmuth 4 | * 5 | * Licensed under the MIT License (the "License"); you may not use this 6 | * file except in compliance with the License. You may obtain a copy of 7 | * the License at: http://opensource.org/licenses/MIT 8 | */ 9 | #include 10 | #include "smatrix.h" 11 | 12 | #ifndef SMATRIX_RUBY_H 13 | #define SMATRIX_RUBY_H 14 | 15 | #ifndef RUBY_T_STRING 16 | #define RUBY_T_STRING T_STRING 17 | #endif 18 | 19 | void smatrix_rb_gethandle(VALUE self, smatrix_t** handle); 20 | VALUE smatrix_rb_get(VALUE self, VALUE x, VALUE y); 21 | VALUE smatrix_rb_initialize(VALUE self, VALUE filename); 22 | VALUE smatrix_rb_incr(VALUE self, VALUE x, VALUE y, VALUE value); 23 | VALUE smatrix_rb_decr(VALUE self, VALUE x, VALUE y, VALUE value); 24 | VALUE smatrix_rb_set(VALUE self, VALUE x, VALUE y, VALUE value); 25 | void smatrix_rb_free(smatrix_t* smatrix); 26 | void Init_smatrix_ruby(); 27 | void Init_smatrix(); 28 | 29 | #endif 30 | --------------------------------------------------------------------------------