├── .gitignore ├── README.md ├── simpledb.nimble ├── src └── simpledb.nim └── tests ├── config.nims └── tests.nim /.gitignore: -------------------------------------------------------------------------------- 1 | test.db 2 | tests/tests -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SimpleDB 2 | 3 | ![](https://img.shields.io/badge/status-beta-orange) 4 | ![](https://img.shields.io/badge/platforms-native%20only-orange) 5 | 6 | A very simple NoSQL JSON document database written on top of SQLite. 7 | 8 | ## Usage 9 | 10 | ```nim 11 | import simpledb 12 | import json 13 | 14 | # Open or create a database 15 | var db = SimpleDB.init("database.db") 16 | 17 | # Write a document 18 | db.put(%* { 19 | "id": "1234", 20 | "timestamp": 123456, 21 | "type": "example", 22 | "text": "Hello world!" 23 | }) 24 | 25 | # Get a specific document by it's ID (null if not found) 26 | var doc = db.get("1234") 27 | 28 | # Fetch a document with a query 29 | var doc = db.query().where("type", "==", "example").get() 30 | 31 | # Fetch a list of documents with a query 32 | var docs = db.query() 33 | .where("timestamp", ">=", 1000) 34 | .where("timestamp", "<=", 2000) 35 | .limit(5) 36 | .offset(2) 37 | .list() 38 | 39 | # Delete documents 40 | db.remove("1234") 41 | db.query().where("type", "==", "example").remove() 42 | 43 | # Batch modifications 44 | db.batch: 45 | db.put(%* { "name": "item1" }) 46 | db.put(%* { "name": "item2" }) 47 | db.put(%* { "name": "item3" }) 48 | 49 | # Close the database 50 | db.close() 51 | ``` 52 | 53 | See [tests.nim](tests/tests.nim) for more examples. -------------------------------------------------------------------------------- /simpledb.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "0.1.0" 4 | author = "jjv360" 5 | description = "A simple NoSQL JSON document database" 6 | license = "MIT" 7 | srcDir = "src" 8 | 9 | 10 | # Dependencies 11 | 12 | requires "nim >= 1.6.10" 13 | requires "classes >= 0.2.13" -------------------------------------------------------------------------------- /src/simpledb.nim: -------------------------------------------------------------------------------- 1 | import classes 2 | import std/db_sqlite 3 | import std/json 4 | import std/oids 5 | import std/strutils 6 | import std/sequtils 7 | 8 | 9 | ## 10 | ## Query filter info 11 | class SimpleDBFilter: 12 | var field = "" 13 | var operation = "" 14 | var value = "" 15 | var fieldIsNumber = false 16 | 17 | 18 | ## 19 | ## Query builder 20 | class SimpleDBQuery: 21 | 22 | ## Reference to the database 23 | var db: RootRef 24 | 25 | ## List of filters 26 | var filters : seq[SimpleDBFilter] 27 | 28 | ## Sort field 29 | var sortField = "" 30 | var sortAscending = true 31 | var sortIsNumber = true 32 | 33 | ## Limit 34 | var pLimit = -1 35 | 36 | ## Offset 37 | var pOffset = 0 38 | 39 | ## (chainable) Add a filter. Operation is one of: `==` `!=` `<` `<=` `>` `>=` 40 | method where(field: string, operation: string, value: string): SimpleDBQuery = 41 | 42 | # Check input 43 | if field.len == 0: raiseAssert("No field provided") 44 | if operation.len == 0: raiseAssert("No operation provided") 45 | if operation != "==" and operation != "!=" and operation != "<" and operation != "<=" and operation != ">" and operation != ">=": raiseAssert("Unknown operation '" & operation & "'") 46 | 47 | # Add it 48 | let filter = SimpleDBFilter(field: field, operation: operation, value: value, fieldIsNumber: false) 49 | this.filters.add(filter) 50 | return this 51 | 52 | 53 | ## (chainable) Add a filter. Operation is one of: `==` `!=` `<` `<=` `>` `>=` 54 | method where(field: string, operation: string, value: float): SimpleDBQuery = 55 | 56 | # Check input 57 | if field.len == 0: raiseAssert("No field provided") 58 | if operation.len == 0: raiseAssert("No operation provided") 59 | if operation != "==" and operation != "!=" and operation != "<" and operation != "<=" and operation != ">" and operation != ">=": raiseAssert("Unknown operation '" & operation & "'") 60 | 61 | # Add it 62 | let filter = SimpleDBFilter(field: field, operation: operation, value: $value, fieldIsNumber: true) 63 | this.filters.add(filter) 64 | return this 65 | 66 | 67 | ## (chainable) Set sort field 68 | method sort(field: string, ascending: bool = true, isNumber: bool = true): SimpleDBQuery = 69 | 70 | # Check input 71 | if field.len == 0: raiseAssert("No field provided") 72 | 73 | # Store it 74 | this.sortField = field 75 | this.sortAscending = ascending 76 | this.sortIsNumber = isNumber 77 | return this 78 | 79 | 80 | ## (chainable) Set the maximum number of documents to return, or -1 to return all documents. 81 | method limit(count: int): SimpleDBQuery = 82 | 83 | # Check input 84 | if count < -1: raiseAssert("Cannot use negative numbers for the limit") 85 | 86 | # Store it 87 | this.pLimit = count 88 | return this 89 | 90 | 91 | ## (chainable) Set the number of documents to skip 92 | method offset(count: int): SimpleDBQuery = 93 | 94 | # Check input 95 | if count < 0: raiseAssert("Cannot use negative numbers for the offset") 96 | 97 | # Store it 98 | this.pOffset = count 99 | return this 100 | 101 | 102 | 103 | 104 | ## 105 | ## A simple NoSQL database written in Nim. 106 | class SimpleDB: 107 | 108 | ## (private) Database connection 109 | var conn : DbConn 110 | 111 | ## (private) True if the database has been prepared yet 112 | var hasPrepared = false 113 | 114 | ## (private) Extra columns that have been created for indexing 115 | var extraColumns: seq[string] = @["id_TEXT"] 116 | 117 | ## (private) List of hashes of generated indexes 118 | var createdIndexHashes: seq[string] 119 | 120 | ## Constructor 121 | method init(filename: string) = 122 | 123 | # Create the database connection 124 | this.conn = open(filename, "", "", "") 125 | 126 | 127 | ## Close the database 128 | method close() = 129 | 130 | # Close database 131 | if this.conn != nil: 132 | this.conn.close() 133 | this.conn = nil 134 | 135 | 136 | ## (private) Prepare the datatabase for use 137 | method prepareDB() = 138 | 139 | # Only do once 140 | if this.hasPrepared: return 141 | this.hasPrepared = true 142 | 143 | # Create main table if it doesn't exist 144 | this.conn.exec(sql"CREATE TABLE IF NOT EXISTS documents (id_TEXT TEXT PRIMARY KEY, _json TEXT)") 145 | 146 | # Get list of all columns in the table 147 | for row in this.conn.rows(sql"PRAGMA table_info(documents)"): 148 | 149 | # Add to the extra columns array 150 | let columnName = row[1] 151 | if columnName == "_json": continue 152 | if not this.extraColumns.contains(columnName): 153 | this.extraColumns.add(columnName) 154 | 155 | 156 | ## Execute a batch of transactions. Either they all succeed, or the database will not be updated. This is also much faster when saving lots of documents at once. 157 | method batch(code: proc()) = 158 | 159 | # Prepate database 160 | this.prepareDB() 161 | 162 | # Start a transaction 163 | this.conn.exec sql"BEGIN TRANSACTION" 164 | 165 | # Catch errors 166 | try: 167 | 168 | # Execute the caller's code 169 | code() 170 | 171 | except: 172 | 173 | # Rollback the transaction 174 | this.conn.exec sql"ROLLBACK TRANSACTION" 175 | 176 | # Pass the error on to the caller 177 | raise getCurrentException() 178 | 179 | # Complete the transaction 180 | this.conn.exec sql"COMMIT TRANSACTION" 181 | 182 | 183 | ## Start a query 184 | method query(): SimpleDBQuery = 185 | 186 | # Prepare database 187 | this.prepareDB() 188 | 189 | # Create query object 190 | let q = SimpleDBQuery.init() 191 | q.db = this 192 | return q 193 | 194 | 195 | ## (private) Ensure column exists for the specified field 196 | method createIndexableColumnForField(name: string, sqlName: string, sqlType: string) = 197 | 198 | # Stop if already created 199 | if this.extraColumns.contains(sqlName): 200 | return 201 | 202 | # Begin an update transaction 203 | this.batch: 204 | 205 | # Create new field on the table 206 | let str = "ALTER TABLE documents ADD \"" & sqlName & "\" " & sqlType 207 | this.conn.exec(sql(str)) 208 | 209 | # Fetch all existing documents ... this is heavy, but we can't iterate and modify at the same time 210 | let sqlUpdateRow = sql("UPDATE documents SET \"" & sqlName & "\" = ? WHERE id_TEXT = ?") 211 | for row in this.conn.getAllRows(sql"SELECT id_TEXT, _json FROM documents"): 212 | 213 | # Parse this document 214 | let id = row[0] 215 | let json = parseJson(row[1]) 216 | 217 | # Get field value 218 | let node = json{name} 219 | var value = "" 220 | if node.isNil: value = "" 221 | elif node.kind == JString: value = node.getStr() 222 | elif node.kind == JFloat: value = $node.getFloat() 223 | elif node.kind == JInt: value = $node.getInt() 224 | 225 | # Set row value 226 | if value.len > 0: 227 | this.conn.exec(sqlUpdateRow, value, id) 228 | 229 | # Done, update extra columns 230 | this.extraColumns.add(sqlName) 231 | 232 | 233 | ## (private) Create an index for the specified query, if needed 234 | method createIndex(query: SimpleDBQuery) = 235 | 236 | # Stop if no index is needed, ie this query returns all data directly 237 | if query.sortField == "" and query.filters.len == 0: 238 | return 239 | 240 | # Check if index created 241 | let indexHash = query.filters.mapIt(it.field).join("_") & query.sortField 242 | if this.createdIndexHashes.contains(indexHash): 243 | return 244 | 245 | # Create SQL 246 | var sqlStr = "CREATE INDEX IF NOT EXISTS \"documents_" & indexHash & "\" ON documents (" 247 | 248 | # Add filter fields 249 | var addedFirst = false 250 | for filter in query.filters: 251 | 252 | # Get SQL column info 253 | var sqlType = if filter.fieldIsNumber: "REAL" else: "TEXT" 254 | var sqlName = filter.field & "_" & sqlType 255 | 256 | # Add the separator if this is not the first filter 257 | if addedFirst: sqlStr &= ", " 258 | addedFirst = true 259 | 260 | # Add the filter 261 | sqlStr &= "\"" & sqlName & "\"" 262 | 263 | # Add sort field 264 | if query.sortField.len > 0: 265 | 266 | # Get SQL column info 267 | var sqlType = if query.sortIsNumber: "REAL" else: "TEXT" 268 | var sqlName = query.sortField & "_" & sqlType 269 | 270 | # Add the separator if this is not the first filter 271 | if addedFirst: sqlStr &= ", " 272 | addedFirst = true 273 | 274 | # Add the filter 275 | sqlStr &= "\"" & sqlName & "\"" 276 | 277 | # Close the SQL 278 | sqlStr &= ")" 279 | 280 | # Execute it 281 | this.conn.exec(sql(sqlStr)) 282 | 283 | # Done, store index hash 284 | this.createdIndexHashes.add(indexHash) 285 | 286 | 287 | 288 | ## Execute the query and return all documents. 289 | proc prepareQuerySql(this: SimpleDBQuery, sqlPrefix: string): (string, seq[string]) = 290 | 291 | # Get database reference 292 | let db = cast[SimpleDB](this.db) 293 | 294 | # Build query 295 | var bindValues : seq[string] 296 | var sqlStr = sqlPrefix 297 | 298 | # Add filters 299 | if this.filters.len > 0: 300 | 301 | # Add WHERE clause 302 | sqlStr &= " WHERE " 303 | var addedFirst = false 304 | for filter in this.filters: 305 | 306 | # Get SQL column info 307 | var sqlType = if filter.fieldIsNumber: "REAL" else: "TEXT" 308 | var sqlName = filter.field & "_" & sqlType 309 | 310 | # Ensure an indexable column exists for this field 311 | db.createIndexableColumnForField(filter.field, sqlName, sqlType) 312 | 313 | # Add the 'AND' if this is not the first filter 314 | if addedFirst: sqlStr &= " AND " 315 | addedFirst = true 316 | 317 | # Add the filter 318 | sqlStr &= "\"" & sqlName & "\" " & filter.operation & " ?" 319 | bindValues.add(filter.value) 320 | 321 | # Add sort 322 | if this.sortField.len > 0: 323 | 324 | # Get SQL column info 325 | var sqlType = if this.sortIsNumber: "REAL" else: "TEXT" 326 | var sqlName = this.sortField & "_" & sqlType 327 | 328 | # Ensure an indexable column exists for this field 329 | db.createIndexableColumnForField(this.sortField, sqlName, sqlType) 330 | 331 | # Add the sort 332 | sqlStr &= " ORDER BY \"" & sqlName & "\" " & (if this.sortAscending: "asc" else: "desc") 333 | 334 | # Add limit 335 | if this.pLimit >= 0: 336 | sqlStr &= " LIMIT " & $this.pLimit 337 | 338 | # Add offset 339 | if this.pOffset > 0: 340 | sqlStr &= " OFFSET " & $this.pOffset 341 | 342 | # Create index for this query if needed 343 | db.createIndex(this) 344 | 345 | # Done, prepare and bind the query 346 | return (sqlStr, bindValues) 347 | 348 | 349 | ## Execute the query and return all documents. 350 | proc list*(this: SimpleDBQuery): seq[JsonNode] = 351 | 352 | # Get database reference 353 | let db = cast[SimpleDB](this.db) 354 | 355 | # Prepare the query 356 | let (sqlStr, bindValues) = prepareQuerySql(this, "SELECT _json FROM documents") 357 | 358 | # Run the query 359 | var docs : seq[JsonNode] 360 | for row in db.conn.rows(sql(sqlStr), bindValues): 361 | 362 | # Parse JSON for each result 363 | docs.add(parseJson(row[0])) 364 | 365 | # Done 366 | return docs 367 | 368 | 369 | ## Execute the query and iterate through the resulting documents. 370 | iterator list*(this: SimpleDBQuery): JsonNode = 371 | 372 | # Get database reference 373 | let db = cast[SimpleDB](this.db) 374 | 375 | # Prepare the query 376 | let (sqlStr, bindValues) = prepareQuerySql(this, "SELECT _json FROM documents") 377 | 378 | # Run the query 379 | for row in db.conn.rows(sql(sqlStr), bindValues): 380 | 381 | # Parse JSON for each result and yield it 382 | yield parseJson(row[0]) 383 | 384 | 385 | ## Remove the documents matched by this query. 386 | proc remove*(this: SimpleDBQuery): int {.discardable.} = 387 | 388 | # Get database reference 389 | let db = cast[SimpleDB](this.db) 390 | 391 | # Prepare the query 392 | let (sqlStr, bindValues) = prepareQuerySql(this, "DELETE FROM documents") 393 | 394 | # Run the query 395 | return int db.conn.execAffectedRows(sql(sqlStr), bindValues) 396 | 397 | 398 | ## Execute the query and return the first document found, or null if not found. 399 | proc get*(this: SimpleDBQuery): JsonNode = 400 | 401 | # Limit to one 402 | this.pLimit = 1 403 | 404 | # Execute query 405 | let docs = this.list() 406 | if docs.len == 0: 407 | return nil 408 | else: 409 | return docs[0] 410 | 411 | 412 | ## Helper: Get a document with the specified ID, or return nil if not found 413 | proc get*(this: SimpleDB, id: string): JsonNode = 414 | return this.query().where("id", "==", id).get() 415 | 416 | 417 | ## Helper: Remove a document with the specified ID. Returns true if the document was removed, or false if no document was found with this ID. 418 | proc remove*(this: SimpleDB, id: string): bool {.discardable.} = 419 | let numRemoved = this.query().where("id", "==", id).limit(1).remove() 420 | return if numRemoved > 0: true else: false 421 | 422 | ## Put a new document into the database, or replace it if it already exists 423 | proc writeDocument(this: SimpleDB, document: JsonNode) = 424 | 425 | # Check input 426 | if document == nil: raiseAssert("Cannot put a null document into the database.") 427 | if document.kind != JObject: raiseAssert("Document must be an object.") 428 | if document{"id"}.isNil: document["id"] = % $genOid() 429 | if document{"id"}.kind != JString: raiseAssert("ID must be a string.") 430 | 431 | # Prepare database 432 | this.prepareDB() 433 | 434 | # Create query including all fields 435 | let str = "INSERT OR REPLACE INTO documents (_json, " & this.extraColumns.join(", ") & ") VALUES (?, " & this.extraColumns.mapIt("?").join(", ") & ")" 436 | let cmd = sql(str) 437 | 438 | # First field is the JSON content 439 | var args = @[ $document ] 440 | 441 | # Add fields for the extra columns 442 | for columnName in this.extraColumns: 443 | 444 | # Get field name by removing the sql suffix 445 | let fieldName = columnName.substr(0, columnName.len - 6) 446 | 447 | # Add it 448 | args.add document{fieldName}.getStr() 449 | 450 | # Bind and execute the query 451 | this.conn.exec(cmd, args) 452 | 453 | 454 | ## Put a new document into the database, merging the fields if it already exists 455 | proc put*(this: SimpleDB, document: JsonNode, merge: bool = false) = 456 | 457 | # If not merging, just write it 458 | if not merge: 459 | this.writeDocument(document) 460 | 461 | # Check input 462 | if document == nil: raiseAssert("Cannot put a null document into the database.") 463 | if document.kind != JObject: raiseAssert("Document must be an object.") 464 | if document{"id"}.isNil: 465 | this.writeDocument(document) 466 | return 467 | if document{"id"}.kind != JString: raiseAssert("ID must be a string.") 468 | 469 | # Get existing document, or just save it normally if not found 470 | let id = document["id"].getStr() 471 | var existingDoc = this.get(id) 472 | if existingDoc == nil: 473 | this.writeDocument(document) 474 | return 475 | 476 | # Merge new fields 477 | for key, value in document.pairs: 478 | existingDoc[key] = value 479 | 480 | # Write it 481 | this.writeDocument(existingDoc) -------------------------------------------------------------------------------- /tests/config.nims: -------------------------------------------------------------------------------- 1 | switch("path", "$projectDir/../src") -------------------------------------------------------------------------------- /tests/tests.nim: -------------------------------------------------------------------------------- 1 | # This is just an example to get you started. You may wish to put all of your 2 | # tests into a single file, or separate them into multiple `test1`, `test2` 3 | # etc. files (better names are recommended, just make sure the name starts with 4 | # the letter 't'). 5 | # 6 | # To run these tests, simply execute `nimble test`. 7 | 8 | import simpledb 9 | import std/times 10 | import std/json 11 | import std/random 12 | import std/os 13 | import std/terminal 14 | 15 | 16 | 17 | # Helpers for testing 18 | proc group(str: string) = styledEcho "\n", fgBlue, "+ ", fgDefault, str 19 | proc test(str: string) = styledEcho fgGreen, " + ", fgDefault, str 20 | proc warn(str: string) = styledEcho fgRed, " ! ", fgDefault, str 21 | 22 | 23 | 24 | 25 | # Remove existing database if it exists 26 | group "Cleanup" 27 | test "Remove existing database" 28 | if fileExists("test.db"): 29 | removeFile("test.db") 30 | 31 | 32 | 33 | 34 | 35 | # Open the database 36 | group "Database tests" 37 | test "Open database" 38 | var db = SimpleDB.init("test.db") 39 | 40 | 41 | 42 | 43 | 44 | # Add a document 45 | test "Add a document" 46 | db.put(%* { 47 | "id": "1234", 48 | "type": "replaced", 49 | }) 50 | 51 | 52 | 53 | 54 | 55 | # Update a document 56 | test "Replace a document" 57 | db.put(%* { 58 | "id": "1234", 59 | "type": "example", 60 | "data": "123456", 61 | "timestamp": cpuTime(), 62 | "isExample": true, 63 | "otherInfo": nil 64 | }) 65 | 66 | 67 | 68 | 69 | 70 | # Merge update a document 71 | test "Update a document" 72 | db.put(%* { 73 | "id": "1234", 74 | "otherInfo": "test" 75 | }, merge = true) 76 | 77 | # Test it 78 | let exampleDoc = db.get("1234") 79 | if exampleDoc{"type"}.getStr() != "example": raiseAssert("Wrong document returned.") 80 | if exampleDoc{"otherInfo"}.getStr() != "test": raiseAssert("Merged content was not saved.") 81 | if exampleDoc{"data"}.getStr() != "123456": raiseAssert("Content was not merged correctly.") 82 | 83 | 84 | 85 | 86 | 87 | # Batch add documents 88 | test "Batch updates" 89 | randomize() 90 | let batchedCount = 1000 91 | db.batch: 92 | for i in 0 .. batchedCount: 93 | db.put(%* { "type": "batched", "index": i, "random": rand(1.0) }) 94 | 95 | 96 | 97 | 98 | 99 | # Close and reopen the database 100 | test "Close and reopen database" 101 | db.close() 102 | db = SimpleDB.init("test.db") 103 | 104 | 105 | 106 | 107 | 108 | # Fetch a specific document 109 | test "Fetch a document by ID" 110 | let doc = db.get("1234") 111 | 112 | # Test results 113 | if doc == nil: raiseAssert("Unable to read document.") 114 | if doc{"type"}.getStr() != "example": raiseAssert("Invalid data") 115 | 116 | 117 | 118 | 119 | 120 | # Do a complex query 121 | test "Complex queries" 122 | let docs = db.query() 123 | .where("type", "==", "batched") 124 | .where("index", ">=", 100) 125 | .where("index", "<", 120) 126 | .sort("index", ascending = false) 127 | .offset(5) 128 | .limit(2) 129 | .list() 130 | 131 | # Test results 132 | if docs.len != 2: raiseAssert("Wrong number of documents returned") 133 | if docs[0]["index"].getInt() != 114: raiseAssert("Wrong document returned") 134 | if docs[1]["index"].getInt() != 113: raiseAssert("Wrong document returned") 135 | 136 | 137 | 138 | 139 | 140 | # Iterator test 141 | test "Iterator" 142 | var count = 0 143 | for doc in db.query().where("type", "==", "batched").list(): 144 | count += 1 145 | if doc{"type"}.getStr() != "batched": raiseAssert("Wrong document returned") 146 | if count > 5: break 147 | 148 | 149 | 150 | 151 | 152 | # Delete a single item 153 | test "Delete a single document" 154 | db.remove("1234") 155 | 156 | 157 | 158 | 159 | 160 | # Delete multiple items 161 | test "Delete multiple documents" 162 | let removedCount = db.query() 163 | .where("type", "==", "batched") 164 | .where("index", ">", 100) 165 | .remove() 166 | 167 | # Test results 168 | if removedCount != batchedCount - 100: raiseAssert("Different number of documents were removed than expected. expected=" & $(batchedCount - 100) & " removed=" & $removedCount) --------------------------------------------------------------------------------