├── .gitignore ├── duckdb.nimble ├── .github └── workflows │ └── install_and_test.yml ├── CHANGELOG.md ├── README.md ├── tests └── test_duckdb.nim ├── src ├── duckdb │ └── duckdb_wrapper.nim └── duckdb.nim └── LICENSE.txt /.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !**/ 3 | !*.* 4 | 5 | nimcache/ 6 | build/ 7 | docs/build/ 8 | .cache 9 | .DS_Store 10 | .vscode 11 | 12 | # Package specific 13 | *.duckdb 14 | *.duckdb.wal -------------------------------------------------------------------------------- /duckdb.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "0.4.0" 4 | author = "ayman albaz" 5 | description = "A DuckDB wrapper written in Nim." 6 | license = "Apache-2.0" 7 | srcDir = "src" 8 | 9 | 10 | # Dependencies 11 | 12 | requires "nim >= 2.0.0" 13 | requires "nimterop >= 0.6.13" -------------------------------------------------------------------------------- /.github/workflows/install_and_test.yml: -------------------------------------------------------------------------------- 1 | name: InstallAndTest 2 | 3 | on: [push] 4 | 5 | jobs: 6 | tests-linux: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: jiro4989/setup-nim-action@v1 11 | with: 12 | nim-version: "2.0.0" 13 | - run: nimble install -y 14 | - run: nimble test -y 15 | tests-windows: 16 | runs-on: windows-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | - uses: jiro4989/setup-nim-action@v1 20 | with: 21 | nim-version: "2.0.0" 22 | - run: nimble install -y 23 | - run: nimble test -y 24 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | 10 | ## [0.4.0] - 2024-03-03 11 | ### Added 12 | - Upgraded duckdb library from v0.5.0 to v0.10.0 13 | 14 | ## [0.3.1] - 2022-11-16 15 | ### Added 16 | - Upgraded duckdb library from v0.5.0 to v0.5.1 17 | 18 | ## [0.3.0] - 2022-09-11 19 | ### Added 20 | - Changed library API to match sqlite in stdlib 21 | - Library now more memory safe due to use of destructors 22 | 23 | ## [0.2.0] - 2022-09-10 24 | ### Added 25 | - Added better error messaging 26 | - Removed config options 27 | 28 | 29 | ## [0.1.0] - 2021-11-28 30 | ### Added 31 | - Created nim-duckdb library 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Linux Build Status (Github Actions)](https://github.com/ayman-albaz/nim-duckdb/actions/workflows/install_and_test.yml/badge.svg) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 2 | 3 | # nim-duckdb 4 | 5 | This library is a DuckDB wrapper in nim. It uses nimterop to generate the C Bindings. 6 | 7 | 8 | ## Supported Functions 9 | 10 | Opening/closing a database: 11 | ```Nim 12 | import duckdb 13 | 14 | # Open database and connection. 15 | # Connection and database closed automatically with destructors 16 | var dbConn = connect("mydb.db") 17 | ``` 18 | 19 | Opening/closing an in-memory database: 20 | ```Nim 21 | import duckdb 22 | 23 | # Open database and connection 24 | # Connection and database closed automatically with destructors 25 | var dbConn = connect() 26 | ``` 27 | 28 | Executing commands and fetching a prepared statement 29 | ```Nim 30 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 31 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 32 | var items: seq[seq[string]] 33 | for item in dbConn.rows("SELECT * FROM integers WHERE i = ? or j = ?", 3, "6"): 34 | items.add(item) 35 | assert items == @[@["3", "4"], @["5", "6"]] 36 | ``` 37 | 38 | Executing commands and fast inserting. Fast inserting is much faster than inserting in a loop. 39 | ```Nim 40 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 41 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 42 | dbConn.fastInsert( 43 | "integers", 44 | @[ 45 | @["11", "NULL"] 46 | ], 47 | ) 48 | var items: seq[seq[string]] 49 | for item in dbConn.rows( 50 | """SELECT i, j, i + j 51 | FROM integers""" 52 | ): items.add(item) 53 | 54 | check items == @[@["3", "4", "7"], @["5", "6", "11"], @["7", "NULL", "NULL"], @["11", "NULL", "NULL"]] 55 | ``` 56 | 57 | Null items are represented by `"NULL"` 58 | 59 | 60 | ## Acknowledgments 61 | Special thanks to [@Clonkk](https://github.com/Clonkk/duckdb_wrapper) as I used his nimterop script to generate the wrapper. 62 | 63 | 64 | ## Contact 65 | I can be reached at aymanalbaz98@gmail.com -------------------------------------------------------------------------------- /tests/test_duckdb.nim: -------------------------------------------------------------------------------- 1 | import unittest 2 | import ../src/duckdb 3 | 4 | 5 | suite "tests": 6 | 7 | test "DuckDB init": 8 | var dbConn = connect() 9 | 10 | test "Bad result": 11 | var dbConn = connect() 12 | expect DuckDBOperationError: 13 | dbConn.exec("CREATE BAD COMMAND!") 14 | 15 | test "DuckDB exec": 16 | var dbConn = connect() 17 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 18 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 19 | 20 | test "DuckDB exec prepared rows": 21 | var dbConn = connect() 22 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 23 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (?, NULL);", 7) 24 | var items: seq[seq[string]] 25 | for item in dbConn.rows("SELECT * FROM integers"): items.add(item) 26 | check items == @[@["3", "4"], @["5", "6"], @["7", "NULL"]] 27 | 28 | test "DuckDB exec bad prepared rows": 29 | var dbConn = connect() 30 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 31 | expect DuckDBOperationError: 32 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (?, NULL);", "NANA") 33 | 34 | test "DuckDB exec rows prepared": 35 | var dbConn = connect() 36 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 37 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 38 | var items: seq[seq[string]] 39 | for item in dbConn.rows("SELECT * FROM integers WHERE i = ? or j = ?", 3, "6"): items.add(item) 40 | check items == @[@["3", "4"], @["5", "6"]] 41 | 42 | test "DuckDB exec rows bad prepared": 43 | var dbConn = connect() 44 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 45 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 46 | var items: seq[seq[string]] 47 | expect DuckDBOperationError: 48 | for item in dbConn.rows("SELECT * FROM integers WHERE i = ? or j = ?", 3, "NANA"): items.add(item) 49 | 50 | test "DuckDB exec fast insert": 51 | var dbConn = connect() 52 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 53 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 54 | dbConn.fastInsert( 55 | "integers", 56 | @[ 57 | @["11", "NULL"] 58 | ], 59 | ) 60 | var items: seq[seq[string]] 61 | for item in dbConn.rows( 62 | """SELECT i, j, i + j 63 | FROM integers""" 64 | ): items.add(item) 65 | 66 | check items == @[@["3", "4", "7"], @["5", "6", "11"], @["7", "NULL", "NULL"], @["11", "NULL", "NULL"]] 67 | 68 | test "DuckDB exec bad fast insert": 69 | var dbConn = connect() 70 | dbConn.exec("CREATE TABLE integers(i INTEGER, j INTEGER);") 71 | dbConn.exec("INSERT INTO integers VALUES (3, 4), (5, 6), (7, NULL);") 72 | expect DuckDBOperationError: 73 | dbConn.fastInsert( 74 | "integers", 75 | @[ 76 | @["11", "NANA"] 77 | ], 78 | ) 79 | -------------------------------------------------------------------------------- /src/duckdb/duckdb_wrapper.nim: -------------------------------------------------------------------------------- 1 | import nimterop/cimport 2 | import nimterop/build 3 | import os 4 | 5 | when defined(buildDuckDb): 6 | const duckdbUrl = "https://github.com/duckdb/duckdb/releases/download/v0.10.0/libduckdb-src.zip" 7 | 8 | else: 9 | when defined(Linux): 10 | const duckdbUrl = "https://github.com/duckdb/duckdb/releases/download/v0.10.0/libduckdb-linux-amd64.zip" 11 | const duckdbLib = "libduckdb.so" 12 | when defined(macosx): 13 | const duckdbUrl = "https://github.com/duckdb/duckdb/releases/download/v0.10.0/libduckdb-osx-universal.zip" 14 | const duckdbLib = "libduckdb.dylib" 15 | when defined(Windows): 16 | const duckdbLib = "duckdb.dll" 17 | when defined(cpu64): 18 | const duckdbUrl = "https://github.com/duckdb/duckdb/releases/download/v0.10.0/libduckdb-windows-amd64.zip" 19 | when defined(cpu32): 20 | const duckdbUrl = "https://github.com/duckdb/duckdb/releases/download/v0.10.0/libduckdb-windows-i386.zip" 21 | 22 | const baseDir = getProjectCacheDir("duckdb") & "/" 23 | 24 | static: 25 | # cDebug() 26 | 27 | const duckdbZip = lastPathPart(duckdbUrl) 28 | downloadUrl(duckdbUrl, baseDir) 29 | 30 | const outdir = "." 31 | extractZip(normalizedPath(baseDir & duckdbzip), outdir) 32 | 33 | cPlugin: 34 | proc onSymbol*(sym: var Symbol) {.exportc, dynlib.} = 35 | case sym.name: 36 | of "_Bool": sym.name = "bool" 37 | of "__deprecated_data": sym.name = "deprecated_data" 38 | of "__deprecated_nullmask": sym.name = "deprecated_nullmask" 39 | of "__deprecated_type": sym.name = "deprecated_type" 40 | of "__deprecated_name": sym.name = "deprecated_name" 41 | of "__deprecated_column_count": sym.name = "deprecated_column_count" 42 | of "__deprecated_row_count": sym.name = "deprecated_row_count" 43 | of "__deprecated_rows_changed": sym.name = "deprecated_rows_changed" 44 | of "__deprecated_columns": sym.name = "deprecated_columns" 45 | of "__deprecated_error_message": sym.name = "deprecated_error_message" 46 | of "_duckdb_vector": sym.name = "duckdb_vector" 47 | of "__vctr": sym.name = "vctr" 48 | of "_duckdb_database": sym.name = "duckdb_database" 49 | of "__db": sym.name = "db" 50 | of "_duckdb_connection": sym.name = "duckdb_connection" 51 | of "__conn": sym.name = "conn" 52 | of "_duckdb_prepared_statement": sym.name = "duckdb_prepared_statement" 53 | of "__prep": sym.name = "prep" 54 | of "_duckdb_extracted_statements": sym.name = "duckdb_extracted_statements" 55 | of "__extrac": sym.name = "extrac" 56 | of "_duckdb_pending_result": sym.name = "duckdb_pending_result" 57 | of "__pend": sym.name = "pend" 58 | of "_duckdb_appender": sym.name = "duckdb_appender" 59 | of "__appn": sym.name = "appn" 60 | of "_duckdb_config": sym.name = "duckdb_config" 61 | of "_duckdb_logical_type": sym.name = "duckdb_logical_type" 62 | of "__cnfg": sym.name = "cnfg" 63 | of "__lglt": sym.name = "lglt" 64 | of "_duckdb_data_chunk": sym.name = "duckdb_data_chunk" 65 | of "__dtck": sym.name = "dtck" 66 | of "_duckdb_value": sym.name = "duckdb_value" 67 | of "__val": sym.name = "val" 68 | of "_duckdb_arrow": sym.name = "duckdb_arrow" 69 | of "__arrw": sym.name = "arrw" 70 | of "_duckdb_arrow_stream": sym.name = "duckdb_arrow_stream" 71 | of "__arrwstr": sym.name = "arrwstr" 72 | of "_duckdb_arrow_schema": sym.name = "duckdb_arrow_schema" 73 | of "__arrs": sym.name = "arrs" 74 | of "_duckdb_arrow_array": sym.name = "duckdb_arrow_array" 75 | of "__arra": sym.name = "arra" 76 | else: discard 77 | 78 | const duckDbH = normalizedPath(baseDir & "duckdb.h") 79 | 80 | when defined(buildDuckDb): 81 | const duckDbSource = normalizedPath(baseDir & "duckdb.cpp") 82 | cPassL("-lstdc++ -lpthread") 83 | cCompile(duckDbSource) 84 | cImport(duckDbH) 85 | 86 | else: 87 | const duckDbLibPath = normalizedPath(baseDir & duckdbLib) 88 | cImport(duckDbH, recurse = true, dynlib = duckDbLibPath) 89 | -------------------------------------------------------------------------------- /src/duckdb.nim: -------------------------------------------------------------------------------- 1 | import duckdb/duckdb_wrapper 2 | 3 | type 4 | DuckDBBaseError* = object of CatchableError 5 | DuckDBOperationError* = object of DuckDBBaseError 6 | DuckDBRow* = seq[string] 7 | DuckDBState* = duckdb_state 8 | # User defined objects 9 | DuckDBConn* = object 10 | database: duckdbDatabase 11 | connection: duckdbConnection 12 | DuckDBResult* = object 13 | result: duckdbResult 14 | DuckDBPreparedStatement = object 15 | statement: duckdbPreparedStatement 16 | DuckDBValueVarchar = object 17 | varchar: cstring 18 | DuckDBAppender = object 19 | appender: duckdbAppender 20 | 21 | proc close*(conn: DuckDBConn) = 22 | ## Closes a duckDB database. 23 | duckdbClose(conn.database.addr) 24 | 25 | proc disconnect*(conn: DuckDBConn) = 26 | ## Disconnects the connection to a duckDB database. 27 | duckdbDisconnect(conn.connection.addr) 28 | 29 | proc `=destroy`(conn: var DuckDBConn) = 30 | if not isNil(conn.connection.addr): 31 | conn.disconnect() 32 | if not isNil(conn.database.addr): 33 | conn.close() 34 | 35 | proc `=destroy`(result: var DuckDBResult) = 36 | if not isNil(result.result.addr): 37 | duckdbDestroyResult(result.result.addr) 38 | 39 | proc `=destroy`(statement: var DuckDBPreparedStatement) = 40 | if not isNil(statement.statement.addr): 41 | duckdbDestroyPrepare(statement.statement.addr) 42 | 43 | proc `=destroy`(varchar: var DuckDBValueVarchar) = 44 | if not isNil(varchar.varchar): 45 | duckdbFree(varchar.varchar) 46 | 47 | proc `=destroy`(appender: var DuckDBAppender) = 48 | if not isNil(appender.appender.addr): 49 | discard duckdbAppenderDestroy(appender.appender.addr) 50 | 51 | proc isStateSuccessful(state: DuckDBState): bool = 52 | result = state == DuckDBSuccess 53 | 54 | proc checkStateSuccessful(state: DuckDBState) = 55 | if not isStateSuccessful(state): 56 | raise newException(DuckDBOperationError, "DuckDB operation did not complete sucessfully.") 57 | 58 | proc checkStateSuccessful(state: DuckDBState, result: DuckDBResult) = 59 | if not isStateSuccessful(state): 60 | let errorMessage = result.result.addr.duckdbResultError() 61 | raise newException(DuckDBOperationError, 62 | "DuckDB operation did not complete sucessfully. Reason:\n" & $errorMessage) 63 | 64 | proc checkStateSuccessful(state: DuckDBState, 65 | statement: DuckDBPreparedStatement) = 66 | if not isStateSuccessful(state): 67 | let errorMessage = statement.statement.duckdbPrepareError() 68 | raise newException(DuckDBOperationError, 69 | "DuckDB operation did not complete sucessfully. Reason:\n" & $errorMessage) 70 | 71 | proc checkStateSuccessful(state: DuckDBState, appender: DuckDBAppender) = 72 | if not isStateSuccessful(state): 73 | let errorMessage = appender.appender.duckdbAppenderError() 74 | raise newException(DuckDBOperationError, 75 | "DuckDB operation did not complete sucessfully. Reason:\n" & $errorMessage) 76 | 77 | proc connect*(path: string): DuckDBConn = 78 | ## Opens a DuckDB database 79 | ## `path` is the path of the output DuckDB. Set to ":memory:" to open an in-memory database. 80 | var state1 = duckdbOpen(path.cstring, result.database.addr) 81 | checkStateSuccessful(state1) 82 | var state2 = duckdbConnect(result.database, result.connection.addr) 83 | checkStateSuccessful(state2) 84 | 85 | proc connect*(): DuckDBConn = 86 | ## Opens an in-memory DuckDB database. 87 | result = connect(":memory:") 88 | 89 | proc execWithoutArgs(conn: DuckDBConn, sqlQuery: string) = 90 | ## Executes a SQL query to a duckDB database. 91 | var result = DuckDBResult() 92 | var state = duckdbQuery(conn.connection, sqlQuery.cstring, result.result.addr) 93 | checkStateSuccessful(state, result) 94 | 95 | proc execWithArgs(conn: DuckDBConn, sqlQuery: string, args: varargs[string, `$`]) = 96 | ## Executes a SQL query to a duckDB database. 97 | var statement = DuckDBPreparedStatement() 98 | 99 | # Create prepared statement 100 | let state1 = duckdbPrepare(conn.connection, sqlQuery.cstring, 101 | statement.statement.addr) 102 | checkStateSuccessful(state1, statement) 103 | 104 | # Parse prepared statement 105 | for i, arg in args: 106 | let state2 = duckdbBindVarchar(statement.statement, (i + 1).idx_t, arg.cstring) 107 | checkStateSuccessful(state2) 108 | 109 | # Result handler 110 | let result = DuckDBResult() 111 | let state3 = duckdbExecutePrepared(statement.statement, 112 | result.result.addr) 113 | checkStateSuccessful(state3, result) 114 | 115 | proc exec*(conn: DuckDBConn, sqlQuery: string, args: varargs[string, `$`]) = 116 | if args.len() == 0: conn.execWithoutArgs(sqlQuery) 117 | else: conn.execWithArgs(sqlQuery, args) 118 | 119 | iterator getRows(result: DuckDBResult): DuckDBRow = 120 | var rowCount = result.result.addr.duckdbRowCount() 121 | var columnCount = result.result.addr.duckdbColumnCount() 122 | var duckDBRow = newSeq[string](columnCount) 123 | for idxRow in 0..