├── .gitignore
├── README.md
├── db_clickhouse.nimble
├── src
    └── db_clickhouse.nim
└── tests
    ├── test_clickhouse.nim
    └── test_clickhouse.nims


/.gitignore:
--------------------------------------------------------------------------------
1 | nimcache/
2 | tests/test_clickhouse
3 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Nim Clickhouse Interface
  2 | 
  3 | ## Introduction
  4 | 
  5 | This package let's you use the [Clickhouse](https://clickhouse.yandex/)
  6 | analytical database from the [Nim](https://nim-lang.org) language, using an
  7 | interface similar to the ones provided for SQLite and for PostgreSQL.
  8 | 
  9 | It contains only pure Nim code and doesn't require any external library.
 10 | 
 11 | Internally, it uses the HTTP interface of Clickhouse, since the TCP transport
 12 | is not meant to be used by client applications but only for cross-engine
 13 | communications, as you can see in
 14 | [this bug report](https://github.com/yandex/ClickHouse/issues/45).
 15 | 
 16 | ## Installation instructions
 17 | 
 18 | You can find this package inside the [Nim Package Directory](https://nimble.directory/pkg/dbclickhouse).
 19 | 
 20 | If you are using Nimble you can simply add the following line inside
 21 | your dependencies:
 22 | 
 23 | ```nim
 24 | requires "dbclickhouse"
 25 | ```
 26 | 
 27 | You can also set a version constraint on this package to select a certain
 28 | release.
 29 | 
 30 | ## Usage
 31 | 
 32 | ```nim
 33 | import db_clickhouse
 34 | 
 35 | var db:DbConn = db_clickhouse.open("clickhouse-server")
 36 | 
 37 | for row in db.getAllRows("SELECT * FROM test_table ORDER BY test_column"):
 38 |   echo row[0], row[1]
 39 | 
 40 | db.close()
 41 | ```
 42 | 
 43 | You can find other examples in the unit tests of this package.
 44 | 
 45 | ## Tests
 46 | 
 47 | If you want to run to unit tests included with this package, you will need to
 48 | have an available ClickHouse instance with a test table with the following
 49 | structure:
 50 | 
 51 | ```sql
 52 | CREATE TABLE test_table
 53 | (
 54 |     test_column String,
 55 |     test_column_two String
 56 | ) ENGINE Memory;
 57 | ```
 58 | 
 59 | If you are using Docker, you can create a new ClickHouse instance with the
 60 | following command:
 61 | 
 62 | ```
 63 | $ docker run -d \
 64 |     --name clickhouse-server \
 65 |     -p 9000:9000 \
 66 |     -p 8123:8123 \
 67 |     -v clickhouse:/var/lib/clickhouse \
 68 |     yandex/clickhouse-server
 69 | ```
 70 | 
 71 | You can create the required table starting a clickhouse client like this:
 72 | 
 73 | ```
 74 | $ docker run \
 75 |     -ti --rm \
 76 |     --link clickhouse-server:clickhouse-server \
 77 |     yandex/clickhouse-client \
 78 |     --host clickhouse-server
 79 | 
 80 | ClickHouse client version 1.1.54383.
 81 | Connecting to clickhouse-server:9000.
 82 | Connected to ClickHouse server version 1.1.54383.
 83 | 
 84 | dec0c5819f76 :) CREATE TABLE test_table
 85 | :-] (
 86 | :-]     test_column String,
 87 | :-]     test_column_two String
 88 | :-] ) ENGINE Memory;
 89 | 
 90 | CREATE TABLE test_table
 91 | (
 92 |     test_column String,
 93 |     test_column_two String
 94 | )
 95 | ENGINE = Memory
 96 | 
 97 | Ok.
 98 | 
 99 | 0 rows in set. Elapsed: 0.099 sec.
100 | 
101 | dec0c5819f76 :)
102 | ```
103 | 
104 | You can then execute the unit tests using nimble:
105 | 
106 | ```
107 | $ nimble tests
108 | ```
109 | 
110 | If you want you can customize the host that will be used as ClickHouse server
111 | during the unit tests using the `TEST_DB_HOSTNAME` environment variable, and
112 | this will be needed if you are using Docker Machine.
113 | 


--------------------------------------------------------------------------------
/db_clickhouse.nimble:
--------------------------------------------------------------------------------
 1 | # Package
 2 | 
 3 | version       = "0.3.0"
 4 | author        = "Leonardo Cecchi <leonardo.cecchi@gmail.com>"
 5 | description   = "ClickHouse Nim interface"
 6 | license       = "MIT"
 7 | srcDir        = "src"
 8 | 
 9 | # Dependencies
10 | 
11 | requires "nim >= 0.19.0"
12 | 


--------------------------------------------------------------------------------
/src/db_clickhouse.nim:
--------------------------------------------------------------------------------
  1 | import httpclient, uri, system, strutils, sequtils, httpcore
  2 | 
  3 | type
  4 |   DbConn* = ref object
  5 |     hostName: string ## We connect to this host name
  6 |     httpPort: int    ## and using this port
  7 |     client: HttpClient
  8 | 
  9 |   ## This error is raised when the query execution raised an error
 10 |   ## in the clickhouse engine
 11 |   DbError* = object of IOError
 12 | 
 13 |   ## A row
 14 |   Row* = seq[string]
 15 | 
 16 | ## Encode a string in the TabSeparated format
 17 | proc encodeString*(arg: string): string =
 18 |   result = ""
 19 |   for i in 0..len(arg)-1:
 20 |     if arg[i] == '\t':
 21 |       result &= "\\t"
 22 |     elif arg[i] == '\b':
 23 |       result &= "\\b"
 24 |     elif arg[i] == '\f':
 25 |       result &= "\\f"
 26 |     elif arg[i] == '\r':
 27 |       result &= "\\r"
 28 |     elif arg[i] == '\n':
 29 |       result &= "\\n"
 30 |     elif arg[i] == '\0':
 31 |       result &= "\\0"
 32 |     elif arg[i] == '\'':
 33 |       result &= "\\'"
 34 |     elif arg[i] == '\\':
 35 |       result &= "\\\\"
 36 |     else:
 37 |       result &= arg[i]
 38 | 
 39 | ## The following functions implement the TabSeparated format
 40 | ## ---------------------------------------------------------
 41 | 
 42 | ## Encode a row in the TabSeparated format
 43 | proc encodeRow*(args:varargs[string, `$`]): string =
 44 |   result = join(map(args, encodeString), "\t")
 45 | 
 46 | ## Encode a row in the TabSeparated format
 47 | proc encodeRow*(args:seq[string]): string =
 48 |   result = join(map(args, encodeString), "\t")
 49 | 
 50 | ## Encode a list of rows in the TabSeparated format
 51 | proc encodeRows*(data:seq[seq[string]]): string =
 52 |   result = join(map(data, encodeRow), "\n")
 53 | 
 54 | ## Decode a string
 55 | proc decodeString*(content: string): string =
 56 |   result = content
 57 |   while true:
 58 |     let idx = result.find("\\")
 59 |     if idx == -1:
 60 |       break
 61 |     if idx == result.len() - 1:
 62 |       break
 63 |     if result[idx+1] == 'n':
 64 |       result = result[0..<idx] & "\n" & result[idx+2..^1]
 65 |     elif result[idx+1] == 't':
 66 |       result = result[0..<idx] & "\t" & result[idx+2..^1]
 67 |     elif result[idx+1] == 'b':
 68 |       result = result[0..<idx] & "\b" & result[idx+2..^1]
 69 |     elif result[idx+1] == 'f':
 70 |       result = result[0..<idx] & "\f" & result[idx+2..^1]
 71 |     elif result[idx+1] == 'r':
 72 |       result = result[0..<idx] & "\r" & result[idx+2..^1]
 73 |     elif result[idx+1] == '0':
 74 |       result = result[0..<idx] & "\0" & result[idx+2..^1]
 75 |     elif result[idx+1] == '\'':
 76 |       result = result[0..<idx] & "\'" & result[idx+2..^1]
 77 |     elif result[idx+1] == '\\':
 78 |       result = result[0..<idx] & "\\" & result[idx+2..^1]
 79 | 
 80 | ## Decode a row from a string
 81 | proc decodeRow*(row:string): Row =
 82 |   result = row.split('\t').map(decodeString)
 83 | 
 84 | ## Decode a list of rows in the TabSeparated format
 85 | proc decodeRows*(data:string): seq[Row] =
 86 |   if data.endsWith('\n'):
 87 |     result = data[0..<len(data)-1].split('\n').map(decodeRow)
 88 |   else:
 89 |     result = data.split('\n').map(decodeRow)
 90 | 
 91 | ## Extract the first row of a TabSeparated query data
 92 | proc extractFirstRow*(data: string): string =
 93 |   let idx = data.find("\n")
 94 |   if idx == -1:
 95 |     result = data
 96 |   else:
 97 |     result = data[0..<idx]
 98 | 
 99 | ## Extract the first column of a TabSeparated query data
100 | proc extractFirstCol*(data: string): string =
101 |   let idx = data.find("\t")
102 |   if idx == -1:
103 |     result = data
104 |   else:
105 |     result = data[0..<idx]
106 | 
107 | 
108 | ## The following functions implement the DbConn interface
109 | ## ------------------------------------------------------
110 | 
111 | ## Create a new clickhouse client
112 | proc open*(hostName: string, httpPort:int = 8123): DbConn =
113 |   new result
114 |   result.hostName = hostName
115 |   result.httpPort = httpPort
116 |   result.client = newHttpClient()
117 | 
118 | ## Create a query URL, for internal use only
119 | proc createQueryUrl*(self: DbConn, query: string): string =
120 |   result = "http://" & self.hostName & ":" & $self.httpPort & "/?query=" & encodeUrl(query)
121 | 
122 | ## Ping the clickhouse database issuing a simple query, to find if
123 | ## it's available or not
124 | proc ping*(self: DbConn): bool =
125 |   let url = self.createQueryUrl("SELECT 1")
126 |   let test = self.client.getContent(url)
127 |   result = (test.strip() == "1")
128 | 
129 | proc checkForErrors(response: Response) =
130 |   if response.status[0] in {'3', '4', '5'}:
131 |     raise newException(DbError, response.body.strip())
132 | 
133 | ## Exec a query against the clickhouse database, without parsing the result
134 | proc exec*(self: DbConn, query:string, args: varargs[string, `$`]) =
135 |   let body = encodeRow(args)
136 |   let xh = newHttpHeaders({
137 |     "Content-Length": $len(body)
138 |   })
139 |   let response = self.client.request(
140 |     self.createQueryUrl(query),
141 |     body=body,
142 |     httpMethod=HttpPost,
143 |     headers=xh)
144 |   checkForErrors(response)
145 | 
146 | ## Exec a query without formatting the input parameters and without parsing
147 | ## the output results. This is useful for bulk imports/exports, even if it
148 | ## doesn't behave to the original interface
149 | proc execRaw*(self: DbConn, query:string, body:string = ""): string =
150 |   let xh = newHttpHeaders({
151 |     "Content-Length": $len(body)
152 |   })
153 |   let response = self.client.request(
154 |     self.createQueryUrl(query),
155 |     body=body,
156 |     httpMethod=HttpPost,
157 |     headers=xh)
158 |   checkForErrors(response)
159 |   result = response.body
160 | 
161 | ## Exec a query against the clickhouse database, without parsing the result
162 | proc execMultiple*(self: DbConn, query:string, args: seq[seq[string]]) =
163 |   let body = encodeRows(args)
164 |   let xh = newHttpHeaders({
165 |     "Content-Length": $len(body)
166 |   })
167 |   let response = self.client.request(
168 |     self.createQueryUrl(query),
169 |     body=body,
170 |     httpMethod=HttpPost,
171 |     headers=xh)
172 |   checkForErrors(response)
173 | 
174 | ## Close the http client
175 | proc close*(self: DbConn) =
176 |   self.client.close()
177 | 
178 | ## Executes the query and returns the whole result dataset
179 | proc getAllRows*(self: DbConn, query: string, args: varargs[string, `$`]): seq[Row] =
180 |   let body = encodeRow(args)
181 |   let xh = newHttpHeaders({
182 |     "Content-Length": $len(body)
183 |   })
184 |   let response = self.client.request(
185 |     self.createQueryUrl(query),
186 |     body=body,
187 |     httpMethod=HttpGet,
188 |     headers=xh)
189 |   checkForErrors(response)
190 |   result = decodeRows(response.body)
191 | 
192 | ## Executes the query and returns the first row of the dataset
193 | proc getRow*(self: DbConn, query: string, args: varargs[string, `$`]): Row =
194 |   let body = encodeRow(args)
195 |   let xh = newHttpHeaders({
196 |     "Content-Length": $len(body)
197 |   })
198 |   let response = self.client.request(
199 |     self.createQueryUrl(query),
200 |     body=body,
201 |     httpMethod=HttpGet,
202 |     headers=xh)
203 |   checkForErrors(response)
204 | 
205 |   result = decodeRow(extractFirstRow(response.body))
206 | 
207 | ## Executes the query and returns the first column of the
208 | ## first row of the dataset
209 | proc getValue*(self: DbConn, query: string, args: varargs[string, `$`]): string =
210 |   let body = encodeRow(args)
211 |   let xh = newHttpHeaders({
212 |     "Content-Length": $len(body)
213 |   })
214 |   let response = self.client.request(
215 |     self.createQueryUrl(query),
216 |     body=body,
217 |     httpMethod=HttpGet,
218 |     headers=xh)
219 |   checkForErrors(response)
220 | 
221 |   result = extractFirstCol(extractFirstRow(response.body))
222 | 


--------------------------------------------------------------------------------
/tests/test_clickhouse.nim:
--------------------------------------------------------------------------------
  1 | import unittest, db_clickhouse, os, strutils
  2 | 
  3 | ## To execute this suite of tests, you need to have a table generated
  4 | ## by the following query:
  5 | ##
  6 | ##     CREATE TABLE test_table
  7 | ##     (
  8 | ##         test_column String,
  9 | ##         test_column_two String
 10 | ##     ) ENGINE Memory;
 11 | ##
 12 | ## You can choose the hostname of the ClickHouse server you want to use as
 13 | ## testbed with the TEST_DB_HOSTNAME environment variable. The default
 14 | ## behavior is to use `localhost`
 15 | 
 16 | ## This function gets the hostname to use as a testing environment
 17 | proc getTestHostname(): string =
 18 |   let env = getEnv("TEST_DB_HOSTNAME")
 19 |   if env.len()==0:
 20 |     result = "localhost"
 21 |   else:
 22 |     result = env
 23 | 
 24 | suite "Clickhouse DB client tests":
 25 |   var client:DbConn = db_clickhouse.open(getTestHostname())
 26 | 
 27 |   test "create query URL":
 28 |     let u = "http://" & getTestHostname() & ":8123/?query=SELECT+1"
 29 |     check(client.createQueryUrl("SELECT 1") == u)
 30 | 
 31 |   test "ping":
 32 |     check(client.ping)
 33 | 
 34 |   test "a wrong query causes an exception":
 35 |     expect DbError:
 36 |       client.exec("SELECT COUNT(*) FROM unexistent_table")
 37 | 
 38 |   test "a legitimate query wan't cause any exception":
 39 |     client.exec("SELECT 1 FROM test_table")
 40 | 
 41 |   test "insert one row in the test table":
 42 |     client.exec("INSERT INTO test_table FORMAT TabSeparated", "first string", "second string")
 43 | 
 44 |   test "insert two rows in the test table":
 45 |     client.execMultiple(
 46 |       "INSERT INTO test_table FORMAT TabSeparated",
 47 |       @[@["one", "two"],
 48 |         @["three", "four"],
 49 |         @["0", "1"]])
 50 | 
 51 |   test "get all rows":
 52 |     let result = client.getAllRows("SELECT * FROM test_table ORDER BY test_column")
 53 |     check(result.len() > 0)
 54 |     check(result[0].len() == 2)
 55 |     check(result[0] == @["0", "1"])
 56 | 
 57 |   test "get first row":
 58 |     let row = client.getRow("SELECT * FROM test_table ORDER BY test_column")
 59 |     check(row.len() == 2)
 60 |     check(row == @["0", "1"])
 61 | 
 62 |   test "get first row":
 63 |     let value = client.getValue("SELECT * FROM test_table ORDER BY test_column")
 64 |     check(value == "0")
 65 | 
 66 |   test "exec raw data":
 67 |     discard client.execRaw("INSERT INTO test_table FORMAT TabSeparated", "first\tsecond")
 68 |     check(client.execRaw("SELECT * FROM test_table FORMAT TabSeparated").splitLines().len() > 2)
 69 | 
 70 |   test "get all rows returns the correct number of rows":
 71 |     client.exec("TRUNCATE test_table")
 72 |     client.exec("INSERT INTO test_table FORMAT TabSeparated", "first string", "second string")
 73 |     client.exec("INSERT INTO test_table FORMAT TabSeparated", "third string", "fourth string")
 74 |     let result = client.getAllRows("SELECT * FROM test_table ORDER BY test_column")
 75 |     check(result.len() == 2)
 76 | 
 77 | 
 78 | suite "TabSeparated encoding tests":
 79 |   test "basic string decoding":
 80 |     check(decodeString("ciao") == "ciao")
 81 |     check(decodeString("ci\\tao") == "ci\tao")
 82 | 
 83 |   test "basic row decoding":
 84 |     check(decodeRow("ciao\tda\tm\\te") == @["ciao", "da", "m\te"])
 85 | 
 86 |   test "basic data decoding":
 87 |     check(decodeRows("ciao\tda\tme\nprova\tper\tte") == @[@["ciao", "da", "me"], @["prova", "per", "te"]])
 88 | 
 89 |   test "basic string encoding":
 90 |     check(encodeString("cia\to") == "cia\\to")
 91 | 
 92 |   test "basic row encoding":
 93 |     check(encodeRow(1, "ciao", 3) == "1\tciao\t3")
 94 | 
 95 |   test "basic table encoding":
 96 |     check(encodeRows(@[@["one", "two"], @["three", "four"]]) == "one\ttwo\nthree\tfour")
 97 | 
 98 |   test "extract first row":
 99 |     check(extractFirstRow("test") == "test")
100 |     check(extractFirstRow("test\ttwo\nthree\tfour") == "test\ttwo")
101 | 
102 |   test "extract first column":
103 |     check(extractFirstCol("test") == "test")
104 |     check(extractFirstCol("test\ttwo") == "test")


--------------------------------------------------------------------------------
/tests/test_clickhouse.nims:
--------------------------------------------------------------------------------
1 | switch("path", "$projectDir/../src")


--------------------------------------------------------------------------------