190 |
191 |
192 | To successfully query, you need at least one database connection available.
193 | To create a database connection, you can add the Database connection smart cell.
194 |
195 |
196 |
To successfully query with {{ connectionType }}, you need to add the following dependency:
197 |
{{ missingDep }}
198 |
199 |
244 |
245 | To dynamically inject values into the query use double curly braces, like {{name}}.
246 |
247 |
248 | `,
249 |
250 | data() {
251 | return {
252 | isHelpBoxHidden: true,
253 | isSettingsBoxHidden: true,
254 | isConnectionExistent: false,
255 | isConnectionDisabled: true,
256 | payload: payload,
257 | missingDep: payload.missing_dep,
258 | availableDatabases: {
259 | postgres: "PostgreSQL",
260 | mysql: "MySQL",
261 | sqlite: "SQLite",
262 | bigquery: "Google BigQuery",
263 | athena: "AWS Athena",
264 | snowflake: "Snowflake",
265 | sqlserver: "SQL Server",
266 | duckdb: "DuckDB",
267 | clickhouse: "Clickhouse"
268 | },
269 | };
270 | },
271 |
272 | computed: {
273 | availableConnections() {
274 | const connection = this.payload.connection;
275 | const connections = this.payload.connections;
276 |
277 | const availableConnection =
278 | connection &&
279 | connections.some((conn) => conn.variable === connection.variable);
280 |
281 | if (connection === null) {
282 | this.isConnectionExistent = false;
283 | this.isConnectionDisabled = true;
284 | return [];
285 | } else if (availableConnection) {
286 | this.isConnectionExistent = true;
287 | this.isConnectionDisabled = false;
288 | return this.buildSelectConnectionOptions(connections);
289 | } else {
290 | this.isConnectionExistent = false;
291 | this.isConnectionDisabled = false;
292 | return this.buildSelectConnectionOptions([
293 | connection,
294 | ...connections,
295 | ]);
296 | }
297 | },
298 | hasCache() {
299 | return this.payload.connection?.type === 'athena';
300 | },
301 | hasTimeout() {
302 | const hasTimeout = ["postgres", "mysql", "sqlite", "sqlserver"];
303 | return hasTimeout.includes(this.payload.connection?.type);
304 | },
305 | connectionType() {
306 | return this.payload.connection?.type;
307 | }
308 | },
309 |
310 | methods: {
311 | buildSelectConnectionOptions(connections) {
312 | return connections.map((conn) => {
313 | return {
314 | label: `${conn.variable} (${this.availableDatabases[conn.type]})`,
315 | value: conn.variable,
316 | };
317 | });
318 | },
319 |
320 | handleResultVariableChange({ target: { value } }) {
321 | ctx.pushEvent("update_result_variable", value);
322 | },
323 |
324 | handleCacheQueryChange({ target: { checked } }) {
325 | ctx.pushEvent("update_cache_query", checked);
326 | },
327 |
328 | handleTimeoutChange({ target: { value } }) {
329 | ctx.pushEvent("update_timeout", value);
330 | },
331 |
332 | handleConnectionChange({ target: { value } }) {
333 | ctx.pushEvent("update_connection", value);
334 | },
335 |
336 | toggleHelpBox(_) {
337 | this.isHelpBoxHidden = !this.isHelpBoxHidden;
338 | },
339 |
340 | toggleSettingsBox(_) {
341 | this.isSettingsBoxHidden = !this.isSettingsBoxHidden;
342 | },
343 | },
344 | }).mount(ctx.root);
345 |
346 | ctx.handleEvent("update_result_variable", (variable) => {
347 | app.payload.result_variable = variable;
348 | });
349 |
350 | ctx.handleEvent("update_connection", (variable) => {
351 | const connection = app.payload.connections.find(
352 | (conn) => conn.variable === variable
353 | );
354 | app.payload.connection = connection;
355 | });
356 |
357 | ctx.handleEvent("update_cache_query", (value) => {
358 | app.payload.cache_query = value;
359 | });
360 |
361 | ctx.handleEvent("update_timeout", (timeout) => {
362 | app.payload.timeout = timeout;
363 | });
364 |
365 | ctx.handleEvent("connections", ({ connections, connection }) => {
366 | app.payload.connections = connections;
367 | app.payload.connection = connection;
368 | });
369 |
370 | ctx.handleEvent("missing_dep", ({ dep }) => {
371 | app.missingDep = dep;
372 | });
373 |
374 | ctx.handleSync(() => {
375 | // Synchronously invokes change listeners
376 | document.activeElement &&
377 | document.activeElement.dispatchEvent(
378 | new Event("change", { bubbles: true })
379 | );
380 | });
381 | }
382 |
--------------------------------------------------------------------------------
/lib/kino_db/sql_cell.ex:
--------------------------------------------------------------------------------
1 | defmodule KinoDB.SQLCell do
2 | @moduledoc false
3 |
4 | use Kino.JS, assets_path: "lib/assets/sql_cell"
5 | use Kino.JS.Live
6 | use Kino.SmartCell, name: "SQL query"
7 |
8 | @default_query "select * from table_name limit 100"
9 |
10 | @impl true
11 | def init(attrs, ctx) do
12 | connection =
13 | if conn_attrs = attrs["connection"] do
14 | %{variable: conn_attrs["variable"], type: conn_attrs["type"]}
15 | end
16 |
17 | query = attrs["query"] || @default_query
18 |
19 | ctx =
20 | assign(ctx,
21 | connections: [],
22 | connection: connection,
23 | result_variable: Kino.SmartCell.prefixed_var_name("result", attrs["result_variable"]),
24 | query: query,
25 | timeout: attrs["timeout"],
26 | cache_query: Map.get(attrs, "cache_query", true),
27 | missing_dep: missing_dep(connection)
28 | )
29 |
30 | {:ok, ctx, editor: [source: query, language: "sql"]}
31 | end
32 |
33 | @impl true
34 | def handle_connect(ctx) do
35 | payload = %{
36 | connections: ctx.assigns.connections,
37 | connection: ctx.assigns.connection,
38 | result_variable: ctx.assigns.result_variable,
39 | timeout: ctx.assigns.timeout,
40 | cache_query: ctx.assigns.cache_query,
41 | missing_dep: ctx.assigns.missing_dep
42 | }
43 |
44 | {:ok, payload, ctx}
45 | end
46 |
47 | @impl true
48 | def handle_editor_change(source, ctx) do
49 | {:ok, assign(ctx, query: source)}
50 | end
51 |
52 | @impl true
53 | def handle_event("update_connection", variable, ctx) do
54 | connection = Enum.find(ctx.assigns.connections, &(&1.variable == variable))
55 | ctx = assign(ctx, connection: connection)
56 | missing_dep = missing_dep(connection)
57 |
58 | ctx =
59 | if missing_dep == ctx.assigns.missing_dep do
60 | ctx
61 | else
62 | broadcast_event(ctx, "missing_dep", %{"dep" => missing_dep})
63 | assign(ctx, missing_dep: missing_dep)
64 | end
65 |
66 | broadcast_event(ctx, "update_connection", connection.variable)
67 |
68 | {:noreply, ctx}
69 | end
70 |
71 | def handle_event("update_result_variable", variable, ctx) do
72 | ctx =
73 | if Kino.SmartCell.valid_variable_name?(variable) do
74 | broadcast_event(ctx, "update_result_variable", variable)
75 | assign(ctx, result_variable: variable)
76 | else
77 | broadcast_event(ctx, "update_result_variable", ctx.assigns.result_variable)
78 | ctx
79 | end
80 |
81 | {:noreply, ctx}
82 | end
83 |
84 | def handle_event("update_timeout", timeout, ctx) do
85 | timeout =
86 | case Integer.parse(timeout) do
87 | {n, ""} -> n
88 | _ -> nil
89 | end
90 |
91 | ctx = assign(ctx, timeout: timeout)
92 | broadcast_event(ctx, "update_timeout", timeout)
93 | {:noreply, ctx}
94 | end
95 |
96 | def handle_event("update_cache_query", cache_query?, ctx) do
97 | ctx = assign(ctx, cache_query: cache_query?)
98 | broadcast_event(ctx, "update_cache_query", cache_query?)
99 | {:noreply, ctx}
100 | end
101 |
102 | @impl true
103 | def scan_binding(pid, binding, _env) do
104 | connections =
105 | for {key, value} <- binding,
106 | is_atom(key),
107 | type = connection_type(value),
108 | do: %{variable: Atom.to_string(key), type: type}
109 |
110 | send(pid, {:connections, connections})
111 | end
112 |
113 | @impl true
114 | def handle_info({:connections, connections}, ctx) do
115 | connection = search_connection(connections, ctx.assigns.connection)
116 | missing_dep = missing_dep(connection)
117 |
118 | ctx =
119 | if missing_dep == ctx.assigns.missing_dep do
120 | ctx
121 | else
122 | broadcast_event(ctx, "missing_dep", %{"dep" => missing_dep})
123 | assign(ctx, missing_dep: missing_dep)
124 | end
125 |
126 | broadcast_event(ctx, "connections", %{
127 | "connections" => connections,
128 | "connection" => connection
129 | })
130 |
131 | {:noreply,
132 | assign(ctx,
133 | connections: connections,
134 | connection: connection
135 | )}
136 | end
137 |
138 | defp search_connection([connection | _], nil), do: connection
139 |
140 | defp search_connection(connections, connection) do
141 | Enum.find(connections, connection, &(&1.variable == connection.variable))
142 | end
143 |
144 | @compile {:no_warn_undefined, {DBConnection, :connection_module, 1}}
145 |
146 | defp connection_type(connection) when is_pid(connection) do
147 | with true <- Code.ensure_loaded?(DBConnection),
148 | {:ok, module} <- DBConnection.connection_module(connection) do
149 | case Atom.to_string(module) do
150 | "Elixir.Postgrex" <> _ -> "postgres"
151 | "Elixir.MyXQL" <> _ -> "mysql"
152 | "Elixir.Exqlite" <> _ -> "sqlite"
153 | "Elixir.Tds" <> _ -> "sqlserver"
154 | _ -> nil
155 | end
156 | else
157 | _ -> connection_type_from_adbc(connection)
158 | end
159 | end
160 |
161 | defp connection_type(connection) when is_struct(connection, Req.Request) do
162 | cond do
163 | Keyword.has_key?(connection.request_steps, :athena_run) -> "athena"
164 | Keyword.has_key?(connection.request_steps, :clickhouse_run) -> "clickhouse"
165 | true -> nil
166 | end
167 | end
168 |
169 | defp connection_type(_connection), do: nil
170 |
171 | defp connection_type_from_adbc(connection) when is_pid(connection) do
172 | with true <- Code.ensure_loaded?(Adbc),
173 | {:ok, driver} <- Adbc.Connection.get_driver(connection) do
174 | Atom.to_string(driver)
175 | else
176 | _ -> nil
177 | end
178 | end
179 |
180 | @impl true
181 | def to_attrs(ctx) do
182 | %{
183 | "connection" =>
184 | if connection = ctx.assigns.connection do
185 | %{"variable" => connection.variable, "type" => connection.type}
186 | end,
187 | "result_variable" => ctx.assigns.result_variable,
188 | "query" => ctx.assigns.query,
189 | "timeout" => ctx.assigns.timeout,
190 | "cache_query" => ctx.assigns.cache_query
191 | }
192 | end
193 |
194 | @impl true
195 | def to_source(attrs) do
196 | attrs |> to_quoted() |> Kino.SmartCell.quoted_to_string()
197 | end
198 |
199 | # DBConnection-based
200 | defp to_quoted(%{"connection" => %{"type" => "postgres"}} = attrs) do
201 | to_quoted(attrs, quote(do: Postgrex), fn n -> "$#{n}" end)
202 | end
203 |
204 | defp to_quoted(%{"connection" => %{"type" => "mysql"}} = attrs) do
205 | to_quoted(attrs, quote(do: MyXQL), fn _n -> "?" end)
206 | end
207 |
208 | defp to_quoted(%{"connection" => %{"type" => "sqlite"}} = attrs) do
209 | to_quoted(attrs, quote(do: Exqlite), fn n -> "?#{n}" end)
210 | end
211 |
212 | defp to_quoted(%{"connection" => %{"type" => "sqlserver"}} = attrs) do
213 | to_quoted(attrs, quote(do: Tds), fn n -> "@#{n}" end)
214 | end
215 |
216 | # Explorer-based
217 | defp to_quoted(%{"connection" => %{"type" => "snowflake"}} = attrs) do
218 | to_quoted_adbc(attrs, fn n -> "?#{n}" end)
219 | end
220 |
221 | defp to_quoted(%{"connection" => %{"type" => "duckdb"}} = attrs) do
222 | to_quoted_adbc(attrs, fn n -> "?#{n}" end)
223 | end
224 |
225 | defp to_quoted(%{"connection" => %{"type" => "bigquery"}} = attrs) do
226 | to_quoted_adbc(attrs, fn _n -> "?" end)
227 | end
228 |
229 | # Req-based
230 | defp to_quoted(%{"connection" => %{"type" => "athena"}} = attrs) do
231 | to_quoted_req_query(attrs, quote(do: ReqAthena), fn _n -> "?" end)
232 | end
233 |
234 | defp to_quoted(%{"connection" => %{"type" => "clickhouse"}} = attrs) do
235 | to_quoted_req_query(attrs, quote(do: ReqCH), fn n, inner ->
236 | name =
237 | if String.match?(inner, ~r/[^a-z0-9_]/) do
238 | "param_#{n}"
239 | else
240 | inner
241 | end
242 |
243 | "{#{name}:String}"
244 | end)
245 | end
246 |
247 | defp to_quoted(_ctx) do
248 | quote do
249 | end
250 | end
251 |
252 | defp to_quoted(attrs, quoted_module, next) do
253 | {query, params} = parameterize(attrs["query"], attrs["connection"]["type"], next)
254 | opts_args = query_opts_args(attrs)
255 |
256 | quote do
257 | unquote(quoted_var(attrs["result_variable"])) =
258 | unquote(quoted_module).query!(
259 | unquote(quoted_var(attrs["connection"]["variable"])),
260 | unquote(quoted_query(query)),
261 | unquote(params),
262 | unquote_splicing(opts_args)
263 | )
264 | end
265 | end
266 |
267 | defp to_quoted_adbc(attrs, next) do
268 | {query, params} = parameterize(attrs["query"], attrs["connection"]["type"], next)
269 |
270 | quote do
271 | unquote(quoted_var(attrs["result_variable"])) =
272 | Adbc.Connection.query!(
273 | unquote(quoted_var(attrs["connection"]["variable"])),
274 | unquote(quoted_query(query)),
275 | unquote(params)
276 | )
277 | end
278 | end
279 |
280 | defp to_quoted_req_query(attrs, quoted_module, next) do
281 | {query, params} = parameterize(attrs["query"], attrs["connection"]["type"], next)
282 | opts_args = query_opts_args(attrs)
283 | var = quoted_var(attrs["result_variable"])
284 |
285 | quote do
286 | unquote(var) =
287 | unquote(quoted_module).query!(
288 | unquote(quoted_var(attrs["connection"]["variable"])),
289 | unquote(quoted_query(query)),
290 | unquote(params),
291 | unquote_splicing(opts_args)
292 | ).body
293 |
294 | Kino.DataTable.new(unquote(var))
295 | end
296 | end
297 |
298 | defp quoted_var(nil), do: nil
299 | defp quoted_var(string), do: {String.to_atom(string), [], nil}
300 |
301 | defp quoted_query(query) do
302 | if String.contains?(query, "\n") do
303 | {:sigil_S, [delimiter: ~s["""]], [{:<<>>, [], [query <> "\n"]}, []]}
304 | else
305 | {:sigil_S, [delimiter: ~s["]], [{:<<>>, [], [query]}, []]}
306 | end
307 | end
308 |
309 | @connection_types_with_timeout ~w|postgres mysql sqlite sqlserver|
310 |
311 | defp query_opts_args(%{"connection" => %{"type" => type}, "timeout" => timeout})
312 | when timeout != nil and type in @connection_types_with_timeout,
313 | do: [[timeout: timeout * 1000]]
314 |
315 | defp query_opts_args(%{"connection" => %{"type" => "athena"}} = attrs) do
316 | [[format: :explorer] ++ if(attrs["cache_query"], do: [], else: [cache_query: false])]
317 | end
318 |
319 | defp query_opts_args(%{"connection" => %{"type" => "clickhouse"}}),
320 | do: [[format: :explorer]]
321 |
322 | defp query_opts_args(_attrs), do: []
323 |
324 | defp parameterize(query, type, next) do
325 | parameterize(query, "", [], 1, type, next)
326 | end
327 |
328 | defp parameterize("", raw, params, _n, _type, _next) do
329 | {raw, Enum.reverse(params)}
330 | end
331 |
332 | defp parameterize("--" <> _ = query, raw, params, n, type, next) do
333 | {comment, rest} =
334 | case String.split(query, "\n", parts: 2) do
335 | [comment, rest] -> {comment <> "\n", rest}
336 | [comment] -> {comment, ""}
337 | end
338 |
339 | parameterize(rest, raw <> comment, params, n, type, next)
340 | end
341 |
342 | defp parameterize("/*" <> _ = query, raw, params, n, type, next) do
343 | {comment, rest} =
344 | case String.split(query, "*/", parts: 2) do
345 | [comment, rest] -> {comment <> "*/", rest}
346 | [comment] -> {comment, ""}
347 | end
348 |
349 | parameterize(rest, raw <> comment, params, n, type, next)
350 | end
351 |
352 | defp parameterize("{{" <> rest = query, raw, params, n, type, next) do
353 | with [inner, rest] <- String.split(rest, "}}", parts: 2),
354 | sql_param <- apply_next(next, n, inner),
355 | {:ok, param} <- quote_param(type, inner, sql_param) do
356 | parameterize(rest, raw <> sql_param, [param | params], n + 1, type, next)
357 | else
358 | _ -> parameterize("", raw <> query, params, n, type, next)
359 | end
360 | end
361 |
362 | defp parameterize(<
>, raw, params, n, type, next) do
363 | parameterize(rest, <>, params, n, type, next)
364 | end
365 |
366 | defp apply_next(next, n, _inner) when is_function(next, 1), do: next.(n)
367 | defp apply_next(next, n, inner) when is_function(next, 2), do: next.(n, inner)
368 |
369 | defp quote_param("clickhouse", inner, sql_param) do
370 | with {:ok, inner_ast} <- Code.string_to_quoted(inner) do
371 | name =
372 | sql_param |> String.trim_leading("{") |> String.split(":", parts: 2) |> List.first()
373 |
374 | {:ok,
375 | quote do
376 | {unquote(name), unquote(inner_ast)}
377 | end}
378 | end
379 | end
380 |
381 | defp quote_param("sqlserver", inner, sql_param) do
382 | with {:ok, inner_ast} <- Code.string_to_quoted(inner) do
383 | {:ok,
384 | quote do
385 | %Tds.Parameter{name: unquote(sql_param), value: unquote(inner_ast)}
386 | end}
387 | end
388 | end
389 |
390 | defp quote_param(_type, inner, _sql_param) do
391 | Code.string_to_quoted(inner)
392 | end
393 |
394 | defp missing_dep(_), do: nil
395 | end
396 |
--------------------------------------------------------------------------------
/test/kino_db/connection_cell_test.exs:
--------------------------------------------------------------------------------
1 | defmodule KinoDB.ConnectionCellTest do
2 | use ExUnit.Case, async: true
3 |
4 | import Kino.Test
5 |
6 | alias KinoDB.ConnectionCell
7 |
8 | setup :configure_livebook_bridge
9 |
10 | @attrs %{
11 | "variable" => "db",
12 | "type" => "postgres",
13 | "hostname" => "localhost",
14 | "port" => 4444,
15 | "use_ipv6" => false,
16 | "use_ssl" => false,
17 | "cacertfile" => "",
18 | "auth_type" => "auth_jwt",
19 | "username" => "admin",
20 | "password" => "pass",
21 | "use_password_secret" => false,
22 | "password_secret" => "",
23 | "database" => "default",
24 | "database_path" => "/path/to/sqlite3.db",
25 | "project_id" => "foo",
26 | "credentials_json" => "",
27 | "default_dataset_id" => "",
28 | "access_key_id" => "id",
29 | "secret_access_key" => "secret",
30 | "use_secret_access_key_secret" => false,
31 | "secret_access_key_secret" => "",
32 | "private_key" => "-----BEGIN PRIVATE KEY-----...",
33 | "use_private_key_secret" => false,
34 | "private_key_secret" => "",
35 | "use_encrypted_private_key" => false,
36 | "private_key_passphrase" => "passphrase",
37 | "use_private_key_passphrase_secret" => false,
38 | "private_key_passphrase_secret" => "",
39 | "token" => "token",
40 | "region" => "region",
41 | "output_location" => "s3://my-bucket",
42 | "workgroup" => "primary",
43 | "account" => "account",
44 | "schema" => "schema",
45 | "warehouse" => ""
46 | }
47 |
48 | @empty_required_fields %{
49 | "variable" => "db",
50 | "type" => "postgres",
51 | "hostname" => "",
52 | "port" => nil,
53 | "database_path" => "",
54 | "project_id" => "",
55 | "access_key_id" => "",
56 | "secret_access_key" => "",
57 | "region" => "",
58 | "account" => "",
59 | "username" => "",
60 | "auth_type" => ""
61 | }
62 |
63 | describe "initialization" do
64 | test "returns default source when started with missing attrs" do
65 | {_kino, source} = start_smart_cell!(ConnectionCell, %{"variable" => "conn"})
66 |
67 | assert source ==
68 | """
69 | opts = [hostname: "localhost", port: 5432, username: "", password: "", database: ""]
70 | {:ok, conn} = Kino.start_child({Postgrex, opts})\
71 | """
72 | end
73 | end
74 |
75 | describe "code generation" do
76 | test "restores source code from attrs" do
77 | assert ConnectionCell.to_source(@attrs) === ~s'''
78 | opts = [
79 | hostname: "localhost",
80 | port: 4444,
81 | username: "admin",
82 | password: "pass",
83 | database: "default"
84 | ]
85 |
86 | {:ok, db} = Kino.start_child({Postgrex, opts})\
87 | '''
88 |
89 | attrs = Map.put(@attrs, "use_ipv6", true)
90 |
91 | assert ConnectionCell.to_source(attrs) === ~s'''
92 | opts = [
93 | hostname: "localhost",
94 | port: 4444,
95 | username: "admin",
96 | password: "pass",
97 | database: "default",
98 | socket_options: [:inet6]
99 | ]
100 |
101 | {:ok, db} = Kino.start_child({Postgrex, opts})\
102 | '''
103 |
104 | attrs = Map.put(@attrs, "use_ssl", true)
105 |
106 | assert ConnectionCell.to_source(attrs) === ~s'''
107 | opts = [
108 | hostname: "localhost",
109 | port: 4444,
110 | username: "admin",
111 | password: "pass",
112 | database: "default",
113 | ssl: [cacerts: :public_key.cacerts_get()]
114 | ]
115 |
116 | {:ok, db} = Kino.start_child({Postgrex, opts})\
117 | '''
118 |
119 | attrs = Map.merge(@attrs, %{"use_ssl" => true, "cacertfile" => "/path/to/cacertfile"})
120 |
121 | assert ConnectionCell.to_source(attrs) === ~s'''
122 | opts = [
123 | hostname: "localhost",
124 | port: 4444,
125 | username: "admin",
126 | password: "pass",
127 | database: "default",
128 | ssl: [cacertfile: "/path/to/cacertfile"]
129 | ]
130 |
131 | {:ok, db} = Kino.start_child({Postgrex, opts})\
132 | '''
133 |
134 | attrs = Map.delete(@attrs, "password") |> Map.merge(%{"password_secret" => "PASS"})
135 |
136 | assert ConnectionCell.to_source(attrs) === ~s'''
137 | opts = [
138 | hostname: "localhost",
139 | port: 4444,
140 | username: "admin",
141 | password: System.fetch_env!("LB_PASS"),
142 | database: "default"
143 | ]
144 |
145 | {:ok, db} = Kino.start_child({Postgrex, opts})\
146 | '''
147 |
148 | assert ConnectionCell.to_source(put_in(@attrs["type"], "mysql")) == ~s'''
149 | opts = [
150 | hostname: "localhost",
151 | port: 4444,
152 | username: "admin",
153 | password: "pass",
154 | database: "default"
155 | ]
156 |
157 | {:ok, db} = Kino.start_child({MyXQL, opts})\
158 | '''
159 |
160 | assert ConnectionCell.to_source(put_in(@attrs["type"], "sqlite")) == ~s'''
161 | opts = [database: "/path/to/sqlite3.db"]
162 | {:ok, db} = Kino.start_child({Exqlite, opts})\
163 | '''
164 |
165 | assert ConnectionCell.to_source(put_in(@attrs["type"], "athena")) == ~s'''
166 | db =
167 | ReqAthena.new(
168 | access_key_id: "id",
169 | database: "default",
170 | output_location: "s3://my-bucket",
171 | region: "region",
172 | secret_access_key: "secret",
173 | token: "token",
174 | workgroup: "primary",
175 | http_errors: :raise
176 | )
177 |
178 | :ok\
179 | '''
180 |
181 | attrs =
182 | Map.delete(@attrs, "secret_access_key")
183 | |> Map.merge(%{"type" => "athena", "secret_access_key_secret" => "ATHENA_KEY"})
184 |
185 | assert ConnectionCell.to_source(attrs) == ~s'''
186 | db =
187 | ReqAthena.new(
188 | access_key_id: "id",
189 | database: "default",
190 | output_location: "s3://my-bucket",
191 | region: "region",
192 | secret_access_key: System.fetch_env!("LB_ATHENA_KEY"),
193 | token: "token",
194 | workgroup: "primary",
195 | http_errors: :raise
196 | )
197 |
198 | :ok\
199 | '''
200 |
201 | attrs =
202 | Map.delete(@attrs, "password_secret")
203 | |> Map.merge(%{"variable" => "conn", "auth_type" => "auth_snowflake"})
204 |
205 | assert ConnectionCell.to_source(put_in(attrs["type"], "snowflake")) == ~s'''
206 | :ok = Adbc.download_driver!(:snowflake)
207 |
208 | {:ok, db} =
209 | Kino.start_child(
210 | {Adbc.Database,
211 | driver: :snowflake,
212 | username: "admin",
213 | "adbc.snowflake.sql.account": "account",
214 | "adbc.snowflake.sql.db": "default",
215 | "adbc.snowflake.sql.schema": "schema",
216 | "adbc.snowflake.sql.warehouse": "",
217 | "adbc.snowflake.sql.auth_type": "auth_snowflake",
218 | password: "pass"}
219 | )
220 |
221 | {:ok, conn} = Kino.start_child({Adbc.Connection, database: db})\
222 | '''
223 |
224 | attrs =
225 | Map.delete(@attrs, "private_key_secret")
226 | |> Map.merge(%{"variable" => "conn", "auth_type" => "auth_jwt"})
227 |
228 | assert ConnectionCell.to_source(put_in(attrs["type"], "snowflake")) == ~s'''
229 | :ok = Adbc.download_driver!(:snowflake)
230 |
231 | {:ok, db} =
232 | Kino.start_child(
233 | {Adbc.Database,
234 | driver: :snowflake,
235 | username: "admin",
236 | "adbc.snowflake.sql.account": "account",
237 | "adbc.snowflake.sql.db": "default",
238 | "adbc.snowflake.sql.schema": "schema",
239 | "adbc.snowflake.sql.warehouse": "",
240 | "adbc.snowflake.sql.auth_type": "auth_jwt",
241 | "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_value":
242 | "-----BEGIN PRIVATE KEY-----..."}
243 | )
244 |
245 | {:ok, conn} = Kino.start_child({Adbc.Connection, database: db})\
246 | '''
247 |
248 | attrs =
249 | Map.drop(@attrs, ["private_key_secret", "private_key_passphrase_secret"])
250 | |> Map.merge(%{
251 | "variable" => "conn",
252 | "auth_type" => "auth_jwt",
253 | "use_encrypted_private_key" => true
254 | })
255 |
256 | assert ConnectionCell.to_source(put_in(attrs["type"], "snowflake")) == ~s'''
257 | :ok = Adbc.download_driver!(:snowflake)
258 |
259 | {:ok, db} =
260 | Kino.start_child(
261 | {Adbc.Database,
262 | driver: :snowflake,
263 | username: "admin",
264 | "adbc.snowflake.sql.account": "account",
265 | "adbc.snowflake.sql.db": "default",
266 | "adbc.snowflake.sql.schema": "schema",
267 | "adbc.snowflake.sql.warehouse": "",
268 | "adbc.snowflake.sql.auth_type": "auth_jwt",
269 | "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_value":
270 | "-----BEGIN PRIVATE KEY-----...",
271 | "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_password": "passphrase"}
272 | )
273 |
274 | {:ok, conn} = Kino.start_child({Adbc.Connection, database: db})\
275 | '''
276 |
277 | assert ConnectionCell.to_source(put_in(attrs["type"], "clickhouse")) == ~s'''
278 | conn =
279 | ReqCH.new(
280 | database: "default",
281 | auth: {:basic, "admin:pass"},
282 | base_url: "http://localhost:4444"
283 | )
284 |
285 | :ok\
286 | '''
287 |
288 | assert ConnectionCell.to_source(put_in(attrs["type"], "bigquery")) == ~s'''
289 | :ok = Adbc.download_driver!(:bigquery)
290 |
291 | {:ok, db} =
292 | Kino.start_child(
293 | {Adbc.Database, driver: :bigquery, "adbc.bigquery.sql.project_id": "foo"}
294 | )
295 |
296 | {:ok, conn} = Kino.start_child({Adbc.Connection, database: db})\
297 | '''
298 | end
299 |
300 | test "generates empty source code when required fields are missing" do
301 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "postgres")) == ""
302 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "mysql")) == ""
303 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "sqlite")) == ""
304 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "bigquery")) == ""
305 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "athena")) == ""
306 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "snowflake")) == ""
307 | assert ConnectionCell.to_source(put_in(@empty_required_fields["type"], "clickhouse")) == ""
308 | end
309 |
310 | test "generates empty source code when all conditional fields are missing" do
311 | attrs = Map.merge(@attrs, %{"type" => "athena", "workgroup" => "", "output_location" => ""})
312 |
313 | assert ConnectionCell.to_source(attrs) == ""
314 | end
315 | end
316 |
317 | test "when a field changes, broadcasts the change and sends source update" do
318 | {kino, _source} = start_smart_cell!(ConnectionCell, %{"variable" => "conn"})
319 |
320 | push_event(kino, "update_field", %{"field" => "hostname", "value" => "myhost"})
321 |
322 | assert_broadcast_event(kino, "update", %{"fields" => %{"hostname" => "myhost"}})
323 |
324 | assert_smart_cell_update(kino, %{"hostname" => "myhost"}, """
325 | opts = [hostname: "myhost", port: 5432, username: "", password: "", database: ""]
326 | {:ok, conn} = Kino.start_child({Postgrex, opts})\
327 | """)
328 | end
329 |
330 | test "when an invalid variable name is set, restores the previous value" do
331 | {kino, _source} = start_smart_cell!(ConnectionCell, %{"variable" => "db"})
332 |
333 | push_event(kino, "update_field", %{"field" => "variable", "value" => "DB"})
334 |
335 | assert_broadcast_event(kino, "update", %{"fields" => %{"variable" => "db"}})
336 | end
337 |
338 | test "when the database type changes, restores the default port for that database" do
339 | {kino, _source} =
340 | start_smart_cell!(ConnectionCell, %{
341 | "variable" => "conn",
342 | "type" => "postgres",
343 | "port" => 5432
344 | })
345 |
346 | push_event(kino, "update_field", %{"field" => "type", "value" => "mysql"})
347 |
348 | assert_broadcast_event(kino, "update", %{"fields" => %{"type" => "mysql", "port" => 3306}})
349 |
350 | assert_smart_cell_update(kino, %{"type" => "mysql", "port" => 3306}, """
351 | opts = [hostname: "localhost", port: 3306, username: "", password: "", database: ""]
352 | {:ok, conn} = Kino.start_child({MyXQL, opts})\
353 | """)
354 | end
355 |
356 | test "password from secrets" do
357 | {kino, _source} =
358 | start_smart_cell!(ConnectionCell, %{
359 | "variable" => "conn",
360 | "type" => "postgres",
361 | "port" => 5432
362 | })
363 |
364 | push_event(kino, "update_field", %{"field" => "use_password_secret", "value" => true})
365 | assert_broadcast_event(kino, "update", %{"fields" => %{"use_password_secret" => true}})
366 |
367 | push_event(kino, "update_field", %{"field" => "password_secret", "value" => "PASS"})
368 | assert_broadcast_event(kino, "update", %{"fields" => %{"password_secret" => "PASS"}})
369 |
370 | assert_smart_cell_update(
371 | kino,
372 | %{"password_secret" => "PASS"},
373 | """
374 | opts = [
375 | hostname: "localhost",
376 | port: 5432,
377 | username: "",
378 | password: System.fetch_env!("LB_PASS"),
379 | database: ""
380 | ]
381 |
382 | {:ok, conn} = Kino.start_child({Postgrex, opts})\
383 | """
384 | )
385 | end
386 |
387 | test "athena secret key from secrets" do
388 | {kino, _source} =
389 | start_smart_cell!(ConnectionCell, %{
390 | "variable" => "conn",
391 | "type" => "athena",
392 | "database" => "default",
393 | "access_key_id" => "id",
394 | "secret_access_key" => "secret_key",
395 | "token" => "token",
396 | "region" => "region",
397 | "output_location" => "s3://my-bucket",
398 | "workgroup" => "primary"
399 | })
400 |
401 | push_event(kino, "update_field", %{"field" => "use_secret_access_key_secret", "value" => true})
402 |
403 | assert_broadcast_event(kino, "update", %{
404 | "fields" => %{"use_secret_access_key_secret" => true}
405 | })
406 |
407 | push_event(kino, "update_field", %{
408 | "field" => "secret_access_key_secret",
409 | "value" => "ATHENA_KEY"
410 | })
411 |
412 | assert_broadcast_event(kino, "update", %{
413 | "fields" => %{"secret_access_key_secret" => "ATHENA_KEY"}
414 | })
415 |
416 | assert_smart_cell_update(
417 | kino,
418 | %{"secret_access_key_secret" => "ATHENA_KEY"},
419 | """
420 | conn =
421 | ReqAthena.new(
422 | access_key_id: "id",
423 | database: "default",
424 | output_location: "s3://my-bucket",
425 | region: "region",
426 | secret_access_key: System.fetch_env!("LB_ATHENA_KEY"),
427 | token: "token",
428 | workgroup: "primary",
429 | http_errors: :raise
430 | )
431 |
432 | :ok\
433 | """
434 | )
435 | end
436 | end
437 |
--------------------------------------------------------------------------------
/test/kino_db/sql_cell_test.exs:
--------------------------------------------------------------------------------
1 | defmodule KinoDB.SQLCellTest do
2 | use ExUnit.Case, async: true
3 |
4 | import Kino.Test
5 |
6 | alias KinoDB.SQLCell
7 |
8 | setup :configure_livebook_bridge
9 |
10 | describe "initialization" do
11 | test "restores source code from attrs" do
12 | attrs = %{
13 | "connection" => %{"variable" => "db", "type" => "postgres"},
14 | "result_variable" => "ids_result",
15 | "timeout" => nil,
16 | "query" => "SELECT id FROM users",
17 | "data_frame_alias" => Explorer.DataFrame
18 | }
19 |
20 | {_kino, source} = start_smart_cell!(SQLCell, attrs)
21 |
22 | assert source ==
23 | """
24 | ids_result = Postgrex.query!(db, ~S"SELECT id FROM users", [])\
25 | """
26 | end
27 | end
28 |
29 | test "when an invalid result variable name is set, restores the previous value" do
30 | {kino, _source} = start_smart_cell!(SQLCell, %{"result_variable" => "result"})
31 |
32 | push_event(kino, "update_result_variable", "RESULT")
33 |
34 | assert_broadcast_event(kino, "update_result_variable", "result")
35 | end
36 |
37 | test "finds database connections in binding and sends them to the client" do
38 | {kino, _source} = start_smart_cell!(SQLCell, %{})
39 |
40 | conn_pid = spawn_fake_postgrex_connection()
41 |
42 | binding = [non_conn: self(), conn: conn_pid]
43 | env = Code.env_for_eval([])
44 | SQLCell.scan_binding(kino.pid, binding, env)
45 |
46 | connection = %{variable: "conn", type: "postgres"}
47 |
48 | assert_broadcast_event(kino, "connections", %{
49 | "connections" => [^connection],
50 | "connection" => ^connection
51 | })
52 | end
53 |
54 | test "keeps the currently selected connection if not available in binding" do
55 | attrs = %{"connection" => %{"variable" => "conn1", "type" => "postgres"}}
56 | {kino, _source} = start_smart_cell!(SQLCell, attrs)
57 |
58 | conn_pid = spawn_fake_postgrex_connection()
59 |
60 | binding = [conn: conn_pid]
61 | env = Code.env_for_eval([])
62 | SQLCell.scan_binding(kino.pid, binding, env)
63 |
64 | current_connection = %{variable: "conn1", type: "postgres"}
65 | connection = %{variable: "conn", type: "postgres"}
66 |
67 | assert_broadcast_event(kino, "connections", %{
68 | "connections" => [^connection],
69 | "connection" => ^current_connection
70 | })
71 | end
72 |
73 | test "updates the selected connection type when the variable changes" do
74 | attrs = %{"connection" => %{"variable" => "conn", "type" => "sqlite"}}
75 | {kino, _source} = start_smart_cell!(SQLCell, attrs)
76 |
77 | conn_pid = spawn_fake_postgrex_connection()
78 |
79 | binding = [conn: conn_pid]
80 | env = Code.env_for_eval([])
81 | SQLCell.scan_binding(kino.pid, binding, env)
82 |
83 | connection = %{variable: "conn", type: "postgres"}
84 |
85 | assert_broadcast_event(kino, "connections", %{
86 | "connections" => [^connection],
87 | "connection" => ^connection
88 | })
89 | end
90 |
91 | describe "code generation" do
92 | test "uses regular string for a single-line query" do
93 | attrs = %{
94 | "connection" => %{"variable" => "conn", "type" => "postgres"},
95 | "result_variable" => "result",
96 | "cache_query" => true,
97 | "timeout" => nil,
98 | "query" => "SELECT id FROM users",
99 | "data_frame_alias" => Explorer.DataFrame
100 | }
101 |
102 | assert SQLCell.to_source(attrs) == """
103 | result = Postgrex.query!(conn, ~S"SELECT id FROM users", [])\
104 | """
105 |
106 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == """
107 | result = MyXQL.query!(conn, ~S"SELECT id FROM users", [])\
108 | """
109 |
110 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == """
111 | result = Exqlite.query!(conn, ~S"SELECT id FROM users", [])\
112 | """
113 |
114 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == """
115 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
116 | """
117 |
118 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "athena")) == """
119 | result = ReqAthena.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
120 | Kino.DataTable.new(result)\
121 | """
122 |
123 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == """
124 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
125 | """
126 |
127 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == """
128 | result = ReqCH.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
129 | Kino.DataTable.new(result)\
130 | """
131 |
132 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == """
133 | result = Tds.query!(conn, ~S"SELECT id FROM users", [])\
134 | """
135 | end
136 |
137 | test "uses heredoc string for a multi-line query" do
138 | attrs = %{
139 | "connection" => %{"variable" => "conn", "type" => "postgres"},
140 | "result_variable" => "result",
141 | "cache_query" => true,
142 | "timeout" => nil,
143 | "query" => "SELECT id FROM users\nWHERE last_name = 'Sherlock'",
144 | "data_frame_alias" => Explorer.DataFrame
145 | }
146 |
147 | assert SQLCell.to_source(attrs) == ~s'''
148 | result =
149 | Postgrex.query!(
150 | conn,
151 | ~S"""
152 | SELECT id FROM users
153 | WHERE last_name = 'Sherlock'
154 | """,
155 | []
156 | )\
157 | '''
158 |
159 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == ~s'''
160 | result =
161 | MyXQL.query!(
162 | conn,
163 | ~S"""
164 | SELECT id FROM users
165 | WHERE last_name = 'Sherlock'
166 | """,
167 | []
168 | )\
169 | '''
170 |
171 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == ~s'''
172 | result =
173 | Exqlite.query!(
174 | conn,
175 | ~S"""
176 | SELECT id FROM users
177 | WHERE last_name = 'Sherlock'
178 | """,
179 | []
180 | )\
181 | '''
182 |
183 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == ~s'''
184 | result =
185 | Adbc.Connection.query!(
186 | conn,
187 | ~S"""
188 | SELECT id FROM users
189 | WHERE last_name = 'Sherlock'
190 | """,
191 | []
192 | )\
193 | '''
194 |
195 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "athena")) == ~s'''
196 | result =
197 | ReqAthena.query!(
198 | conn,
199 | ~S"""
200 | SELECT id FROM users
201 | WHERE last_name = 'Sherlock'
202 | """,
203 | [],
204 | format: :explorer
205 | ).body
206 |
207 | Kino.DataTable.new(result)\
208 | '''
209 |
210 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == ~s'''
211 | result =
212 | Adbc.Connection.query!(
213 | conn,
214 | ~S"""
215 | SELECT id FROM users
216 | WHERE last_name = 'Sherlock'
217 | """,
218 | []
219 | )\
220 | '''
221 |
222 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == ~s'''
223 | result =
224 | ReqCH.query!(
225 | conn,
226 | ~S"""
227 | SELECT id FROM users
228 | WHERE last_name = 'Sherlock'
229 | """,
230 | [],
231 | format: :explorer
232 | ).body
233 |
234 | Kino.DataTable.new(result)\
235 | '''
236 |
237 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == ~s'''
238 | result =
239 | Tds.query!(
240 | conn,
241 | ~S"""
242 | SELECT id FROM users
243 | WHERE last_name = 'Sherlock'
244 | """,
245 | []
246 | )\
247 | '''
248 | end
249 |
250 | test "parses parameter expressions" do
251 | attrs = %{
252 | "connection" => %{"variable" => "conn", "type" => "postgres"},
253 | "result_variable" => "result",
254 | "cache_query" => true,
255 | "timeout" => nil,
256 | "query" => ~s/SELECT id FROM users WHERE id {{user_id}} AND name LIKE {{search <> "%"}}/,
257 | "data_frame_alias" => Explorer.DataFrame
258 | }
259 |
260 | assert SQLCell.to_source(attrs) == ~s'''
261 | result =
262 | Postgrex.query!(conn, ~S"SELECT id FROM users WHERE id $1 AND name LIKE $2", [
263 | user_id,
264 | search <> "%"
265 | ])\
266 | '''
267 |
268 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == ~s'''
269 | result =
270 | MyXQL.query!(conn, ~S"SELECT id FROM users WHERE id ? AND name LIKE ?", [
271 | user_id,
272 | search <> "%"
273 | ])\
274 | '''
275 |
276 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == ~s'''
277 | result =
278 | Exqlite.query!(conn, ~S"SELECT id FROM users WHERE id ?1 AND name LIKE ?2", [
279 | user_id,
280 | search <> "%"
281 | ])\
282 | '''
283 |
284 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == ~s'''
285 | result =
286 | Adbc.Connection.query!(conn, ~S"SELECT id FROM users WHERE id ? AND name LIKE ?", [
287 | user_id,
288 | search <> \"%\"
289 | ])\
290 | '''
291 |
292 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "athena")) == ~s'''
293 | result =
294 | ReqAthena.query!(
295 | conn,
296 | ~S"SELECT id FROM users WHERE id ? AND name LIKE ?",
297 | [user_id, search <> "%"],
298 | format: :explorer
299 | ).body
300 |
301 | Kino.DataTable.new(result)\
302 | '''
303 |
304 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == ~s'''
305 | result =
306 | Adbc.Connection.query!(conn, ~S"SELECT id FROM users WHERE id ?1 AND name LIKE ?2", [
307 | user_id,
308 | search <> \"%\"
309 | ])\
310 | '''
311 |
312 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == ~s'''
313 | result =
314 | ReqCH.query!(
315 | conn,
316 | ~S"SELECT id FROM users WHERE id {user_id:String} AND name LIKE {param_2:String}",
317 | [{"user_id", user_id}, {"param_2", search <> \"%\"}],
318 | format: :explorer
319 | ).body
320 |
321 | Kino.DataTable.new(result)\
322 | '''
323 |
324 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == ~s'''
325 | result =
326 | Tds.query!(conn, ~S"SELECT id FROM users WHERE id @1 AND name LIKE @2", [
327 | %Tds.Parameter{name: "@1", value: user_id},
328 | %Tds.Parameter{name: "@2", value: search <> "%"}
329 | ])\
330 | '''
331 | end
332 |
333 | test "ignores parameters inside comments" do
334 | attrs = %{
335 | "connection" => %{"variable" => "conn", "type" => "postgres"},
336 | "result_variable" => "result",
337 | "cache_query" => true,
338 | "timeout" => nil,
339 | "query" => """
340 | SELECT id from users
341 | -- WHERE id = {{user_id1}}
342 | /* WHERE id = {{user_id2}} */ WHERE id = {{user_id3}}\
343 | """,
344 | "data_frame_alias" => Explorer.DataFrame
345 | }
346 |
347 | assert SQLCell.to_source(attrs) == ~s'''
348 | result =
349 | Postgrex.query!(
350 | conn,
351 | ~S"""
352 | SELECT id from users
353 | -- WHERE id = {{user_id1}}
354 | /* WHERE id = {{user_id2}} */ WHERE id = $1
355 | """,
356 | [user_id3]
357 | )\
358 | '''
359 |
360 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == ~s'''
361 | result =
362 | MyXQL.query!(
363 | conn,
364 | ~S"""
365 | SELECT id from users
366 | -- WHERE id = {{user_id1}}
367 | /* WHERE id = {{user_id2}} */ WHERE id = ?
368 | """,
369 | [user_id3]
370 | )\
371 | '''
372 |
373 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == ~s'''
374 | result =
375 | Exqlite.query!(
376 | conn,
377 | ~S"""
378 | SELECT id from users
379 | -- WHERE id = {{user_id1}}
380 | /* WHERE id = {{user_id2}} */ WHERE id = ?1
381 | """,
382 | [user_id3]
383 | )\
384 | '''
385 |
386 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == ~s'''
387 | result =
388 | Adbc.Connection.query!(
389 | conn,
390 | ~S"""
391 | SELECT id from users
392 | -- WHERE id = {{user_id1}}
393 | /* WHERE id = {{user_id2}} */ WHERE id = ?
394 | """,
395 | [user_id3]
396 | )\
397 | '''
398 |
399 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "athena")) == ~s'''
400 | result =
401 | ReqAthena.query!(
402 | conn,
403 | ~S"""
404 | SELECT id from users
405 | -- WHERE id = {{user_id1}}
406 | /* WHERE id = {{user_id2}} */ WHERE id = ?
407 | """,
408 | [user_id3],
409 | format: :explorer
410 | ).body
411 |
412 | Kino.DataTable.new(result)\
413 | '''
414 |
415 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == ~s'''
416 | result =
417 | Adbc.Connection.query!(
418 | conn,
419 | ~S"""
420 | SELECT id from users
421 | -- WHERE id = {{user_id1}}
422 | /* WHERE id = {{user_id2}} */ WHERE id = ?1
423 | """,
424 | [user_id3]
425 | )\
426 | '''
427 |
428 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == ~s'''
429 | result =
430 | ReqCH.query!(
431 | conn,
432 | ~S"""
433 | SELECT id from users
434 | -- WHERE id = {{user_id1}}
435 | /* WHERE id = {{user_id2}} */ WHERE id = {user_id3:String}
436 | """,
437 | [{"user_id3", user_id3}],
438 | format: :explorer
439 | ).body
440 |
441 | Kino.DataTable.new(result)\
442 | '''
443 |
444 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == ~s'''
445 | result =
446 | Tds.query!(
447 | conn,
448 | ~S"""
449 | SELECT id from users
450 | -- WHERE id = {{user_id1}}
451 | /* WHERE id = {{user_id2}} */ WHERE id = @1
452 | """,
453 | [%Tds.Parameter{name: "@1", value: user_id3}]
454 | )\
455 | '''
456 | end
457 |
458 | test "passes timeout option when a timeout is specified" do
459 | attrs = %{
460 | "connection" => %{"variable" => "conn", "type" => "postgres"},
461 | "result_variable" => "result",
462 | "cache_query" => true,
463 | "timeout" => 30,
464 | "query" => "SELECT id FROM users",
465 | "data_frame_alias" => Explorer.DataFrame
466 | }
467 |
468 | assert SQLCell.to_source(attrs) == """
469 | result = Postgrex.query!(conn, ~S"SELECT id FROM users", [], timeout: 30000)\
470 | """
471 |
472 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == """
473 | result = MyXQL.query!(conn, ~S"SELECT id FROM users", [], timeout: 30000)\
474 | """
475 |
476 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == """
477 | result = Exqlite.query!(conn, ~S"SELECT id FROM users", [], timeout: 30000)\
478 | """
479 |
480 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == """
481 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
482 | """
483 |
484 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "athena")) == """
485 | result = ReqAthena.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
486 | Kino.DataTable.new(result)\
487 | """
488 |
489 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == """
490 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
491 | """
492 |
493 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == """
494 | result = ReqCH.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
495 | Kino.DataTable.new(result)\
496 | """
497 |
498 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == """
499 | result = Tds.query!(conn, ~S"SELECT id FROM users", [], timeout: 30000)\
500 | """
501 | end
502 |
503 | test "passes cache_query option when supported" do
504 | attrs = %{
505 | "connection" => %{"variable" => "conn", "type" => "postgres"},
506 | "result_variable" => "result",
507 | "cache_query" => true,
508 | "query" => "SELECT id FROM users"
509 | }
510 |
511 | assert SQLCell.to_source(attrs) == """
512 | result = Postgrex.query!(conn, ~S"SELECT id FROM users", [])\
513 | """
514 |
515 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "mysql")) == """
516 | result = MyXQL.query!(conn, ~S"SELECT id FROM users", [])\
517 | """
518 |
519 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlite")) == """
520 | result = Exqlite.query!(conn, ~S"SELECT id FROM users", [])\
521 | """
522 |
523 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "snowflake")) == """
524 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
525 | """
526 |
527 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "clickhouse")) == """
528 | result = ReqCH.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
529 | Kino.DataTable.new(result)\
530 | """
531 |
532 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "bigquery")) == """
533 | result = Adbc.Connection.query!(conn, ~S"SELECT id FROM users", [])\
534 | """
535 |
536 | athena = put_in(attrs["connection"]["type"], "athena")
537 |
538 | assert SQLCell.to_source(put_in(athena["cache_query"], true)) == """
539 | result = ReqAthena.query!(conn, ~S"SELECT id FROM users", [], format: :explorer).body
540 | Kino.DataTable.new(result)\
541 | """
542 |
543 | assert SQLCell.to_source(put_in(athena["cache_query"], false)) == """
544 | result =
545 | ReqAthena.query!(conn, ~S"SELECT id FROM users", [],
546 | format: :explorer,
547 | cache_query: false
548 | ).body
549 |
550 | Kino.DataTable.new(result)\
551 | """
552 |
553 | assert SQLCell.to_source(put_in(attrs["connection"]["type"], "sqlserver")) == """
554 | result = Tds.query!(conn, ~S"SELECT id FROM users", [])\
555 | """
556 | end
557 |
558 | test "escapes interpolation" do
559 | attrs = %{
560 | "connection" => %{"variable" => "conn", "type" => "postgres"},
561 | "result_variable" => "result",
562 | "cache_query" => true,
563 | "timeout" => nil,
564 | "query" => "SELECT id FROM users WHERE last_name = '\#{user_id}'",
565 | "data_frame_alias" => Explorer.DataFrame
566 | }
567 |
568 | assert SQLCell.to_source(attrs) == """
569 | result =
570 | Postgrex.query!(conn, ~S"SELECT id FROM users WHERE last_name = '\#{user_id}'", [])\
571 | """
572 |
573 | athena = put_in(attrs["query"], "SELECT id FROM users\nWHERE last_name = '\#{user_id}'")
574 |
575 | assert SQLCell.to_source(put_in(athena["cache_query"], true)) == ~s'''
576 | result =
577 | Postgrex.query!(
578 | conn,
579 | ~S"""
580 | SELECT id FROM users
581 | WHERE last_name = '\#{user_id}'
582 | """,
583 | []
584 | )\
585 | '''
586 | end
587 | end
588 |
589 | defp spawn_fake_postgrex_connection() do
590 | parent = self()
591 |
592 | conn =
593 | spawn_link(fn ->
594 | # Pretend we are a connection pool for Postgrex
595 | DBConnection.register_as_pool(Postgrex.Protocol)
596 | send(parent, {:ready, self()})
597 | receive do: (:stop -> :ok)
598 | end)
599 |
600 | on_exit(fn ->
601 | send(conn, :stop)
602 | end)
603 |
604 | receive do: ({:ready, ^conn} -> conn)
605 | end
606 | end
607 |
--------------------------------------------------------------------------------
/lib/kino_db/connection_cell.ex:
--------------------------------------------------------------------------------
1 | defmodule KinoDB.ConnectionCell do
2 | @moduledoc false
3 |
4 | # A smart cell used to establish connection to a database.
5 |
6 | use Kino.JS, assets_path: "lib/assets/connection_cell"
7 | use Kino.JS.Live
8 | use Kino.SmartCell, name: "Database connection"
9 |
10 | @default_port_by_type %{"postgres" => 5432, "mysql" => 3306, "sqlserver" => 1433}
11 |
12 | @impl true
13 | def init(attrs, ctx) do
14 | type = attrs["type"] || default_db_type()
15 | default_port = @default_port_by_type[type]
16 |
17 | password = attrs["password"] || ""
18 | secret_access_key = attrs["secret_access_key"] || ""
19 | priv_key = attrs["private_key"] || ""
20 | priv_key_passphrase = attrs["private_key_passphrase"] || ""
21 |
22 | auth_type = attrs["auth_type"] || ""
23 |
24 | fields = %{
25 | "variable" => Kino.SmartCell.prefixed_var_name("conn", attrs["variable"]),
26 | "type" => type,
27 | "hostname" => attrs["hostname"] || "localhost",
28 | "database_path" => attrs["database_path"] || "",
29 | "port" => attrs["port"] || default_port,
30 | "use_ipv6" => Map.get(attrs, "use_ipv6", false),
31 | "use_ssl" => Map.get(attrs, "use_ssl", false),
32 | "cacertfile" => attrs["cacertfile"] || "",
33 | "auth_type" => auth_type,
34 | "username" => attrs["username"] || "",
35 | "password" => password,
36 | "use_password_secret" => Map.has_key?(attrs, "password_secret") || password == "",
37 | "password_secret" => attrs["password_secret"] || "",
38 | "database" => attrs["database"] || "",
39 | "project_id" => attrs["project_id"] || "",
40 | "default_dataset_id" => attrs["default_dataset_id"] || "",
41 | "credentials_json" => attrs["credentials_json"] || "",
42 | "access_key_id" => attrs["access_key_id"] || "",
43 | "secret_access_key" => secret_access_key,
44 | "use_secret_access_key_secret" =>
45 | Map.has_key?(attrs, "secret_access_key_secret") || secret_access_key == "",
46 | "secret_access_key_secret" => attrs["secret_access_key_secret"] || "",
47 | "private_key" => priv_key,
48 | "use_private_key_secret" => Map.has_key?(attrs, "use_private_key_secret") || priv_key == "",
49 | "private_key_secret" => attrs["private_key_secret"] || "",
50 | "use_encrypted_private_key" => Map.get(attrs, "use_encrypted_private_key", false),
51 | "private_key_passphrase" => priv_key_passphrase,
52 | "use_private_key_passphrase_secret" =>
53 | Map.has_key?(attrs, "use_private_key_passphrase_secret") || priv_key_passphrase == "",
54 | "private_key_passphrase_secret" => attrs["private_key_passphrase_secret"] || "",
55 | "token" => attrs["token"] || "",
56 | "region" => attrs["region"] || "us-east-1",
57 | "workgroup" => attrs["workgroup"] || "",
58 | "output_location" => attrs["output_location"] || "",
59 | "account" => attrs["account"] || "",
60 | "schema" => attrs["schema"] || "",
61 | "warehouse" => attrs["warehouse"] || "",
62 | "instance" => attrs["instance"] || ""
63 | }
64 |
65 | ctx =
66 | assign(ctx,
67 | fields: fields,
68 | missing_dep: missing_dep(fields),
69 | help_box: help_box(fields),
70 | has_aws_credentials: Code.ensure_loaded?(:aws_credentials)
71 | )
72 |
73 | {:ok, ctx}
74 | end
75 |
76 | @impl true
77 | def handle_connect(ctx) do
78 | payload = %{
79 | fields: ctx.assigns.fields,
80 | missing_dep: ctx.assigns.missing_dep,
81 | help_box: ctx.assigns.help_box,
82 | has_aws_credentials: ctx.assigns.has_aws_credentials
83 | }
84 |
85 | {:ok, payload, ctx}
86 | end
87 |
88 | @impl true
89 | def handle_event("update_field", %{"field" => field, "value" => value}, ctx) do
90 | updated_fields = to_updates(ctx.assigns.fields, field, value)
91 | ctx = update(ctx, :fields, &Map.merge(&1, updated_fields))
92 |
93 | missing_dep = missing_dep(ctx.assigns.fields)
94 |
95 | ctx =
96 | if missing_dep == ctx.assigns.missing_dep do
97 | ctx
98 | else
99 | broadcast_event(ctx, "missing_dep", %{"dep" => missing_dep})
100 | assign(ctx, missing_dep: missing_dep)
101 | end
102 |
103 | broadcast_event(ctx, "update", %{"fields" => updated_fields})
104 |
105 | {:noreply, ctx}
106 | end
107 |
108 | defp to_updates(_fields, "port", value) do
109 | port =
110 | case Integer.parse(value) do
111 | {n, ""} -> n
112 | _ -> nil
113 | end
114 |
115 | %{"port" => port}
116 | end
117 |
118 | defp to_updates(_fields, "type", value) do
119 | %{"type" => value, "port" => @default_port_by_type[value]}
120 | end
121 |
122 | defp to_updates(fields, "variable", value) do
123 | if Kino.SmartCell.valid_variable_name?(value) do
124 | %{"variable" => value}
125 | else
126 | %{"variable" => fields["variable"]}
127 | end
128 | end
129 |
130 | defp to_updates(_fields, field, value), do: %{field => value}
131 |
132 | @default_keys ["type", "variable"]
133 |
134 | @impl true
135 | def to_attrs(%{assigns: %{fields: fields}}) do
136 | connection_keys =
137 | case fields["type"] do
138 | "sqlite" ->
139 | ~w|database_path|
140 |
141 | "duckdb" ->
142 | ~w|database_path|
143 |
144 | "bigquery" ->
145 | ~w|project_id default_dataset_id credentials_json|
146 |
147 | "athena" ->
148 | if fields["use_secret_access_key_secret"],
149 | do:
150 | ~w|access_key_id secret_access_key_secret token region workgroup output_location database|,
151 | else:
152 | ~w|access_key_id secret_access_key token region workgroup output_location database|
153 |
154 | "snowflake" ->
155 | snowflake_fields(fields)
156 |
157 | "sqlserver" ->
158 | if fields["use_password_secret"],
159 | do:
160 | ~w|database hostname port use_ipv6 username password_secret use_ssl cacertfile instance|,
161 | else:
162 | ~w|database hostname port use_ipv6 username password use_ssl cacertfile instance|
163 |
164 | "clickhouse" ->
165 | if fields["use_password_secret"],
166 | do: ~w|hostname port use_ssl username password_secret database|,
167 | else: ~w|hostname port use_ssl username password database|
168 |
169 | type when type in ["postgres", "mysql"] ->
170 | if fields["use_password_secret"],
171 | do: ~w|database hostname port use_ipv6 use_ssl cacertfile username password_secret|,
172 | else: ~w|database hostname port use_ipv6 use_ssl cacertfile username password|
173 | end
174 |
175 | Map.take(fields, @default_keys ++ connection_keys)
176 | end
177 |
178 | @default_snowflake_keys ~w|account username database schema warehouse auth_type|
179 | defp snowflake_fields(fields),
180 | do: snowflake_fields(fields["auth_type"], fields, @default_snowflake_keys)
181 |
182 | defp snowflake_fields("auth_jwt", fields, keys) do
183 | pk_keys =
184 | if fields["use_private_key_secret"],
185 | do: ~w|private_key_secret use_encrypted_private_key|,
186 | else: ~w|private_key use_encrypted_private_key|
187 |
188 | phrase_keys =
189 | if fields["use_encrypted_private_key"] do
190 | if fields["use_private_key_passphrase_secret"],
191 | do: ~w|private_key_passphrase_secret|,
192 | else: ~w|private_key_passphrase|
193 | else
194 | []
195 | end
196 |
197 | keys ++ pk_keys ++ phrase_keys
198 | end
199 |
200 | defp snowflake_fields(_, fields, keys) do
201 | keys ++
202 | if fields["use_password_secret"], do: ~w|password_secret|, else: ~w|password|
203 | end
204 |
205 | @impl true
206 | def to_source(attrs) do
207 | required_keys =
208 | case attrs["type"] do
209 | "sqlite" ->
210 | ~w|database_path|
211 |
212 | "duckdb" ->
213 | []
214 |
215 | "bigquery" ->
216 | ~w|project_id|
217 |
218 | "athena" ->
219 | if Code.ensure_loaded?(:aws_credentials),
220 | do: ~w|database|,
221 | else:
222 | if(Map.has_key?(attrs, "secret_access_key"),
223 | do: ~w|access_key_id secret_access_key region database|,
224 | else: ~w|access_key_id secret_access_key_secret region database|
225 | )
226 |
227 | "snowflake" ->
228 | snowflake_source_required_keys(attrs)
229 |
230 | "sqlserver" ->
231 | ~w|hostname port|
232 |
233 | "clickhouse" ->
234 | ~w|hostname port|
235 |
236 | type when type in ["postgres", "mysql"] ->
237 | ~w|hostname port|
238 | end
239 |
240 | conditional_keys =
241 | case attrs["type"] do
242 | "athena" -> ~w|workgroup output_location|
243 | _ -> []
244 | end
245 |
246 | if all_fields_filled?(attrs, required_keys) and
247 | any_fields_filled?(attrs, conditional_keys) do
248 | attrs |> to_quoted() |> Kino.SmartCell.quoted_to_string()
249 | else
250 | ""
251 | end
252 | end
253 |
254 | defp snowflake_source_required_keys(attrs),
255 | do: snowflake_source_required_keys(attrs["auth_type"], attrs, ~w|account username auth_type|)
256 |
257 | defp snowflake_source_required_keys("auth_jwt", attrs, keys) do
258 | keys ++
259 | if Map.has_key?(attrs, "private_key_secret"),
260 | do: ~w|private_key_secret|,
261 | else: ~w|private_key|
262 | end
263 |
264 | defp snowflake_source_required_keys(_, attrs, keys) do
265 | keys ++ if Map.has_key?(attrs, "password_secret"), do: ~w|password_secret|, else: ~w|password|
266 | end
267 |
268 | defp all_fields_filled?(attrs, keys) do
269 | not Enum.any?(keys, fn key -> attrs[key] in [nil, ""] end)
270 | end
271 |
272 | defp any_fields_filled?(_, []), do: true
273 |
274 | defp any_fields_filled?(attrs, keys) do
275 | Enum.any?(keys, fn key -> attrs[key] not in [nil, ""] end)
276 | end
277 |
278 | defp trim_opts(opts) do
279 | Enum.map(opts, fn
280 | {key, value} when is_binary(value) -> {key, String.trim(value)}
281 | {key, value} -> {key, value}
282 | end)
283 | end
284 |
285 | defp to_quoted(%{"type" => "sqlite"} = attrs) do
286 | quote do
287 | opts = [database: unquote(attrs["database_path"])]
288 |
289 | {:ok, unquote(quoted_var(attrs["variable"]))} = Kino.start_child({Exqlite, opts})
290 | end
291 | end
292 |
293 | defp to_quoted(%{"type" => "snowflake"} = attrs) do
294 | var = quoted_var(attrs["variable"])
295 |
296 | snowflake_opts =
297 | attrs
298 | |> trim_opts()
299 | |> Map.new()
300 | |> snowflake_options()
301 |
302 | quote do
303 | :ok = Adbc.download_driver!(:snowflake)
304 | {:ok, db} = Kino.start_child({Adbc.Database, unquote(snowflake_opts)})
305 | {:ok, unquote(var)} = Kino.start_child({Adbc.Connection, database: db})
306 | end
307 | end
308 |
309 | defp to_quoted(%{"type" => "duckdb"} = attrs) do
310 | var = quoted_var(attrs["variable"])
311 |
312 | opts =
313 | case String.trim(attrs["database_path"]) do
314 | "" -> [driver: :duckdb]
315 | path -> [driver: :duckdb, path: path]
316 | end
317 |
318 | quote do
319 | :ok = Adbc.download_driver!(:duckdb)
320 | {:ok, db} = Kino.start_child({Adbc.Database, unquote(opts)})
321 | {:ok, unquote(var)} = Kino.start_child({Adbc.Connection, database: db})
322 | end
323 | end
324 |
325 | defp to_quoted(%{"type" => "bigquery"} = attrs) do
326 | var = quoted_var(attrs["variable"])
327 |
328 | opts =
329 | [
330 | driver: :bigquery,
331 | "adbc.bigquery.sql.project_id": attrs["project_id"]
332 | ] ++
333 | case attrs["default_dataset_id"] do
334 | "" -> []
335 | dataset_id -> ["adbc.bigquery.sql.dataset_id": dataset_id]
336 | end ++
337 | case attrs["credentials_json"] do
338 | "" ->
339 | []
340 |
341 | credentials_json ->
342 | [
343 | "adbc.bigquery.sql.auth_type": "adbc.bigquery.sql.auth_type.json_credential_string",
344 | "adbc.bigquery.sql.auth_credentials":
345 | {:sigil_S, [delimiter: ~s["""]], [{:<<>>, [], [credentials_json <> "\n"]}, []]}
346 | ]
347 | end
348 |
349 | quote do
350 | :ok = Adbc.download_driver!(:bigquery)
351 | {:ok, db} = Kino.start_child({Adbc.Database, unquote(opts)})
352 | {:ok, unquote(var)} = Kino.start_child({Adbc.Connection, database: db})
353 | end
354 | end
355 |
356 | defp to_quoted(%{"type" => "postgres"} = attrs) do
357 | quote do
358 | opts = unquote(trim_opts(shared_options(attrs) ++ postgres_and_mysql_options(attrs)))
359 |
360 | {:ok, unquote(quoted_var(attrs["variable"]))} = Kino.start_child({Postgrex, opts})
361 | end
362 | end
363 |
364 | defp to_quoted(%{"type" => "mysql"} = attrs) do
365 | quote do
366 | opts = unquote(trim_opts(shared_options(attrs) ++ postgres_and_mysql_options(attrs)))
367 |
368 | {:ok, unquote(quoted_var(attrs["variable"]))} = Kino.start_child({MyXQL, opts})
369 | end
370 | end
371 |
372 | defp to_quoted(%{"type" => "sqlserver"} = attrs) do
373 | quote do
374 | opts = unquote(trim_opts(shared_options(attrs) ++ sqlserver_options(attrs)))
375 |
376 | {:ok, unquote(quoted_var(attrs["variable"]))} = Kino.start_child({Tds, opts})
377 | end
378 | end
379 |
380 | defp to_quoted(%{"type" => "athena"} = attrs) do
381 | quote do
382 | unquote(quoted_var(attrs["variable"])) =
383 | ReqAthena.new(
384 | access_key_id: unquote(attrs["access_key_id"]),
385 | database: unquote(attrs["database"]),
386 | output_location: unquote(attrs["output_location"]),
387 | region: unquote(attrs["region"]),
388 | secret_access_key: unquote(quoted_access_key(attrs)),
389 | token: unquote(attrs["token"]),
390 | workgroup: unquote(attrs["workgroup"]),
391 | http_errors: :raise
392 | )
393 |
394 | :ok
395 | end
396 | end
397 |
398 | defp to_quoted(%{"type" => "clickhouse"} = attrs) do
399 | trimmed = attrs |> trim_opts() |> Map.new()
400 | shared_opts = shared_options(trimmed)
401 |
402 | clickhouse_opts = trimmed |> clickhouse_options(shared_opts)
403 |
404 | quote do
405 | unquote(quoted_var(attrs["variable"])) = ReqCH.new(unquote(clickhouse_opts))
406 |
407 | :ok
408 | end
409 | end
410 |
411 | defp quoted_access_key(%{"secret_access_key" => password}), do: password
412 |
413 | defp quoted_access_key(%{"secret_access_key_secret" => ""}), do: ""
414 |
415 | defp quoted_access_key(%{"secret_access_key_secret" => secret}) do
416 | quote do
417 | System.fetch_env!(unquote("LB_#{secret}"))
418 | end
419 | end
420 |
421 | defp quoted_private_key(%{"private_key" => pk}), do: pk
422 |
423 | defp quoted_private_key(%{"private_key_secret" => ""}), do: ""
424 |
425 | defp quoted_private_key(%{"private_key_secret" => secret}) do
426 | quote do
427 | System.fetch_env!(unquote("LB_#{secret}"))
428 | end
429 | end
430 |
431 | defp quoted_private_key_passphrase(%{"private_key_passphrase" => phrase}), do: phrase
432 |
433 | defp quoted_private_key_passphrase(%{"private_key_passphrase_secret" => ""}), do: ""
434 |
435 | defp quoted_private_key_passphrase(%{"private_key_passphrase_secret" => secret}) do
436 | quote do
437 | System.fetch_env!(unquote("LB_#{secret}"))
438 | end
439 | end
440 |
441 | defp snowflake_options(attrs) do
442 | shared_opts =
443 | [
444 | driver: :snowflake,
445 | username: attrs["username"],
446 | "adbc.snowflake.sql.account": attrs["account"],
447 | "adbc.snowflake.sql.db": attrs["database"],
448 | "adbc.snowflake.sql.schema": attrs["schema"],
449 | "adbc.snowflake.sql.warehouse": attrs["warehouse"]
450 | ]
451 |
452 | snowflake_auth_opts(attrs["auth_type"], attrs, shared_opts)
453 | end
454 |
455 | defp snowflake_auth_opts("auth_jwt", attrs, opts) do
456 | opts ++
457 | [
458 | "adbc.snowflake.sql.auth_type": "auth_jwt",
459 | "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_value": quoted_private_key(attrs)
460 | ] ++
461 | if attrs["use_encrypted_private_key"] do
462 | [
463 | "adbc.snowflake.sql.client_option.jwt_private_key_pkcs8_password":
464 | quoted_private_key_passphrase(attrs)
465 | ]
466 | else
467 | []
468 | end
469 | end
470 |
471 | defp snowflake_auth_opts(_, attrs, opts) do
472 | opts ++
473 | [
474 | "adbc.snowflake.sql.auth_type": "auth_snowflake",
475 | password: quoted_pass(attrs)
476 | ]
477 | end
478 |
479 | defp shared_options(attrs) do
480 | opts = [
481 | hostname: attrs["hostname"],
482 | port: attrs["port"],
483 | username: attrs["username"],
484 | password: quoted_pass(attrs),
485 | database: attrs["database"]
486 | ]
487 |
488 | if attrs["use_ipv6"] do
489 | opts ++ [socket_options: [:inet6]]
490 | else
491 | opts
492 | end
493 | end
494 |
495 | defp postgres_and_mysql_options(attrs) do
496 | if attrs["use_ssl"] do
497 | cacertfile = attrs["cacertfile"]
498 |
499 | ssl_opts =
500 | if cacertfile && cacertfile != "" do
501 | [cacertfile: cacertfile]
502 | else
503 | [cacerts: quote(do: :public_key.cacerts_get())]
504 | end
505 |
506 | [ssl: ssl_opts]
507 | else
508 | []
509 | end
510 | end
511 |
512 | defp sqlserver_options(attrs) do
513 | opts =
514 | if attrs["use_ssl"] do
515 | cacertfile = attrs["cacertfile"]
516 |
517 | ssl_opts =
518 | if cacertfile && cacertfile != "" do
519 | [cacertfile: cacertfile]
520 | else
521 | [cacerts: quote(do: :public_key.cacerts_get())]
522 | end
523 |
524 | [ssl: true, ssl_opts: ssl_opts]
525 | else
526 | []
527 | end
528 |
529 | instance = attrs["instance"]
530 |
531 | if instance && instance != "" do
532 | opts ++ [instance: instance]
533 | else
534 | opts
535 | end
536 | end
537 |
538 | defp clickhouse_options(attrs) do
539 | scheme = if attrs["use_ssl"], do: "https", else: "http"
540 |
541 | [scheme: scheme]
542 | end
543 |
544 | defp clickhouse_options(attrs, shared_options) do
545 | attrs
546 | |> clickhouse_options()
547 | |> build_clickhouse_base_url(shared_options)
548 | |> maybe_add_req_basic_auth(shared_options)
549 | |> maybe_add_clickhouse_database(shared_options)
550 | end
551 |
552 | defp build_clickhouse_base_url(opts, shared_opts) do
553 | host = Keyword.fetch!(shared_opts, :hostname)
554 | port = Keyword.fetch!(shared_opts, :port)
555 | scheme = Keyword.fetch!(opts, :scheme)
556 |
557 | uri = %URI{scheme: scheme, port: port, host: host}
558 |
559 | opts
560 | |> Keyword.put_new(:base_url, URI.to_string(uri))
561 | |> Keyword.delete(:scheme)
562 | end
563 |
564 | defp maybe_add_req_basic_auth(opts, shared_opts) do
565 | username = shared_opts[:username]
566 |
567 | if username != "" do
568 | password = shared_opts[:password]
569 |
570 | auth =
571 | if is_binary(password) do
572 | "#{username}:#{password}"
573 | else
574 | quote do
575 | unquote(username) <> ":" <> unquote(password)
576 | end
577 | end
578 |
579 | Keyword.put_new(opts, :auth, {:basic, auth})
580 | else
581 | opts
582 | end
583 | end
584 |
585 | defp maybe_add_clickhouse_database(opts, shared_opts) do
586 | if shared_opts[:database] != "" do
587 | Keyword.put_new(opts, :database, shared_opts[:database])
588 | else
589 | opts
590 | end
591 | end
592 |
593 | defp quoted_var(string), do: {String.to_atom(string), [], nil}
594 |
595 | defp quoted_pass(%{"password" => password}), do: password
596 |
597 | defp quoted_pass(%{"password_secret" => ""}), do: ""
598 |
599 | defp quoted_pass(%{"password_secret" => secret}) do
600 | quote do
601 | System.fetch_env!(unquote("LB_#{secret}"))
602 | end
603 | end
604 |
605 | defp default_db_type() do
606 | cond do
607 | Code.ensure_loaded?(Postgrex) -> "postgres"
608 | Code.ensure_loaded?(MyXQL) -> "mysql"
609 | Code.ensure_loaded?(Exqlite) -> "sqlite"
610 | Code.ensure_loaded?(ReqAthena) -> "athena"
611 | Code.ensure_loaded?(ReqCH) -> "clickhouse"
612 | Code.ensure_loaded?(Adbc) -> adbc_default_db_type()
613 | Code.ensure_loaded?(Tds) -> "sqlserver"
614 | true -> "postgres"
615 | end
616 | end
617 |
618 | defp adbc_default_db_type() do
619 | drivers = Application.get_env(:adbc, :drivers, [])
620 | driver = Enum.find([:duckdb, :snowflake, :bigquery], :duckdb, &(&1 in drivers))
621 | Atom.to_string(driver)
622 | end
623 |
624 | defp missing_dep(%{"type" => "postgres"}) do
625 | unless Code.ensure_loaded?(Postgrex) do
626 | ~s/{:postgrex, "~> 0.18"}/
627 | end
628 | end
629 |
630 | defp missing_dep(%{"type" => "mysql"}) do
631 | unless Code.ensure_loaded?(MyXQL) do
632 | ~s/{:myxql, "~> 0.7"}/
633 | end
634 | end
635 |
636 | defp missing_dep(%{"type" => "sqlite"}) do
637 | unless Code.ensure_loaded?(Exqlite) do
638 | ~s/{:exqlite, "~> 0.11"}/
639 | end
640 | end
641 |
642 | defp missing_dep(%{"type" => "athena"}) do
643 | missing_many_deps([
644 | {ReqAthena, ~s|{:req_athena, "~> 0.3"}|},
645 | {Explorer, ~s|{:explorer, "~> 0.10"}|}
646 | ])
647 | end
648 |
649 | defp missing_dep(%{"type" => adbc}) when adbc in ~w[duckdb snowflake bigquery] do
650 | unless Code.ensure_loaded?(Adbc) do
651 | ~s|{:adbc, "~> 0.3"}|
652 | end
653 | end
654 |
655 | defp missing_dep(%{"type" => "sqlserver"}) do
656 | unless Code.ensure_loaded?(Tds) do
657 | ~s|{:tds, "~> 2.3"}|
658 | end
659 | end
660 |
661 | defp missing_dep(%{"type" => "clickhouse"}) do
662 | missing_many_deps([
663 | {ReqCH, ~s|{:req_ch, "~> 0.1"}|},
664 | {Explorer, ~s|{:explorer, "~> 0.10"}|}
665 | ])
666 | end
667 |
668 | defp missing_dep(_ctx), do: nil
669 |
670 | defp missing_many_deps(deps) do
671 | deps = for {module, dep} <- deps, not Code.ensure_loaded?(module), do: dep
672 |
673 | if deps != [] do
674 | Enum.join(deps, ", ")
675 | end
676 | end
677 |
678 | defp help_box(%{"type" => "bigquery"}) do
679 | if Code.ensure_loaded?(Mint.HTTP) do
680 | if running_on_google_metadata?() do
681 | "You are running inside Google Cloud. Uploading the credentials above is optional."
682 | else
683 | ~s|You must upload your Google BigQuery Credentials (find them here) or authenticate your machine with gcloud CLI authentication.|
684 | end
685 | end
686 | end
687 |
688 | defp help_box(%{"type" => "athena"}) do
689 | if Code.ensure_loaded?(:aws_credentials) do
690 | "You must fill in the fields above accordingly or authenticate your machine with AWS CLI authentication."
691 | end
692 | end
693 |
694 | defp help_box(_ctx), do: nil
695 |
696 | defp running_on_google_metadata? do
697 | with {:ok, conn} <- Mint.HTTP.connect(:http, "metadata.google.internal", 80),
698 | {:ok, _} <- Mint.HTTP.set_mode(conn, :passive),
699 | do: true,
700 | else: (_ -> false)
701 | end
702 | end
703 |
--------------------------------------------------------------------------------
/lib/assets/connection_cell/main.js:
--------------------------------------------------------------------------------
1 | import * as Vue from "https://cdn.jsdelivr.net/npm/vue@3.2.26/dist/vue.esm-browser.prod.js";
2 |
3 | export function init(ctx, info) {
4 | ctx.importCSS("main.css");
5 | ctx.importCSS(
6 | "https://fonts.googleapis.com/css2?family=Inter:wght@400;500&display=swap"
7 | );
8 |
9 | const BaseSelect = {
10 | name: "BaseSelect",
11 |
12 | props: {
13 | label: {
14 | type: String,
15 | default: "",
16 | },
17 | selectClass: {
18 | type: String,
19 | default: "input",
20 | },
21 | modelValue: {
22 | type: String,
23 | default: "",
24 | },
25 | options: {
26 | type: Array,
27 | default: [],
28 | required: true,
29 | },
30 | required: {
31 | type: Boolean,
32 | default: false,
33 | },
34 | inline: {
35 | type: Boolean,
36 | default: false,
37 | },
38 | grow: {
39 | type: Boolean,
40 | default: false,
41 | },
42 | },
43 |
44 | methods: {
45 | available(value, options) {
46 | return value
47 | ? options.map((option) => option.value).includes(value)
48 | : true;
49 | },
50 | },
51 |
52 | template: `
53 |
54 |
57 |
76 |
77 | `,
78 | };
79 |
80 | const BaseInput = {
81 | name: "BaseInput",
82 |
83 | props: {
84 | label: {
85 | type: String,
86 | default: "",
87 | },
88 | inputClass: {
89 | type: String,
90 | default: "input",
91 | },
92 | modelValue: {
93 | type: [String, Number],
94 | default: "",
95 | },
96 | inline: {
97 | type: Boolean,
98 | default: false,
99 | },
100 | grow: {
101 | type: Boolean,
102 | default: false,
103 | },
104 | number: {
105 | type: Boolean,
106 | default: false,
107 | },
108 | },
109 |
110 | computed: {
111 | emptyClass() {
112 | if (this.modelValue === "") {
113 | return "empty";
114 | }
115 | },
116 | },
117 |
118 | template: `
119 |
120 |
123 |
129 |
130 | `,
131 | };
132 |
133 | const BaseSwitch = {
134 | name: "BaseSwitch",
135 |
136 | props: {
137 | label: {
138 | type: String,
139 | default: "",
140 | },
141 | modelValue: {
142 | type: Boolean,
143 | default: true,
144 | },
145 | inline: {
146 | type: Boolean,
147 | default: false,
148 | },
149 | grow: {
150 | type: Boolean,
151 | default: false,
152 | },
153 | },
154 |
155 | template: `
156 |
157 |
160 |
173 |
307 |
315 |
316 | `,
317 | };
318 |
319 | const DuckDBForm = {
320 | name: "DuckDBForm",
321 |
322 | components: {
323 | BaseInput: BaseInput,
324 | },
325 |
326 | props: {
327 | fields: {
328 | type: Object,
329 | default: {},
330 | },
331 | },
332 |
333 | template: `
334 |
335 |
343 |
344 | `,
345 | };
346 |
347 | const DefaultSQLForm = {
348 | name: "DefaultSQLForm",
349 |
350 | components: {
351 | BaseInput: BaseInput,
352 | BaseSwitch: BaseSwitch,
353 | BaseSecret: BaseSecret,
354 | },
355 |
356 | props: {
357 | fields: {
358 | type: Object,
359 | default: {},
360 | },
361 | },
362 |
363 | template: `
364 |
365 |
374 |
383 |
388 |
393 |
394 |
395 |
404 |
405 |
406 |
414 |
422 |
432 |
433 | `,
434 | };
435 |
436 | const SQLServerForm = {
437 | name: "SQLServerForm",
438 |
439 | components: {
440 | BaseInput: BaseInput,
441 | BaseSwitch: BaseSwitch,
442 | BaseSecret: BaseSecret,
443 | },
444 |
445 | props: {
446 | fields: {
447 | type: Object,
448 | default: {},
449 | },
450 | execution_modes: {
451 | type: Object,
452 | default: {},
453 | },
454 | datefirst: {
455 | type: Object,
456 | default: {},
457 | },
458 | },
459 |
460 | template: `
461 |
462 |
471 |
479 |
488 |
493 |
498 |
499 |
500 |
509 |
510 |
511 |
519 |
527 |
537 |
538 | `,
539 | };
540 |
541 | const AthenaForm = {
542 | name: "AthenaForm",
543 |
544 | components: {
545 | BaseInput: BaseInput,
546 | BaseSecret: BaseSecret,
547 | },
548 |
549 | props: {
550 | fields: {
551 | type: Object,
552 | default: {},
553 | },
554 | helpBox: {
555 | type: String,
556 | default: "",
557 | },
558 | hasAwsCredentials: {
559 | type: Boolean,
560 | default: false,
561 | },
562 | },
563 |
564 | methods: {
565 | areFieldsEmpty(currentField, otherField) {
566 | if (currentField === "" && otherField === "") {
567 | return true;
568 | }
569 |
570 | return false;
571 | },
572 | },
573 |
574 | template: `
575 |
576 |
585 |
595 |
596 |
597 |
605 |
614 |
615 |
616 |
625 |
634 |
643 |
644 |
645 | `,
646 | };
647 |
648 | const BigQueryForm = {
649 | name: "BigQueryForm",
650 |
651 | components: {
652 | BaseInput: BaseInput,
653 | },
654 |
655 | props: {
656 | fields: {
657 | type: Object,
658 | default: {},
659 | },
660 | helpBox: {
661 | type: String,
662 | default: "",
663 | },
664 | },
665 |
666 | methods: {
667 | credentialsChange(_) {
668 | this.updateCredentials(this.$refs.credentials.files);
669 | },
670 |
671 | credentialsClick(_) {
672 | this.$refs.credentials.click();
673 | },
674 |
675 | dragOver(event) {
676 | event.preventDefault();
677 | },
678 |
679 | dragLeave(_) {},
680 |
681 | drop(event) {
682 | event.preventDefault();
683 | this.updateCredentials(event.dataTransfer.files);
684 | },
685 |
686 | updateCredentials(fileList) {
687 | const file = fileList[0];
688 |
689 | if (file && file.type === "application/json") {
690 | const reader = new FileReader();
691 |
692 | reader.onload = (res) => {
693 | // Reformat the JSON into a compact form
694 | const value = JSON.stringify(JSON.parse(res.target.result));
695 | ctx.pushEvent("update_field", { field: "credentials_json", value });
696 | };
697 |
698 | reader.readAsText(file);
699 | }
700 | },
701 | },
702 |
703 | template: `
704 |
705 |
714 |
722 |
723 |
724 |
725 |
729 |
730 |
731 |
765 |
774 |
783 |
784 |
785 |
793 |
801 |
809 |
810 |
811 |
822 |
834 |
840 |
841 |
842 |
854 |
865 |
866 | `,
867 | };
868 |
869 | const ClickhouseForm = {
870 | name: "ClickhouseForm",
871 |
872 | components: {
873 | BaseInput: BaseInput,
874 | BaseSwitch: BaseSwitch,
875 | BaseSecret: BaseSecret,
876 | BaseSelect: BaseSelect,
877 | },
878 |
879 | props: {
880 | fields: {
881 | type: Object,
882 | default: {},
883 | },
884 | },
885 |
886 | template: `
887 |
888 |
893 |
902 |
911 |
912 |
913 |
921 |
929 |
939 |
940 | `,
941 | };
942 |
943 | const app = Vue.createApp({
944 | components: {
945 | BaseInput: BaseInput,
946 | BaseSelect: BaseSelect,
947 | SQLiteForm: SQLiteForm,
948 | DuckDBForm: DuckDBForm,
949 | SnowflakeForm: SnowflakeForm,
950 | DefaultSQLForm: DefaultSQLForm,
951 | SQLServerForm: SQLServerForm,
952 | BigQueryForm: BigQueryForm,
953 | AthenaForm: AthenaForm,
954 | ClickhouseForm: ClickhouseForm,
955 | },
956 |
957 | template: `
958 |
959 |
960 |
961 |
To successfully connect, you need to add the following dependency:
962 |
{{ missingDep }}
963 |
964 |
996 |