├── .gitignore ├── README.md ├── build-and-copy ├── deps.edn ├── resources └── metabase-plugin.yaml └── src ├── .DS_Store └── metabase ├── .DS_Store └── driver ├── http.clj └── http ├── parameters.clj └── query_processor.clj /.gitignore: -------------------------------------------------------------------------------- 1 | \#*\# 2 | .\#* 3 | /target/ 4 | /.nrepl-port 5 | .cpcache 6 | .class 7 | 8 | # lsp: ignore all but the config file 9 | .lsp/* 10 | !.lsp/config.edn 11 | .clj-kondo/.cache 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTP Metabase Driver 2 | 3 | This is a proof-of-concept HTTP "driver" for [Metabase](https://metabase.com/). 4 | 5 | Previous discussion: https://github.com/metabase/metabase/pull/7047 6 | 7 | ## Usage 8 | 9 | Currently the simplest "native" query for this driver is simply an object with a `url` property: 10 | 11 | ```json 12 | { "url": "https://api.coinmarketcap.com/v1/ticker/" } 13 | ``` 14 | 15 | The driver will make a `GET` request and parse the resulting JSON array into rows. Currently it only supports JSON. 16 | 17 | You can provide a different `method` as well as `headers` and a JSON `body`: 18 | 19 | ```json 20 | { 21 | "url": "https://api.coinmarketcap.com/v1/ticker/", 22 | "method": "POST", 23 | "headers": { 24 | "Authentication": "SOMETOKEN" 25 | }, 26 | "body": { 27 | "foo": "bar" 28 | } 29 | } 30 | ``` 31 | 32 | Additionally, you can provide a `result` object with a JSONPath to the "root" in the response, and/or a list of `fields`: 33 | 34 | ```json 35 | { 36 | "url": "https://blockchain.info/blocks?format=json", 37 | "result": { 38 | "path": "blocks", 39 | "fields": ["height", "time"] 40 | } 41 | } 42 | ``` 43 | 44 | You can also predefine "tables" in the database configuration's `Table Definitions` setting. These tables will appear in the graphical query builder: 45 | 46 | ```json 47 | { 48 | "tables": [ 49 | { 50 | "name": "Blocks", 51 | "url": "https://blockchain.info/blocks?format=json", 52 | "fields": [ 53 | { "name": "height", "type": "number" }, 54 | { "name": "hash", "type": "string" }, 55 | { "name": "time", "type": "number" }, 56 | { "type": "boolean", "name": "main_chain" } 57 | ], 58 | "result": { 59 | "path": "blocks" 60 | } 61 | } 62 | ] 63 | } 64 | ``` 65 | 66 | There is limited support for aggregations and breakouts, but this is very experimental and may be removed in future versions. 67 | 68 | ## Building the driver 69 | 70 | ### Prereq: Install Metabase as a local maven dependency, compiled for building drivers 71 | 72 | Clone the [Metabase repo](https://github.com/metabase/metabase) first if you haven't already done so. 73 | 74 | ### Metabase 0.46.0 75 | 76 | - The process for building a driver has changed slightly in Metabase 0.46.0. Your build command should now look 77 | something like this: 78 | 79 | ```sh 80 | # Example for building the driver with bash or similar 81 | 82 | # switch to the local checkout of the Metabase repo 83 | cd /path/to/metabase/repo 84 | 85 | # get absolute path to the driver project directory 86 | DRIVER_PATH=`readlink -f ~/path/to/metabase-http-driver` 87 | 88 | # Build driver. See explanation in sample HTTP driver README 89 | clojure \ 90 | -Sdeps "{:aliases {:http {:extra-deps {com.metabase/http-driver {:local/root \"$DRIVER_PATH\"}}}}}" \ 91 | -X:build:http \ 92 | build-drivers.build-driver/build-driver! \ 93 | "{:driver :http, :project-dir \"$DRIVER_PATH\", :target-dir \"$DRIVER_PATH/target\"}" 94 | ``` 95 | 96 | Take a look at our [build instructions for the sample Sudoku driver](https://github.com/metabase/sudoku-driver#build-it-updated-for-build-script-changes-in-metabase-0460) for an explanation of the command. 97 | 98 | Note that while this command itself is quite a lot to type, you no longer need to specify a `:build` alias in your 99 | driver's `deps.edn` file. 100 | 101 | Please upvote https://ask.clojure.org/index.php/7843/allow-specifying-aliases-coordinates-that-point-projects , 102 | which will allow us to simplify the driver build command in the future. 103 | 104 | 105 | ### Copy it to your plugins dir and restart Metabase 106 | 107 | ```bash 108 | mkdir -p /path/to/metabase/plugins/ 109 | cp target/uberjar/http.metabase-driver.jar /path/to/metabase/plugins/ 110 | jar -jar /path/to/metabase/metabase.jar 111 | ``` 112 | 113 | ### Or [Adding external dependencies or plugins](https://www.metabase.com/docs/latest/installation-and-operation/running-metabase-on-docker#adding-external-dependencies-or-plugins) 114 | To add external dependency JAR files, you’ll need to: 115 | 116 | - create a plugins directory in your host system 117 | - bind that directory so it’s available to Metabase as the path /plugins (using either --mount or -v/--volume). 118 | 119 | For example, if you have a directory named /path/to/plugins on your host system, you can make its contents available to Metabase using the --mount option as follows: 120 | 121 | ```bash 122 | docker run -d -p 3000:3000 \ 123 | --mount type=bind,source=/path/to/plugins,destination=/plugins \ 124 | --name metabase metabase/metabase 125 | ``` 126 | 127 | ### Note that Metabase will use this directory to extract plugins bundled with the default Metabase distribution (such as drivers for various databases such as SQLite), thus it must be readable and writable by Docker. -------------------------------------------------------------------------------- /build-and-copy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -eu 4 | 5 | lein clean 6 | DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar 7 | 8 | mkdir -p ../metabase/plugins/ 9 | cp target/uberjar/http.metabase-driver.jar ../metabase/plugins/ -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths 2 | ["src" "resources"] 3 | 4 | :deps 5 | { 6 | org.clojure/core.logic {:mvn/version "1.0.0"} 7 | clj-http/clj-http { 8 | :mvn/version "3.12.3" ; HTTP client 9 | :exclusions [commons-codec/commons-codec 10 | commons-io/commons-io 11 | slingshot/slingshot]} 12 | com.jayway.jsonpath/json-path {:mvn/version "2.4.0"} 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /resources/metabase-plugin.yaml: -------------------------------------------------------------------------------- 1 | # Complete list of options here: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference 2 | info: 3 | name: Metabase HTTP Driver 4 | version: 1.0.0 5 | description: HTTP/REST API driver 6 | contact-info: 7 | name: Ahamove-HTTP 8 | address: https://github.com/AhaMove/metabase-http-driver 9 | driver: 10 | name: http 11 | lazy-load: true 12 | connection-properties: 13 | - name: definitions 14 | display-name: Table Definitions 15 | default: "{\n \"tables\": [\n ]\n}" 16 | required: true 17 | init: 18 | - step: load-namespace 19 | namespace: metabase.driver.http 20 | -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhaMove/metabase-http-driver/1b3ef32763738fa22816a751345a952cdb55ef0b/src/.DS_Store -------------------------------------------------------------------------------- /src/metabase/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AhaMove/metabase-http-driver/1b3ef32763738fa22816a751345a952cdb55ef0b/src/metabase/.DS_Store -------------------------------------------------------------------------------- /src/metabase/driver/http.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.http 2 | "HTTP API driver." 3 | (:require [cheshire.core :as json] 4 | [metabase.driver :as driver] 5 | [metabase.models.metric :as metric :refer [Metric]] 6 | [metabase.driver.http.query-processor :as http.qp] 7 | [metabase.query-processor.store :as qp.store] 8 | [metabase.util :as u] 9 | [metabase.driver.http.parameters :as parameters])) 10 | 11 | (defn find-first 12 | [f coll] 13 | (first (filter f coll))) 14 | 15 | (defn- database->definitions 16 | [database] 17 | (json/parse-string (:definitions (:details database)) keyword)) 18 | 19 | (defn- database->table-defs 20 | [database] 21 | (or (:tables (database->definitions database)) [])) 22 | 23 | (defn- database->table-def 24 | [database name] 25 | (first (filter #(= (:name %) name) (database->table-defs database)))) 26 | 27 | (defn table-def->field 28 | [table-def name] 29 | (find-first #(= (:name %) name) (:fields table-def))) 30 | 31 | (defn mbql-field->expression 32 | [table-def expr] 33 | (let [field (table-def->field table-def (:field-name expr))] 34 | (or (:expression field) (:name field)))) 35 | 36 | (defn mbql-aggregation->aggregation 37 | [table-def mbql-aggregation] 38 | (if (:field mbql-aggregation) 39 | [(:aggregation-type mbql-aggregation) 40 | (mbql-field->expression table-def (:field mbql-aggregation))] 41 | [(:aggregation-type mbql-aggregation)])) 42 | 43 | (def json-type->base-type 44 | {:string :type/Text 45 | :number :type/Float 46 | :boolean :type/Boolean}) 47 | 48 | (driver/register! :http) 49 | 50 | (defmethod driver/supports? [:http :basic-aggregations] [_ _] false) 51 | 52 | (defmethod driver/can-connect? :http [_ _] 53 | true) 54 | 55 | (defmethod driver/describe-database :http [_ database] 56 | (let [table-defs (database->table-defs database)] 57 | {:tables (set (for [table-def table-defs] 58 | {:name (:name table-def) 59 | :schema (:schema table-def)}))})) 60 | 61 | (defmethod driver/describe-table :http [_ database table] 62 | (let [table-def (database->table-def database (:name table))] 63 | {:name (:name table-def) 64 | :schema (:schema table-def) 65 | :fields (set (for [field (:fields table-def)] 66 | {:name (:name field) 67 | :database-type (:type field) 68 | :base-type (or (:base_type field) 69 | (json-type->base-type (keyword (:type field))))}))})) 70 | 71 | (defmethod driver/mbql->native :http [_ query] 72 | (let [database (qp.store/database) 73 | table (qp.store/table (:source-table (:query query))) 74 | table-def (database->table-def database (:name table)) 75 | breakout (map (partial mbql-field->expression table-def) (:breakout (:query query))) 76 | aggregation (map (partial mbql-aggregation->aggregation table-def) (:aggregation (:query query)))] 77 | {:query (merge (select-keys table-def [:method :url :headers]) 78 | {:result (merge (:result table-def) 79 | {:breakout breakout 80 | :aggregation aggregation})}) 81 | :mbql? true})) 82 | 83 | (driver/register! :http) 84 | 85 | (defmethod driver/supports? [:http :native-parameters] [_ _] true) 86 | 87 | (defmethod driver/supports? [:http :foreign-keys] [_ _] false) 88 | (defmethod driver/supports? [:http :nested-fields] [_ _] false) 89 | (defmethod driver/supports? [:http :set-timezone] [_ _] false) 90 | (defmethod driver/supports? [:http :basic-aggregations] [_ _] false) 91 | (defmethod driver/supports? [:http :expressions] [_ _] false) 92 | (defmethod driver/supports? [:http :expression-aggregations] [_ _] false) 93 | (defmethod driver/supports? [:http :nested-queries] [_ _] false) 94 | (defmethod driver/supports? [:http :binning] [_ _] false) 95 | (defmethod driver/supports? [:http :case-sensitivity-string-filter-options] [_ _] false) 96 | (defmethod driver/supports? [:http :left-join] [_ _] false) 97 | (defmethod driver/supports? [:http :right-join] [_ _] false) 98 | (defmethod driver/supports? [:http :inner-join] [_ _] false) 99 | (defmethod driver/supports? [:http :full-join] [_ _] false) 100 | 101 | (defmethod driver/substitute-native-parameters :http 102 | [driver inner-query] 103 | (parameters/substitute-native-parameters driver inner-query)) 104 | 105 | (defmethod driver/execute-reducible-query :http [_ {native-query :native} _ respond] 106 | (http.qp/execute-http-request native-query respond)) -------------------------------------------------------------------------------- /src/metabase/driver/http/parameters.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.http.parameters 2 | (:require [clojure 3 | [string :as str] 4 | [walk :as walk]] 5 | [clojure.tools.logging :as log] 6 | [java-time :as t] 7 | [metabase.driver.common.parameters :as params] 8 | [metabase.driver.common.parameters 9 | [dates :as date-params] 10 | [parse :as parse] 11 | [values :as values] 12 | ] 13 | [metabase.query-processor 14 | [error-type :as error-type]] 15 | [metabase.util :as u] 16 | [metabase.util 17 | [date-2 :as u.date] 18 | [i18n :refer [tru]]]) 19 | (:import java.time.temporal.Temporal 20 | [metabase.driver.common.parameters Date])) 21 | 22 | (defn- ->utc-instant [t] 23 | (t/instant 24 | (condp instance? t 25 | java.time.LocalDate (t/zoned-date-time t (t/local-time "00:00") (t/zone-id "UTC")) 26 | java.time.LocalDateTime (t/zoned-date-time t (t/zone-id "UTC")) 27 | t))) 28 | 29 | (defn- param-value->str 30 | [{special-type :special_type, :as field} x] 31 | (println "param-value->str" x) 32 | (cond 33 | ;; sequences get converted to `$in` 34 | (sequential? x) 35 | (format "%s" (str/join ", " (map (partial param-value->str field) x))) 36 | 37 | ;; Date = the Parameters Date type, not an java.util.Date or java.sql.Date type 38 | ;; convert to a `Temporal` instance and recur 39 | (instance? Date x) 40 | (param-value->str field (u.date/parse (:s x))) 41 | 42 | (and (instance? Temporal x) 43 | (isa? special-type :type/UNIXTimestampSeconds)) 44 | (long (/ (t/to-millis-from-epoch (->utc-instant x)) 1000)) 45 | 46 | (and (instance? Temporal x) 47 | (isa? special-type :type/UNIXTimestampMilliseconds)) 48 | (t/to-millis-from-epoch (->utc-instant x)) 49 | 50 | ;; convert temporal types to ISODate("2019-12-09T...") (etc.) 51 | (instance? Temporal x) 52 | (format "%s" (u.date/format x)) 53 | 54 | ;; for everything else, splice it in as its string representation 55 | :else 56 | x)) 57 | 58 | (defn- field->name [field] 59 | (:name field)) 60 | 61 | (defn- substitute-one-field-filter-date-range [{field :field, {value :value} :value}] 62 | (let [{:keys [start end]} (date-params/date-string->range value {:inclusive-end? false}) 63 | start-condition (when start 64 | (format "{%s: {$gte: %s}}" (field->name field) (param-value->str field (u.date/parse start)))) 65 | end-condition (when end 66 | (format "{%s: {$lt: %s}}" (field->name field) (param-value->str field (u.date/parse end))))] 67 | (if (and start-condition end-condition) 68 | (format "{$and: [%s, %s]}" start-condition end-condition) 69 | (or start-condition 70 | end-condition)))) 71 | 72 | ;; Field filter value is either params/no-value (handled in `substitute-param`, a map with `:type` and `:value`, or a 73 | ;; sequence of those maps. 74 | (defn- substitute-one-field-filter [{field :field, {param-type :type, value :value} :value, :as field-filter}] 75 | ;; convert relative dates to approprate date range representations 76 | (cond 77 | (date-params/not-single-date-type? param-type) 78 | (substitute-one-field-filter-date-range field-filter) 79 | 80 | ;; a `date/single` like `2020-01-10` 81 | (and (date-params/date-type? param-type) 82 | (string? value)) 83 | (let [t (u.date/parse value)] 84 | (format "{$and: [%s, %s]}" 85 | (format "{%s: {$gte: %s}}" (field->name field) (param-value->str field t)) 86 | (format "{%s: {$lt: %s}}" (field->name field) (param-value->str field (u.date/add t :day 1))))) 87 | 88 | :else 89 | (format "%s" (param-value->str field value)))) 90 | 91 | (defn- substitute-field-filter [{field :field, {:keys [value]} :value, :as field-filter}] 92 | (if (sequential? value) 93 | (format "%s" (param-value->str field value)) 94 | (substitute-one-field-filter field-filter))) 95 | 96 | (defn- substitute-param [param->value [acc missing] in-optional? {:keys [k], :as param}] 97 | (println "param" param) 98 | (let [v (get param->value k)] 99 | (cond 100 | (not (contains? param->value k)) 101 | [acc (conj missing k)] 102 | 103 | (params/FieldFilter? v) 104 | (let [no-value? (= (:value v) params/no-value)] 105 | (cond 106 | ;; no-value field filters inside optional clauses are ignored and omitted entirely 107 | (and no-value? in-optional?) [acc (conj missing k)] 108 | ;; otherwise replace it with a {} which is the $match equivalent of 1 = 1, i.e. always true 109 | no-value? [(conj acc "{}") missing] 110 | :else [(conj acc (substitute-field-filter v)) 111 | missing])) 112 | 113 | (= v params/no-value) 114 | [acc (conj missing k)] 115 | 116 | :else 117 | [(conj acc (param-value->str nil v)) missing]))) 118 | 119 | (declare substitute*) 120 | 121 | (defn- substitute-optional [param->value [acc missing] {subclauses :args}] 122 | (let [[opt-acc opt-missing] (substitute* param->value subclauses true)] 123 | (if (seq opt-missing) 124 | [acc missing] 125 | [(into acc opt-acc) missing]))) 126 | 127 | (defn- substitute* 128 | "Returns a sequence of `[[replaced...] missing-parameters]`." 129 | [param->value xs in-optional?] 130 | (reduce 131 | (fn [[acc missing] x] 132 | (cond 133 | (string? x) 134 | [(conj acc x) missing] 135 | 136 | (params/Param? x) 137 | (substitute-param param->value [acc missing] in-optional? x) 138 | 139 | (params/Optional? x) 140 | (substitute-optional param->value [acc missing] x) 141 | 142 | :else 143 | (throw (ex-info (tru "Don''t know how to substitute {0} {1}" (.getName (class x)) (pr-str x)) 144 | {:type error-type/driver})))) 145 | [[] nil] 146 | xs)) 147 | 148 | (defn- substitute [param->value xs] 149 | (let [[replaced missing] (substitute* param->value xs false)] 150 | (when (seq missing) 151 | (throw (ex-info (tru "Cannot run query: missing required parameters: {0}" (set missing)) 152 | {:type error-type/invalid-query}))) 153 | (when (seq replaced) 154 | (str/join replaced)))) 155 | 156 | (defn- parse-and-substitute [param->value x] 157 | (if-not (string? x) 158 | x 159 | (u/prog1 (substitute param->value (parse/parse x)) 160 | (when-not (= x <>) 161 | (log/debug (tru "Substituted {0} -> {1}" (pr-str x) (pr-str <>))))))) 162 | 163 | (defn substitute-native-parameters 164 | [_ inner-query] 165 | (let [param->value (values/query->params-map inner-query)] 166 | (println "parse-inner-query" inner-query) 167 | (update inner-query :query (partial walk/postwalk (partial parse-and-substitute param->value))))) -------------------------------------------------------------------------------- /src/metabase/driver/http/query_processor.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.http.query-processor 2 | (:refer-clojure :exclude [==]) 3 | (:require [cheshire.core :as json] 4 | [clojure.walk :as walk] 5 | [clj-http.client :as client]) 6 | (:import [com.jayway.jsonpath JsonPath Predicate])) 7 | 8 | (declare compile-expression compile-function) 9 | 10 | (defn json-path 11 | [query body] 12 | (JsonPath/read body query (into-array Predicate []))) 13 | 14 | (defn compile-function 15 | [[operator & arguments]] 16 | (case (keyword operator) 17 | :count count 18 | :sum #(reduce + (map (compile-expression (first arguments)) %)) 19 | :float #(Float/parseFloat ((compile-expression (first arguments)) %)) 20 | (throw (Exception. (str "Unknown operator: " operator))))) 21 | 22 | (defn compile-expression 23 | [expr] 24 | (cond 25 | (string? expr) (partial json-path expr) 26 | (number? expr) (constantly expr) 27 | (vector? expr) (compile-function expr) 28 | :else (throw (Exception. (str "Unknown expression: " expr))))) 29 | 30 | (defn aggregate 31 | [rows metrics breakouts] 32 | (let [breakouts-fns (map compile-expression breakouts) 33 | breakout-fn (fn [row] (for [breakout breakouts-fns] (breakout row))) 34 | metrics-fns (map compile-expression metrics)] 35 | (for [[breakout-key breakout-rows] (group-by breakout-fn rows)] 36 | (concat breakout-key (for [metrics-fn metrics-fns] 37 | (metrics-fn breakout-rows)))))) 38 | 39 | (defn extract-fields 40 | [rows fields] 41 | (let [fields-fns (map compile-expression fields)] 42 | (for [row rows] 43 | (for [field-fn fields-fns] 44 | (field-fn row))))) 45 | 46 | (defn field-names 47 | [fields] 48 | (vec (for [field fields] 49 | (if (string? field) 50 | {:name field} 51 | {:name (json/generate-string field)})))) 52 | 53 | (defn api-query [query rows respond] 54 | (let [fields (or (:fields (:result query)) (keys (first rows))) 55 | aggregations (or (:aggregation (:result query)) []) 56 | breakouts (or (:breakout (:result query)) []) 57 | raw (and (= (count breakouts) 0) (= (count aggregations) 0)) 58 | columns (if raw 59 | (field-names fields) 60 | (field-names (concat breakouts aggregations))) 61 | result (if raw 62 | (extract-fields rows fields) 63 | (aggregate rows aggregations breakouts))] 64 | (respond {:cols columns} 65 | result))) 66 | 67 | (defn execute-http-request [native-query respond] 68 | (println "native-query" native-query) 69 | (let [query (if (string? (:query native-query)) 70 | (json/parse-string (:query native-query) keyword) 71 | (:query native-query)) 72 | body (if (:body query) (json/generate-string (:body query))) 73 | result (client/request {:method (or (:method query) :get) 74 | :url (:url query) 75 | :headers (:headers query) 76 | :body body 77 | :accept :json 78 | :as :json 79 | :content-type "application/json"}) 80 | rows-path (or (:path (:result query)) "$") 81 | rows (json-path rows-path (walk/stringify-keys (:body result)))] 82 | (api-query query rows respond))) --------------------------------------------------------------------------------