├── .gitignore ├── resources └── metabase-plugin.yaml ├── project.clj ├── README.md └── src └── metabase └── driver ├── http └── query_processor.clj └── http.clj /.gitignore: -------------------------------------------------------------------------------- 1 | \#*\# 2 | .\#* 3 | /target 4 | /.nrepl-port 5 | -------------------------------------------------------------------------------- /resources/metabase-plugin.yaml: -------------------------------------------------------------------------------- 1 | # Complete list of options here: https://github.com/metabase/metabase/wiki/Metabase-Plugin-Manifest-Reference 2 | info: 3 | name: Metabase HTTP Driver 4 | version: 1.0.0 5 | description: HTTP/REST API driver 6 | driver: 7 | name: http 8 | lazy-load: true 9 | connection-properties: 10 | - name: definitions 11 | display-name: Table Definitions 12 | type: json 13 | default: "{\n \"tables\": [\n ]\n}" 14 | init: 15 | - step: load-namespace 16 | namespace: metabase.driver.http 17 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject metabase/http-driver "1.0.0" 2 | :min-lein-version "2.5.0" 3 | 4 | :dependencies 5 | [[com.jayway.jsonpath/json-path "2.3.0"]] 6 | 7 | :jvm-opts 8 | ["-XX:+IgnoreUnrecognizedVMOptions" 9 | "--add-modules=java.xml.bind"] 10 | 11 | :profiles 12 | {:provided 13 | {:dependencies 14 | [[org.clojure/clojure "1.9.0"] 15 | [metabase-core "1.0.0-SNAPSHOT"]]} 16 | 17 | :uberjar 18 | {:auto-clean true 19 | :aot :all 20 | :omit-source true 21 | :javac-options ["-target" "1.8", "-source" "1.8"] 22 | :target-path "target/%s" 23 | :uberjar-name "http.metabase-driver.jar"}}) 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTP Metabase Driver 2 | 3 | This is a proof-of-concept HTTP "driver" for [Metabase](https://metabase.com/). 4 | 5 | Previous discussion: https://github.com/metabase/metabase/pull/7047 6 | 7 | ## Usage 8 | 9 | Currently the simplest "native" query for this driver is simply an object with a `url` property: 10 | 11 | ```json 12 | { "url": "https://api.coinmarketcap.com/v1/ticker/" } 13 | ``` 14 | 15 | The driver will make a `GET` request and parse the resulting JSON array into rows. Currently it only supports JSON. 16 | 17 | You can provide a different `method` as well as `headers` and a JSON `body`: 18 | 19 | ```json 20 | { 21 | "url": "https://api.coinmarketcap.com/v1/ticker/", 22 | "method": "POST", 23 | "headers": { 24 | "Authentication": "SOMETOKEN" 25 | }, 26 | "body": { 27 | "foo": "bar" 28 | } 29 | } 30 | ``` 31 | 32 | Additionally, you can provide a `result` object with a JSONPath to the "root" in the response, and/or a list of `fields`: 33 | 34 | ```json 35 | { 36 | "url" : "https://blockchain.info/blocks?format=json", 37 | "result" : { 38 | "path" : "blocks", 39 | "fields": ["height", "time"] 40 | } 41 | } 42 | ``` 43 | 44 | You can also predefine "tables" in the database configuration's `Table Definitions` setting. These tables will appear in the graphical query builder: 45 | 46 | ```json 47 | { 48 | "tables" : [ 49 | { 50 | "name" : "Blocks", 51 | "url" : "https://blockchain.info/blocks?format=json", 52 | "fields" : [ 53 | { "name" : "height", "type" : "number" }, 54 | { "name" : "hash", "type" : "string" }, 55 | { "name" : "time", "type" : "number" }, 56 | { "type" : "boolean", "name" : "main_chain" } 57 | ], 58 | "result" : { 59 | "path" : "blocks" 60 | } 61 | } 62 | ] 63 | } 64 | ``` 65 | 66 | There is limited support for aggregations and breakouts, but this is very experimental and may be removed in future versions. 67 | 68 | ## Building the driver 69 | 70 | ### Prereq: Install Metabase as a local maven dependency, compiled for building drivers 71 | 72 | Clone the [Metabase repo](https://github.com/metabase/metabase) first if you haven't already done so. 73 | 74 | ```bash 75 | cd /path/to/metabase_source 76 | lein install-for-building-drivers 77 | ``` 78 | 79 | ### Build the HTTP driver 80 | 81 | ```bash 82 | # (In the HTTP driver directory) 83 | lein clean 84 | DEBUG=1 LEIN_SNAPSHOTS_IN_RELEASE=true lein uberjar 85 | ``` 86 | 87 | ### Copy it to your plugins dir and restart Metabase 88 | 89 | ```bash 90 | mkdir -p /path/to/metabase/plugins/ 91 | cp target/uberjar/http.metabase-driver.jar /path/to/metabase/plugins/ 92 | jar -jar /path/to/metabase/metabase.jar 93 | ``` 94 | 95 | _or:_ 96 | 97 | ```bash 98 | mkdir -p /path/to/metabase_source/plugins 99 | cp target/uberjar/http.metabase-driver.jar /path/to/metabase_source/plugins/ 100 | cd /path/to/metabase_source 101 | lein run 102 | ``` 103 | 104 | -------------------------------------------------------------------------------- /src/metabase/driver/http/query_processor.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.http.query-processor 2 | (:refer-clojure :exclude [==]) 3 | (:require [cheshire.core :as json] 4 | [clojure.walk :as walk] 5 | [clj-http.client :as client]) 6 | (:import [com.jayway.jsonpath JsonPath Predicate])) 7 | 8 | 9 | (declare compile-expression compile-function) 10 | 11 | (defn json-path 12 | [query body] 13 | (JsonPath/read body query (into-array Predicate []))) 14 | 15 | (defn compile-function 16 | [[operator & arguments]] 17 | (case (keyword operator) 18 | :count count 19 | :sum #(reduce + (map (compile-expression (first arguments)) %)) 20 | :float #(Float/parseFloat ((compile-expression (first arguments)) %)) 21 | (throw (Exception. (str "Unknown operator: " operator))))) 22 | 23 | (defn compile-expression 24 | [expr] 25 | (cond 26 | (string? expr) (partial json-path expr) 27 | (number? expr) (constantly expr) 28 | (vector? expr) (compile-function expr) 29 | :else (throw (Exception. (str "Unknown expression: " expr))))) 30 | 31 | (defn aggregate 32 | [rows metrics breakouts] 33 | (let [breakouts-fns (map compile-expression breakouts) 34 | breakout-fn (fn [row] (for [breakout breakouts-fns] (breakout row))) 35 | metrics-fns (map compile-expression metrics)] 36 | (for [[breakout-key breakout-rows] (group-by breakout-fn rows)] 37 | (concat breakout-key (for [metrics-fn metrics-fns] 38 | (metrics-fn breakout-rows)))))) 39 | 40 | (defn extract-fields 41 | [rows fields] 42 | (let [fields-fns (map compile-expression fields)] 43 | (for [row rows] 44 | (for [field-fn fields-fns] 45 | (field-fn row))))) 46 | 47 | (defn field-names 48 | [fields] 49 | (for [field fields] 50 | (keyword (if (string? field) 51 | field 52 | (json/generate-string field))))) 53 | 54 | (defn execute-http-request [native-query] 55 | (let [query (if (string? (:query native-query)) 56 | (json/parse-string (:query native-query) keyword) 57 | (:query native-query)) 58 | result (client/request {:method (or (:method query) :get) 59 | :url (:url query) 60 | :headers (:headers query) 61 | :body (if (:body query) (json/generate-string (:body query))) 62 | :accept :json 63 | :as :json}) 64 | rows-path (or (:path (:result query)) "$") 65 | rows (json-path rows-path (walk/stringify-keys (:body result))) 66 | fields (or (:fields (:result query)) (keys (first rows))) 67 | aggregations (or (:aggregation (:result query)) []) 68 | breakouts (or (:breakout (:result query)) []) 69 | raw (and (= (count breakouts) 0) (= (count aggregations) 0))] 70 | {:columns (if raw 71 | (field-names fields) 72 | (field-names (concat breakouts aggregations))) 73 | :rows (if raw 74 | (extract-fields rows fields) 75 | (aggregate rows aggregations breakouts))})) -------------------------------------------------------------------------------- /src/metabase/driver/http.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.http 2 | "HTTP API driver." 3 | (:require [cheshire.core :as json] 4 | [clojure.tools.logging :as log] 5 | [metabase.driver :as driver] 6 | [metabase.driver.http.query-processor :as http.qp] 7 | [metabase.query-processor.store :as qp.store] 8 | [metabase.util :as u])) 9 | 10 | (defn find-first 11 | [f coll] 12 | (first (filter f coll))) 13 | 14 | (defn- database->definitions 15 | [database] 16 | (json/parse-string (:definitions (:details database)) keyword)) 17 | 18 | (defn- database->table-defs 19 | [database] 20 | (or (:tables (database->definitions database)) [])) 21 | 22 | (defn- database->table-def 23 | [database name] 24 | (first (filter #(= (:name %) name) (database->table-defs database)))) 25 | 26 | (defn table-def->field 27 | [table-def name] 28 | (find-first #(= (:name %) name) (:fields table-def))) 29 | 30 | (defn mbql-field->expression 31 | [table-def expr] 32 | (let [field (table-def->field table-def (:field-name expr))] 33 | (or (:expression field) (:name field)))) 34 | 35 | (defn mbql-aggregation->aggregation 36 | [table-def mbql-aggregation] 37 | (if (:field mbql-aggregation) 38 | [(:aggregation-type mbql-aggregation) 39 | (mbql-field->expression table-def (:field mbql-aggregation))] 40 | [(:aggregation-type mbql-aggregation)])) 41 | 42 | (def json-type->base-type 43 | {:string :type/Text 44 | :number :type/Float 45 | :boolean :type/Boolean}) 46 | 47 | (driver/register! :http) 48 | 49 | (defmethod driver/supports? [:http :basic-aggregations] [_ _] false) 50 | 51 | (defmethod driver/can-connect? :http [_ _] 52 | true) 53 | 54 | (defmethod driver/describe-database :http [_ database] 55 | (let [table-defs (database->table-defs database)] 56 | {:tables (set (for [table-def table-defs] 57 | {:name (:name table-def) 58 | :schema (:schema table-def)}))})) 59 | 60 | (defmethod driver/describe-table :http [_ database table] 61 | (let [table-def (database->table-def database (:name table))] 62 | {:name (:name table-def) 63 | :schema (:schema table-def) 64 | :fields (set (for [field (:fields table-def)] 65 | {:name (:name field) 66 | :database-type (:type field) 67 | :base-type (or (:base_type field) 68 | (json-type->base-type (keyword (:type field))))}))})) 69 | 70 | (defmethod driver/mbql->native :http [_ query] 71 | (let [database (qp.store/database) 72 | table (qp.store/table (:source-table (:query query))) 73 | table-def (database->table-def database (:name table)) 74 | breakout (map (partial mbql-field->expression table-def) (:breakout (:query query))) 75 | aggregation (map (partial mbql-aggregation->aggregation table-def) (:aggregation (:query query)))] 76 | {:query (merge (select-keys table-def [:method :url :headers]) 77 | {:result (merge (:result table-def) 78 | {:breakout breakout 79 | :aggregation aggregation})}) 80 | :mbql? true})) 81 | 82 | (defmethod driver/execute-query :http [_ {native-query :native}] 83 | (http.qp/execute-http-request native-query)) --------------------------------------------------------------------------------