├── .gitignore ├── README.md ├── project.clj ├── resources └── es-plugin.properties ├── src └── com │ └── thelastcitadel │ └── es │ ├── core.clj │ ├── engine.clj │ └── plugin.clj └── test ├── com └── thelastcitadel │ └── es │ └── test │ └── engine.clj └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /lib 3 | /classes 4 | /checkouts 5 | pom.xml 6 | pom.xml.asc 7 | *.jar 8 | *.class 9 | .lein-deps-sum 10 | .lein-failures 11 | .lein-plugins 12 | .lein-repl-history 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # elasticsearch-lang-clojure 2 | 3 | A an elasticsearch plugin written in clojure that provides clojure as 4 | a scripting language for elasticsearch queries 5 | 6 | ## Usage 7 | 8 | `lein package` 9 | 10 | elasticsearch is pretty crazy, check the docs, figure out how to 11 | install plugins 12 | 13 | ## License 14 | 15 | Copyright © 2013 Kevin Downey 16 | 17 | Distributed under the Eclipse Public License, the same as Clojure. 18 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject com.thelastcitadel/elasticsearch-lang-clojure "0.1.0-SNAPSHOT" 2 | :description "FIXME: write description" 3 | :url "http://example.com/FIXME" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :dependencies [[org.clojure/clojure "1.5.1"] 7 | [org.elasticsearch/elasticsearch "0.90.0.Beta1"] 8 | [org.clojure/tools.logging "0.2.6"]] 9 | :profiles {:dev {:dependencies [[com.spatial4j/spatial4j "0.3"] 10 | [log4j/log4j "1.2.16"]]}} 11 | :aot #{com.thelastcitadel.es.engine 12 | com.thelastcitadel.es.plugin}) 13 | 14 | (require 'leiningen.compile 15 | 'leiningen.uberjar 16 | 'leiningen.clean 17 | 'leiningen.core.utils 18 | 'leiningen.core.classpath) 19 | 20 | (def ^:dynamic x false) 21 | 22 | (defn package 23 | "" 24 | [project] 25 | (leiningen.clean/clean project) 26 | (leiningen.compile/compile project) 27 | (binding [x true] 28 | (leiningen.uberjar/uberjar 29 | (assoc project 30 | :uberjar-name (str (:name project) "-" (:version project) "-plugin" ".jar")) 31 | nil))) 32 | 33 | (alter-var-root #'leiningen.core.utils/require-resolve 34 | (fn [f] 35 | (fn [& args] 36 | (if (= '[leiningen.package/package] args) 37 | #'package 38 | (apply f args))))) 39 | 40 | (alter-var-root #'leiningen.core.classpath/resolve-dependencies 41 | (fn [f] 42 | (fn [k v & r] 43 | (if x 44 | (apply f k 45 | (update-in v [:dependencies] 46 | (partial remove 47 | (comp (partial = 'org.elasticsearch/elasticsearch) first))) 48 | r) 49 | (apply f k v r))))) 50 | -------------------------------------------------------------------------------- /resources/es-plugin.properties: -------------------------------------------------------------------------------- 1 | plugin=com.thelastcitadel.es.plugin 2 | -------------------------------------------------------------------------------- /src/com/thelastcitadel/es/core.clj: -------------------------------------------------------------------------------- 1 | ;; compilation environment for clojure es scripts 2 | (ns com.thelastcitadel.es.core) 3 | 4 | (defprotocol Value 5 | (value [_])) 6 | 7 | (extend-protocol Value 8 | org.elasticsearch.index.fielddata.ScriptDocValues 9 | (value [x] 10 | (.getValue x)) 11 | Object 12 | (value [x] 13 | x)) 14 | -------------------------------------------------------------------------------- /src/com/thelastcitadel/es/engine.clj: -------------------------------------------------------------------------------- 1 | (ns com.thelastcitadel.es.engine 2 | (:gen-class 3 | :extends org.elasticsearch.common.component.AbstractComponent 4 | :implements [org.elasticsearch.script.ScriptEngineService] 5 | :constructors {^{org.elasticsearch.common.inject.Inject true} [org.elasticsearch.common.settings.Settings] 6 | [org.elasticsearch.common.settings.Settings]}) 7 | (require [clojure.tools.logging :as log] 8 | [com.thelastcitadel.es.core])) 9 | 10 | (defn -init [s] 11 | [[] nil]) 12 | 13 | (declare executable-script 14 | search-script) 15 | 16 | (def types (into-array String ["clojure" "clj"])) 17 | 18 | (def exts (into-array String ["clj"])) 19 | 20 | (defn -types [_] 21 | types) 22 | 23 | (defn -extensions [_] 24 | exts) 25 | 26 | (defn -compile [_ script] 27 | (let [out (java.io.ByteArrayOutputStream.) 28 | fun (with-open [o (java.io.PrintWriter. 29 | (java.io.OutputStreamWriter. out)) 30 | e (java.io.PrintWriter. 31 | (java.io.OutputStreamWriter. out))] 32 | (binding [*warn-on-reflection* true 33 | *out* o 34 | *err* e 35 | *ns* (find-ns 'com.thelastcitadel.es.core)] 36 | (eval (read-string (str "(do " script " )"))))) 37 | out (String. (.toByteArray out))] 38 | (when-not (empty? out) 39 | (log/info out)) 40 | (fn [env] 41 | (try 42 | (fun env) 43 | (catch Exception e 44 | (log/info e) 45 | (throw e)))))) 46 | 47 | (defn -executable [_ compiled-script env] 48 | (executable-script compiled-script (into {} env))) 49 | 50 | (defn -search [_ compiled-script lookup env] 51 | (search-script compiled-script (into {} env) lookup)) 52 | 53 | (defn -execute [_ compiled-script env] 54 | (compiled-script env)) 55 | 56 | (defn -unWrap [_ x] 57 | x) 58 | 59 | (defn -close [_]) 60 | 61 | (defn executable-script [compiled-script env] 62 | (let [env (atom env)] 63 | (reify 64 | org.elasticsearch.script.ExecutableScript 65 | (setNextVar [_ name value] 66 | (swap! env assoc name value)) 67 | (run [_] 68 | (-execute nil compiled-script @env)) 69 | (unwrap [_ x] 70 | x)))) 71 | 72 | (defn search-script [compiled-script env ^org.elasticsearch.search.lookup.SearchLookup lookup] 73 | (let [env (atom (merge (into {} (.asMap lookup)) env))] 74 | (reify 75 | org.elasticsearch.script.SearchScript 76 | (setNextVar [_ name value] 77 | (swap! env assoc name value)) 78 | (run [_] 79 | (compiled-script @env)) 80 | (unwrap [_ x] 81 | x) 82 | (setScorer [_ scorer] 83 | (.setScorer lookup scorer)) 84 | (setNextReader [_ context] 85 | (.setNextReader lookup context)) 86 | (setNextDocId [_ id] 87 | (.setNextDocId lookup (int id))) 88 | (^void setNextSource [_ ^java.util.Map source] 89 | (-> lookup (.source) (.setNextSource source))) 90 | (setNextScore [this score] 91 | (.setNextVar this "_score" score)) 92 | (runAsFloat [this] 93 | (float (.run this))) 94 | (runAsLong [this] 95 | (long (.run this))) 96 | (runAsDouble [this] 97 | (double (.run this)))))) 98 | -------------------------------------------------------------------------------- /src/com/thelastcitadel/es/plugin.clj: -------------------------------------------------------------------------------- 1 | (ns com.thelastcitadel.es.plugin 2 | (:gen-class 3 | :extends org.elasticsearch.plugins.AbstractPlugin 4 | :methods [[onModule [org.elasticsearch.script.ScriptModule] 5 | void]])) 6 | 7 | (defn -name [_] 8 | "lang-clojure") 9 | 10 | (defn -description [_] 11 | "Adds Clojure scripting to ElasticSearch") 12 | 13 | (defn -onModule [_ module] 14 | (.addScriptEngine module (Class/forName "com.thelastcitadel.es.engine"))) 15 | -------------------------------------------------------------------------------- /test/com/thelastcitadel/es/test/engine.clj: -------------------------------------------------------------------------------- 1 | (ns com.thelastcitadel.es.test.engine 2 | (:require [clojure.test :refer :all]) 3 | (:import (org.elasticsearch.node NodeBuilder) 4 | (org.elasticsearch.common.settings ImmutableSettings) 5 | (org.elasticsearch.common.network NetworkUtils) 6 | (org.elasticsearch.index.query QueryBuilders 7 | FilterBuilders) 8 | (org.elasticsearch.client Requests) 9 | (org.elasticsearch.common.xcontent XContentFactory) 10 | (org.elasticsearch.search.sort SortOrder))) 11 | 12 | ;; https://github.com/elasticsearch/elasticsearch-lang-python/blob/master/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java 13 | 14 | (declare ^:dynamic *client*) 15 | 16 | (use-fixtures :each 17 | (fn [f] 18 | (let [node (-> (NodeBuilder/nodeBuilder) 19 | (.settings (-> (ImmutableSettings/settingsBuilder) 20 | (.put "path.data" "target/data") 21 | (.put "cluster.name" (str "test-cluster-" (NetworkUtils/getLocalAddress))) 22 | (.put "gateway.type" "none") 23 | (.put "number_of_shards" 1))) 24 | (.node))] 25 | (binding [*client* (.client node)] 26 | (try 27 | (f) 28 | (finally 29 | (.close *client*) 30 | (.close node))))))) 31 | 32 | (defn json 33 | ([m] 34 | (json (-> (XContentFactory/jsonBuilder) 35 | (.startObject)) 36 | m)) 37 | ([builder m] 38 | (.endObject 39 | (reduce 40 | (fn [obj [k v]] 41 | (.field obj (name k) v)) 42 | builder 43 | m)))) 44 | 45 | (defn exec [o] 46 | (.actionGet (.execute o))) 47 | 48 | (deftest t-filter 49 | (doto *client* 50 | (-> .admin .indices (.prepareCreate "test") exec) 51 | (-> (.prepareIndex "test" "type1" "1") (.setSource (json {:test "value beck" :num1 1.0})) exec) 52 | (-> (.prepareIndex "test" "type1" "2") (.setSource (json {:test "value beck" :num1 2.0})) exec) 53 | (-> (.prepareIndex "test" "type1" "3") (.setSource (json {:test "value beck" :num1 3.0})) exec) 54 | (-> .admin .indices (.refresh (Requests/refreshRequest (make-array String 0))) .actionGet) 55 | ;; 56 | (-> (.prepareSearch (make-array String 0)) 57 | (.setQuery (QueryBuilders/filteredQuery 58 | (QueryBuilders/matchAllQuery) 59 | (-> (FilterBuilders/scriptFilter 60 | (pr-str 61 | '(fn [env] 62 | (clojure.tools.logging/info "Hello World") 63 | (> (value (get (get env "doc") "num1")) 1.0)))) 64 | (.lang "clojure")))) 65 | (.addSort "num1" SortOrder/ASC) 66 | (.addScriptField "sNum1" "clojure" 67 | (pr-str '(fn [env] (value (get (get env "doc") "num1")))) nil) 68 | exec 69 | (doto (-> .getHits .totalHits (= 2) is)) 70 | (doto (-> .getHits (.getAt 0) .id (= "2") is)) 71 | (doto (-> .getHits (.getAt 0) .fields (.get "sNum1") .values (.get 0) (= 2.0) is)) 72 | (doto (-> .getHits (.getAt 1) .id (= "3") is)) 73 | (doto (-> .getHits (.getAt 1) .fields (.get "sNum1") .values (.get 0) (= 3.0) is))) 74 | ;; 75 | (-> (.prepareSearch (make-array String 0)) 76 | (.setQuery (QueryBuilders/filteredQuery 77 | (QueryBuilders/matchAllQuery) 78 | (-> (FilterBuilders/scriptFilter 79 | (pr-str 80 | '(fn [env] 81 | (> (value (get (get env "doc") "num1")) 82 | (get env "param1"))))) 83 | (.lang "clojure") 84 | (.addParam "param1" 2)))) 85 | (.addSort "num1" SortOrder/ASC) 86 | (.addScriptField "sNum1" "clojure" 87 | (pr-str '(fn [env] (value (get (get env "doc") "num1")))) nil) 88 | exec 89 | (doto (-> .getHits .totalHits (= 1) is)) 90 | (doto (-> .getHits (.getAt 0) .id (= "3") is)) 91 | (doto (-> .getHits (.getAt 0) .fields (.get "sNum1") .values (.get 0) (= 3.0) is))) 92 | ;; 93 | (-> (.prepareSearch (make-array String 0)) 94 | (.setQuery (QueryBuilders/filteredQuery 95 | (QueryBuilders/matchAllQuery) 96 | (-> (FilterBuilders/scriptFilter 97 | (pr-str 98 | '(fn [env] 99 | (> (value (get (get env "doc") "num1")) 100 | (get env "param1"))))) 101 | (.lang "clojure") 102 | (.addParam "param1" -1)))) 103 | (.addSort "num1" SortOrder/ASC) 104 | (.addScriptField "sNum1" "clojure" 105 | (pr-str '(fn [env] (value (get (get env "doc") "num1")))) nil) 106 | exec 107 | (doto (-> .getHits .totalHits (= 3) is)) 108 | (doto (-> .getHits (.getAt 0) .id (= "1") is)) 109 | (doto (-> .getHits (.getAt 0) .fields (get "sNum1") .values (.get 0) (= 1.0) is)) 110 | (doto (-> .getHits (.getAt 1) .id (= "2") is)) 111 | (doto (-> .getHits (.getAt 1) .fields (get "sNum1") .values (.get 0) (= 2.0) is)) 112 | (doto (-> .getHits (.getAt 2) .id (= "3") is)) 113 | (doto (-> .getHits (.getAt 2) .fields (get "sNum1") .values (.get 0) (= 3.0) is))))) 114 | -------------------------------------------------------------------------------- /test/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, out 2 | 3 | log4j.appender.out=org.apache.log4j.ConsoleAppender 4 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n 6 | --------------------------------------------------------------------------------