├── cookbooks ├── git │ ├── templates │ │ └── default │ │ │ ├── sv-git-daemon-log-run.erb │ │ │ └── sv-git-daemon-run.erb │ ├── CHANGELOG.md │ ├── metadata.rb │ ├── recipes │ │ ├── server.rb │ │ ├── default.rb │ │ └── source.rb │ ├── CONTRIBUTING │ ├── README.md │ └── attributes │ │ └── default.rb ├── golang │ ├── attributes │ │ └── default.rb │ ├── metadata.rb │ ├── README.md │ └── recipes │ │ ├── ppa.rb │ │ └── default.rb ├── mercurial │ ├── metadata.rb │ ├── CHANGELOG.md │ ├── resources │ │ └── default.rb │ ├── recipes │ │ └── default.rb │ ├── CONTRIBUTING │ ├── providers │ │ └── default.rb │ └── README.md └── build-essential │ ├── CHANGELOG.md │ ├── metadata.rb │ ├── CONTRIBUTING │ ├── attributes │ └── default.rb │ ├── recipes │ └── default.rb │ └── README.md ├── search ├── README ├── facet_test.go ├── sort.go ├── test_test.go ├── facet.go ├── filter_test.go ├── aggregate_test.go ├── aggregate.go ├── filter.go ├── search.go └── query.go ├── tutorial ├── README.md └── start_1.go ├── .travis.yml ├── .gitignore ├── .gitmodules ├── HACKING.md ├── core ├── bulkUDP.go ├── msearch.go ├── delete.go ├── explain.go ├── count.go ├── search_test.go ├── validate.go ├── deleteByQuery.go ├── mget.go ├── moreLikeThis.go ├── percolate.go ├── example_test.go ├── get.go ├── update.go ├── index.go ├── test_test.go └── search.go ├── indices ├── doc.go ├── aliases.go ├── segments.go ├── stats.go ├── createIndex.go ├── deleteMapping.go ├── getSettings.go ├── templates.go ├── openCloseIndex.go ├── updateSettings.go ├── indicesExists.go ├── deleteIndex.go ├── status.go ├── clearCache.go ├── optimize.go ├── snapshot.go ├── refresh.go ├── flush.go ├── analyze.go ├── putMapping.go └── putMapping_test.go ├── cluster ├── nodesHotThreads.go ├── state.go ├── nodesShutdown.go ├── nodesinfo_test.go ├── updateSettings.go ├── clusterreroute.go ├── health.go ├── stats.go └── nodesInfo.go ├── doc.go ├── api ├── shared.go ├── searchdsl.go ├── clusterhealthresponses.go ├── baseResponse.go ├── request_test.go ├── baseRequest.go └── clusterstatresponses.go ├── Vagrantfile ├── client.go └── README.md /cookbooks/git/templates/default/sv-git-daemon-log-run.erb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec svlogd -tt ./main 3 | -------------------------------------------------------------------------------- /search/README: -------------------------------------------------------------------------------- 1 | 2 | 3 | To run tests on this, you must first have run/imported data inside of *core* 4 | 5 | -------------------------------------------------------------------------------- /tutorial/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Tutorials 4 | ======================================= 5 | 6 | To run these tutorials:: 7 | 8 | 9 | go run start_1.go 10 | 11 | # etc -------------------------------------------------------------------------------- /cookbooks/git/templates/default/sv-git-daemon-run.erb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec 2>&1 3 | exec /usr/bin/git daemon --export-all --user=nobody --group=daemon --base-path=/srv/git /srv/git 4 | -------------------------------------------------------------------------------- /cookbooks/golang/attributes/default.rb: -------------------------------------------------------------------------------- 1 | default[:golang] = { 2 | # can be "stable" or "tip" 3 | :version => "stable", 4 | :multi => { 5 | :versions => %w(go1.0.3 go1.1.1), 6 | :default_version => "go1.1.1", 7 | :aliases => { 8 | "go1" => "go1.1.1" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /cookbooks/mercurial/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs mercurial" 5 | version "0.8.0" 6 | 7 | recipe "mercurial", "Installs mercurial" 8 | 9 | %w{ debian ubuntu }.each do |os| 10 | supports os 11 | end 12 | -------------------------------------------------------------------------------- /cookbooks/git/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.0.2: 2 | 3 | * [COOK-1537] - add recipe for source installation 4 | 5 | ## v1.0.0: 6 | 7 | * [COOK-1152] - Add support for Mac OS X 8 | * [COOK-1112] - Add support for Windows 9 | 10 | ## v0.10.0: 11 | 12 | * [COOK-853] - Git client installation on CentOS 13 | 14 | ## v0.9.0: 15 | 16 | * Current public release. 17 | -------------------------------------------------------------------------------- /cookbooks/mercurial/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.0.0: 2 | 3 | * [COOK-1373] - README example correction 4 | * [COOK-1179] - LWRP for repo management 5 | 6 | For further discussion about possible changes to the LWRP, see 7 | COOK-879, whereby it may become a fully fledged provider for chef's 8 | built in scm_repo resource. 9 | 10 | ## v0.7.1: 11 | 12 | * Current public release 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2 5 | 6 | install: 7 | - wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.0.deb 8 | - sudo dpkg -i elasticsearch-1.1.0.deb 9 | - sudo service elasticsearch start 10 | 11 | script: 12 | - cd core 13 | - go get -t 14 | - go build 15 | - go test -v -eshost localhost -loaddata 16 | - go install 17 | -------------------------------------------------------------------------------- /cookbooks/golang/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Matthew Baird" 2 | maintainer_email "mattbaird@gmail.com" 3 | license "Apache 2.0" 4 | description "Installs go language from duh's Ubuntu PPA" 5 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 6 | version "1.0.0" 7 | recipe "golang", "Installs go" 8 | 9 | depends "apt" 10 | supports "ubuntu" 11 | -------------------------------------------------------------------------------- /cookbooks/mercurial/resources/default.rb: -------------------------------------------------------------------------------- 1 | actions :sync, :clone 2 | 3 | attribute :path, :kind_of => String, :name_attribute => true 4 | attribute :repository, :kind_of => String 5 | attribute :reference, :kind_of => [Integer, String] 6 | attribute :key, :kind_of => String 7 | attribute :owner, :kind_of => String 8 | attribute :group, :kind_of => String 9 | attribute :mode, :kind_of => String, :default => '0775' 10 | -------------------------------------------------------------------------------- /cookbooks/build-essential/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.1.2: 2 | 3 | * [COOK-1620] - support OS X 10.8 4 | 5 | ## v1.1.0: 6 | 7 | * [COOK-1098] - support amazon linux 8 | * [COOK-1149] - support Mac OS X 9 | * [COOK-1296] - allow for compile-time installation of packages 10 | through an attribute (see README) 11 | 12 | ## v1.0.2: 13 | 14 | * [COOK-1098] - Add Amazon Linux platform support 15 | * [COOK-1149] - Add OS X platform support 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | .DS_Store 10 | .vagrant 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | 26 | elastigo 27 | 28 | # IDE 29 | 30 | *.iml -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "java"] 2 | path = java 3 | url = https://github.com/cookbooks/java.git 4 | [submodule "cookbooks/java"] 5 | path = cookbooks/java 6 | url = https://github.com/cookbooks/java.git 7 | [submodule "cookbooks/apt"] 8 | path = cookbooks/apt 9 | url = https://github.com/opscode-cookbooks/apt.git 10 | [submodule "cookbooks/elasticsearch"] 11 | path = cookbooks/elasticsearch 12 | url = https://github.com/mattbaird/elasticsearch-chef.git 13 | -------------------------------------------------------------------------------- /cookbooks/build-essential/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs C compiler / build tools" 5 | version "1.1.2" 6 | recipe "build-essential", "Installs packages required for compiling C software from source." 7 | 8 | %w{ fedora redhat centos ubuntu debian amazon }.each do |os| 9 | supports os 10 | end 11 | 12 | supports "mac_os_x", ">= 10.6.0" 13 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | 2 | Testing 3 | ----------------- 4 | 5 | To run tests, this library loads data into an elasticsearch server and tests against that. 6 | 7 | See core/test_test.go. The data set should remain the same as it pulls a known set of github archive data. 8 | 9 | usage: 10 | 11 | $cd core 12 | 13 | $go test -v -host eshost -loaddata # load the data 14 | 15 | $go test -v -host eshost # without load data, which only needs to run once 16 | 17 | Clean out the Elasticsearch index: 18 | 19 | http -v DELETE http://localhost:9200/github 20 | or 21 | curl -XDELETE http://localhost:9200/github 22 | -------------------------------------------------------------------------------- /core/bulkUDP.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | -------------------------------------------------------------------------------- /core/msearch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | -------------------------------------------------------------------------------- /indices/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/aliases.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/segments.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/createIndex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/deleteMapping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/getSettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/templates.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /cluster/nodesHotThreads.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Matthew Baird 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | -------------------------------------------------------------------------------- /indices/openCloseIndex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /indices/updateSettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | -------------------------------------------------------------------------------- /cookbooks/git/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs git and/or sets up a Git server daemon" 5 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 6 | version "1.0.2" 7 | recipe "git", "Installs git" 8 | recipe "git::server", "Sets up a runit_service for git daemon" 9 | recipe "git::source", "Installs git from source" 10 | 11 | %w{ amazon arch centos debian fedora redhat scientific ubuntu windows }.each do |os| 12 | supports os 13 | end 14 | 15 | supports "mac_os_x", ">= 10.6.0" 16 | 17 | %w{ build-essential dmg runit yum }.each do |cb| 18 | depends cb 19 | end 20 | -------------------------------------------------------------------------------- /cookbooks/mercurial/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mercurial 3 | # Recipe:: default 4 | # 5 | # Copyright 2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | package "mercurial" do 20 | action :upgrade 21 | end 22 | -------------------------------------------------------------------------------- /cookbooks/golang/README.md: -------------------------------------------------------------------------------- 1 | # Go Language Chef Cookbook 2 | 3 | This is an OpsCode Chef cookbook for [Go, the programming language](http://golang.org). 4 | 5 | It uses the ["Todd Vierling" Ubuntu PPA](https://launchpad.net/~duh/+archive/golang) 6 | and allows you to tweak version using Chef node attributes. 7 | 8 | It is released under the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 9 | 10 | 11 | ## Recipes 12 | 13 | Main recipe is `golang::default`. 14 | 15 | ## Supported OSes 16 | 17 | Ubuntu 10.10 to 12.04, will likely work just as well on Debian unstable. 18 | 19 | 20 | ## Dependencies 21 | 22 | None. 23 | 24 | 25 | ## Copyright & License 26 | 27 | Matthew Baird, 2013. 28 | 29 | Released under the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 30 | -------------------------------------------------------------------------------- /api/shared.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package api 13 | 14 | type Query struct { 15 | Query Term `json:"query"` 16 | } 17 | 18 | type Term struct { 19 | Term string `json:"term"` 20 | } 21 | 22 | func (q Query) setQuery(query string) { 23 | q.Query.Term = query 24 | } 25 | -------------------------------------------------------------------------------- /api/searchdsl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package api 13 | 14 | type SearchRequest struct { 15 | From int `json:"from,omitempty"` 16 | Size int `json:"size,omitempty"` 17 | Query Query `json:"query,omitempty"` 18 | Filter Filter `json:"filter,omitempty"` 19 | } 20 | 21 | type Filter struct { 22 | Term Term `json:"term"` 23 | } 24 | 25 | type Facets struct { 26 | Tag Terms `json:"tag"` 27 | } 28 | 29 | type Terms struct { 30 | Terms string `json:"terms"` 31 | } 32 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/server.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: server 4 | # 5 | # Copyright 2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | include_recipe "git" 20 | 21 | directory "/srv/git" do 22 | owner "root" 23 | group "root" 24 | mode 0755 25 | end 26 | 27 | case node[:platform] 28 | when "debian", "ubuntu" 29 | include_recipe "runit" 30 | runit_service "git-daemon" 31 | else 32 | log "Platform requires setting up a git daemon service script." 33 | log "Hint: /usr/bin/git daemon --export-all --user=nobody --group=daemon --base-path=/srv/git" 34 | end 35 | -------------------------------------------------------------------------------- /cookbooks/golang/recipes/ppa.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: golang 3 | # Recipe:: ppa 4 | # 5 | # Copyright 2012, Michael S. Klishin, Travis CI Development Team 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | apt_repository "duh-ppa" do 21 | uri "http://ppa.launchpad.net/duh/golang/ubuntu" 22 | distribution node['lsb']['codename'] 23 | components ["main"] 24 | key "60480472" 25 | keyserver "keyserver.ubuntu.com" 26 | action :add 27 | notifies :run, "execute[apt-get update]", :immediately 28 | end 29 | 30 | package "golang" do 31 | # version "1.1.1" 32 | action :install 33 | end 34 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = "lucid64" 6 | config.vm.box_url = "http://files.vagrantup.com/lucid64.box" 7 | config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true 8 | config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative" 9 | 10 | config.vm.provider :virtualbox do |vb| 11 | vb.gui = false 12 | vb.customize ["modifyvm", :id, "--memory", "1024"] 13 | vb.customize ["modifyvm", :id, "--cpus", "1"] 14 | # This allows symlinks to be created within the /vagrant root directory, 15 | # which is something librarian-puppet needs to be able to do. This might 16 | # be enabled by default depending on what version of VirtualBox is used. 17 | vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"] 18 | end 19 | config.vm.provision :chef_solo do |chef| 20 | chef.cookbooks_path = "cookbooks" 21 | chef.add_recipe("apt") 22 | chef.add_recipe("java") 23 | chef.add_recipe("elasticsearch") 24 | chef.add_recipe("git") 25 | chef.add_recipe("mercurial") 26 | chef.add_recipe("build-essential") 27 | chef.add_recipe("golang") 28 | end 29 | end -------------------------------------------------------------------------------- /cookbooks/git/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /cookbooks/mercurial/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /cookbooks/build-essential/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /search/facet_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | //"encoding/json" 16 | "fmt" 17 | "github.com/araddon/gou" 18 | "github.com/bmizerany/assert" 19 | "testing" 20 | ) 21 | 22 | func TestFacetRegex(t *testing.T) { 23 | // This is a possible solution for auto-complete 24 | out, _ := Search("github").Size("0").Facet( 25 | Facet().Regex("repository.name", "no.*").Size("8"), 26 | ).Result() 27 | if out == nil || &out.Hits == nil { 28 | t.Fail() 29 | return 30 | } 31 | //Debug(string(out.Facets)) 32 | fh := gou.NewJsonHelper([]byte(out.Facets)) 33 | facets := fh.Helpers("/repository.name/terms") 34 | assert.T(t, len(facets) == 8, fmt.Sprintf("Should have 8? but was %v", len(facets))) 35 | // for _, f := range facets { 36 | // Debug(f) 37 | // } 38 | } 39 | -------------------------------------------------------------------------------- /indices/indicesExists.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "fmt" 16 | "github.com/buger/elastigo/api" 17 | "strings" 18 | ) 19 | 20 | // IndicesExists checks for the existance of indices. uses http 404 if it does not exist, and 200 if it does 21 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-indices-exists/ 22 | func IndicesExists(indices ...string) (bool, error) { 23 | var url string 24 | if len(indices) > 0 { 25 | url = fmt.Sprintf("/%s", strings.Join(indices, ",")) 26 | } 27 | _, err := api.DoCommand("HEAD", url, nil, nil) 28 | if err != nil { 29 | eserror := err.(api.ESError) 30 | if eserror.Code == 404 { 31 | return false, err 32 | } else { 33 | return eserror.Code == 200, err 34 | } 35 | } 36 | return true, nil 37 | } 38 | -------------------------------------------------------------------------------- /api/clusterhealthresponses.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type ClusterHealthResponse struct { 4 | ClusterName string `json:"cluster_name"` 5 | Status string `json:"status"` 6 | TimedOut bool `json:"timed_out"` 7 | NumberOfNodes int `json:"number_of_nodes"` 8 | NumberOfDataNodes int `json:"number_of_data_nodes"` 9 | ActivePrimaryShards int `json:"active_primary_shards"` 10 | ActiveShards int `json:"active_shards"` 11 | RelocatingShards int `json:"relocating_shards"` 12 | InitializingShards int `json:"initializing_shards"` 13 | UnassignedShards int `json:"unassigned_shards"` 14 | } 15 | 16 | type ClusterStateResponse struct { 17 | ClusterName string `json:"cluster_name"` 18 | MasterNode string `json:"master_node"` 19 | Nodes map[string]ClusterStateNodeReponse `json:"nodes"` 20 | // TODO: Metadata 21 | // TODO: Routing Table 22 | // TODO: Routing Nodes 23 | // TODO: Allocations 24 | 25 | } 26 | 27 | type ClusterStateNodeReponse struct { 28 | Name string `json:"name"` 29 | TransportAddress string `json:"transport_address"` 30 | // TODO: Attributes 31 | } 32 | 33 | type ClusterStateMetadataResponse struct { 34 | // TODO: templates 35 | // TODO: indices 36 | } 37 | 38 | type ClusterStateRoutingTableResponse struct { 39 | // TODO: unassigned 40 | // 41 | } 42 | -------------------------------------------------------------------------------- /core/delete.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // Delete API allows to delete a typed JSON document from a specific index based on its id. 21 | // http://www.elasticsearch.org/guide/reference/api/delete.html 22 | func Delete(index string, _type string, id string, args map[string]interface{}) (api.BaseResponse, error) { 23 | var url string 24 | var retval api.BaseResponse 25 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 26 | body, err := api.DoCommand("DELETE", url, args, nil) 27 | if err != nil { 28 | return retval, err 29 | } 30 | if err == nil { 31 | // marshall into json 32 | jsonErr := json.Unmarshal(body, &retval) 33 | if jsonErr != nil { 34 | return retval, jsonErr 35 | } 36 | } 37 | return retval, err 38 | } 39 | -------------------------------------------------------------------------------- /cluster/state.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | 14 | import ( 15 | "encoding/json" 16 | "github.com/buger/elastigo/api" 17 | ) 18 | 19 | // State gets the comprehensive state information for the whole cluster 20 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-state/ 21 | func UpdateSetting(args map[string]interface{}, filter_indices ...string) (ClusterStateResponse, error) { 22 | var url string 23 | var retval ClusterStateResponse 24 | 25 | url = "/_cluster/state" 26 | 27 | body, err := api.DoCommand("GET", url, args, nil) 28 | if err != nil { 29 | return retval, err 30 | } 31 | if err == nil { 32 | // marshall into json 33 | jsonErr := json.Unmarshal(body, &retval) 34 | if jsonErr != nil { 35 | return retval, jsonErr 36 | } 37 | } 38 | return retval, err 39 | } 40 | 41 | type ClusterStateResponse struct { 42 | } 43 | -------------------------------------------------------------------------------- /cookbooks/golang/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: golang 3 | # Recipe:: default 4 | # 5 | # Copyright 2012, Michael S. Klishin, Travis CI Development Team 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "golang::ppa" 21 | # echo 'export GOBIN=#{node['golang']['gobin']}' >> /home/vagrant/.bash_golang 22 | # echo 'export GOROOT=/usr/local/go/' >> /home/vagrant/.bash_golang 23 | 24 | bash "Export ENV Vars" do 25 | code <<-EOC 26 | mkdir -p /home/vagrant/code/go/ 27 | chown vagrant /home/vagrant/code/go/ 28 | echo 'export GOPATH=/home/vagrant/code/go/' >> /home/vagrant/.bash_golang 29 | echo 'export GOROOT=/usr/lib/go/' >> /home/vagrant/.bash_golang 30 | echo 'export PATH=$PATH:$GOBIN' >> /home/vagrant/.bash_golang 31 | echo 'source /home/vagrant/.bash_golang' >> /home/vagrant/.bashrc 32 | source /home/vagrant/.bashrc 33 | EOC 34 | creates "/home/vagrant/.bash_golang" 35 | end 36 | -------------------------------------------------------------------------------- /indices/deleteIndex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // The delete API allows you to delete one or more indices through an API. This operation may fail 21 | // if the elasitsearch configuration has been set to forbid deleting indexes. 22 | func Delete(index string) (api.BaseResponse, error) { 23 | var url string 24 | var retval api.BaseResponse 25 | 26 | if len(index) > 0 { 27 | url = fmt.Sprintf("/%s", index) 28 | } else { 29 | return retval, fmt.Errorf("You must specify at least one index to delete") 30 | } 31 | 32 | body, err := api.DoCommand("DELETE", url, nil, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | 37 | jsonErr := json.Unmarshal(body, &retval) 38 | if jsonErr != nil { 39 | return retval, jsonErr 40 | } 41 | 42 | return retval, err 43 | } 44 | -------------------------------------------------------------------------------- /indices/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // Status lists status details of all indices or the specified index. 22 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-status.html 23 | func Status(args map[string]interface{}, indices ...string) (api.BaseResponse, error) { 24 | var retval api.BaseResponse 25 | var url string 26 | if len(indices) > 0 { 27 | url = fmt.Sprintf("/%s/_status", strings.Join(indices, ",")) 28 | 29 | } else { 30 | url = "/_status" 31 | } 32 | body, err := api.DoCommand("GET", url, args, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | -------------------------------------------------------------------------------- /cluster/nodesShutdown.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | 14 | import ( 15 | "fmt" 16 | "github.com/buger/elastigo/api" 17 | "net/url" 18 | "strconv" 19 | "strings" 20 | ) 21 | 22 | // NodesShutdown allows the caller to shutdown between one and all nodes in the cluster 23 | // delay is a integer representing number of seconds 24 | // passing "" or "_all" for the nodes parameter will shut down all nodes 25 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-nodes-shutdown/ 26 | func NodesShutdown(delay int, nodes ...string) error { 27 | shutdownUrl := fmt.Sprintf("/_cluster/nodes/%s/_shutdown", strings.Join(nodes, ",")) 28 | if delay > 0 { 29 | var values url.Values = url.Values{} 30 | values.Add("delay", strconv.Itoa(delay)) 31 | shutdownUrl += "?" + values.Encode() 32 | } 33 | _, err := api.DoCommand("POST", shutdownUrl, nil, nil) 34 | if err != nil { 35 | return err 36 | } 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /search/sort.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // SortDsl accepts any number of Sort commands 20 | // 21 | // Query().Sort( 22 | // Sort("last_name").Desc(), 23 | // Sort("age"), 24 | // ) 25 | func Sort(field string) *SortDsl { 26 | return &SortDsl{Name: field} 27 | } 28 | 29 | type SortBody []interface{} 30 | type SortDsl struct { 31 | Name string 32 | IsDesc bool 33 | } 34 | 35 | func (s *SortDsl) Desc() *SortDsl { 36 | s.IsDesc = true 37 | return s 38 | } 39 | func (s *SortDsl) Asc() *SortDsl { 40 | s.IsDesc = false 41 | return s 42 | } 43 | 44 | func (s *SortDsl) MarshalJSON() ([]byte, error) { 45 | if s.IsDesc { 46 | return json.Marshal(map[string]string{s.Name: "desc"}) 47 | } 48 | if s.Name == "_score" { 49 | return []byte(`"_score"`), nil 50 | } 51 | return []byte(fmt.Sprintf(`"%s"`, s.Name)), nil // "user" assuming default = asc? 52 | } 53 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: default 4 | # 5 | # Copyright 2008-2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | case node[:platform] 20 | when "debian", "ubuntu" 21 | package "git-core" 22 | when "centos","redhat","scientific","fedora" 23 | case node[:platform_version].to_i 24 | when 5 25 | include_recipe "yum::epel" 26 | end 27 | package "git" 28 | when "windows" 29 | windows_package "git" do 30 | source node[:git][:url] 31 | checksum node[:git][:checksum] 32 | action :install 33 | not_if { File.exists? 'C:\Program Files (x86)\Git\bin\git.exe' } 34 | end 35 | when "mac_os_x" 36 | dmg_package "GitOSX-Installer" do 37 | app node[:git][:osx_dmg][:app_name] 38 | package_id node[:git][:osx_dmg][:package_id] 39 | volumes_dir node[:git][:osx_dmg][:volumes_dir] 40 | source node[:git][:osx_dmg][:url] 41 | checksum node[:git][:osx_dmg][:checksum] 42 | type "pkg" 43 | action :install 44 | end 45 | else 46 | package "git" 47 | end 48 | -------------------------------------------------------------------------------- /cookbooks/build-essential/attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: build-essential 3 | # Attributes:: default 4 | # 5 | # Copyright 2008-2012, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | default['build_essential']['compiletime'] = false 21 | 22 | case platform 23 | when "mac_os_x" 24 | case 25 | when Chef::VersionConstraint.new("~> 10.7.0").include?(platform_version), 26 | Chef::VersionConstraint.new("~> 10.8.0").include?(platform_version) 27 | default['build_essential']['osx']['gcc_installer_url'] = "https://github.com/downloads/kennethreitz/osx-gcc-installer/GCC-10.7-v2.pkg" 28 | default['build_essential']['osx']['gcc_installer_checksum'] = "df36aa87606feb99d0db9ac9a492819e" 29 | when Chef::VersionConstraint.new("~> 10.6.0").include?(platform_version) 30 | default['build_essential']['osx']['gcc_installer_url'] = "https://github.com/downloads/kennethreitz/osx-gcc-installer/GCC-10.6.pkg" 31 | default['build_essential']['osx']['gcc_installer_checksum'] = "d1db5bab6a3f6b9f3b5577a130baeefa" 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /cookbooks/git/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs git and optionally sets up a git server as a daemon under runit. 5 | 6 | Requirements 7 | ============ 8 | 9 | ## Platform: 10 | 11 | * Debian/Ubuntu 12 | * ArchLinux 13 | 14 | ## Cookbooks: 15 | 16 | * runit 17 | 18 | Recipes 19 | ======= 20 | 21 | ## default 22 | 23 | Installs base git packages based on platform. 24 | 25 | ## server 26 | 27 | Sets up a git daemon to provide a server. 28 | 29 | ## source 30 | 31 | Installs git from source. 32 | 33 | Usage 34 | ===== 35 | 36 | This cookbook primarily installs git core packages. It can also be 37 | used to serve git repositories. 38 | 39 | include_recipe "git::server" 40 | 41 | This creates the directory /srv/git and starts a git daemon, exporting 42 | all repositories found. Repositories need to be added manually, but 43 | will be available once they are created. 44 | 45 | License and Author 46 | ================== 47 | 48 | Author:: Joshua Timberman () 49 | 50 | Copyright:: 2009-2012, Opscode, Inc. 51 | 52 | Licensed under the Apache License, Version 2.0 (the "License"); 53 | you may not use this file except in compliance with the License. 54 | You may obtain a copy of the License at 55 | 56 | http://www.apache.org/licenses/LICENSE-2.0 57 | 58 | Unless required by applicable law or agreed to in writing, software 59 | distributed under the License is distributed on an "AS IS" BASIS, 60 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 61 | See the License for the specific language governing permissions and 62 | limitations under the License. 63 | -------------------------------------------------------------------------------- /search/test_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "flag" 16 | "github.com/araddon/gou" 17 | "github.com/buger/elastigo/api" 18 | "github.com/buger/elastigo/core" 19 | "log" 20 | "os" 21 | //"testing" 22 | ) 23 | 24 | var ( 25 | _ = log.Ldate 26 | hasStartedTesting bool 27 | eshost *string = flag.String("host", "localhost", "Elasticsearch Server Host Address") 28 | logLevel *string = flag.String("logging", "info", "Which log level: [debug,info,warn,error,fatal]") 29 | ) 30 | 31 | /* 32 | 33 | usage: 34 | 35 | test -v -host eshost 36 | 37 | */ 38 | 39 | func init() { 40 | InitTests(false) 41 | if *logLevel == "debug" { 42 | //*logLevel = "debug" 43 | core.DebugRequests = true 44 | } 45 | } 46 | 47 | func InitTests(startIndexer bool) { 48 | if !hasStartedTesting { 49 | flag.Parse() 50 | hasStartedTesting = true 51 | gou.SetLogger(log.New(os.Stderr, "", log.Ltime|log.Lshortfile), *logLevel) 52 | log.SetFlags(log.Ltime | log.Lshortfile) 53 | api.Domain = *eshost 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /cookbooks/mercurial/providers/default.rb: -------------------------------------------------------------------------------- 1 | action :sync do 2 | execute "sync repository #{new_resource.path}" do 3 | not_if "hg identify #{new_resource.path}" 4 | command "hg clone -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository} #{new_resource.path}" 5 | end 6 | execute "pull changes #{new_resource.path}" do 7 | command "cd #{new_resource.path} && hg pull -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository}" 8 | end 9 | execute "update #{new_resource.path}" do 10 | command "cd #{new_resource.path} && hg update -r #{new_resource.reference}" 11 | end 12 | execute "sync update owner #{new_resource.path}" do 13 | command "chown -R #{new_resource.owner}:#{new_resource.group} #{new_resource.path}" 14 | end 15 | execute "sync update permissions #{new_resource.path}" do 16 | command "chmod -R #{new_resource.mode} #{new_resource.path}" 17 | end 18 | end 19 | 20 | action :clone do 21 | execute "clone repository #{new_resource.path}" do 22 | not_if "hg identify #{new_resource.path}" 23 | command "hg clone -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository} #{new_resource.path}" 24 | end 25 | if new_resource.reference 26 | command "cd #{new_resource.path} && hg update -r #{new_resource.reference}" 27 | end 28 | execute "update owner #{new_resource.path}" do 29 | command "chown -R #{new_resource.owner}:#{new_resource.group} #{new_resource.path}" 30 | end 31 | execute "update permissions #{new_resource.path}" do 32 | command "chmod -R #{new_resource.mode} #{new_resource.path}" 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /indices/clearCache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // ClearCache allows to clear either all caches or specific cached associated with one ore more indices. 22 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-clearcache/ 23 | func ClearCache(clearId bool, clearBloom bool, args map[string]interface{}, indices ...string) (api.ExtendedStatus, error) { 24 | var retval api.ExtendedStatus 25 | var clearCacheUrl string 26 | if len(indices) > 0 { 27 | clearCacheUrl = fmt.Sprintf("/%s/_cache/clear", strings.Join(indices, ",")) 28 | 29 | } else { 30 | clearCacheUrl = fmt.Sprintf("/_cache/clear") 31 | } 32 | 33 | body, err := api.DoCommand("POST", clearCacheUrl, args, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | -------------------------------------------------------------------------------- /core/explain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // Explain computes a score explanation for a query and a specific document. 21 | // This can give useful feedback whether a document matches or didn’t match a specific query. 22 | // This feature is available from version 0.19.9 and up. 23 | // see http://www.elasticsearch.org/guide/reference/api/explain.html 24 | func Explain(index string, _type string, id string, args map[string]interface{}, query string) (api.Match, error) { 25 | var url string 26 | var retval api.Match 27 | if len(_type) > 0 { 28 | url = fmt.Sprintf("/%s/%s/_explain", index, _type) 29 | } else { 30 | url = fmt.Sprintf("/%s/_explain", index) 31 | } 32 | body, err := api.DoCommand("GET", url, args, query) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | -------------------------------------------------------------------------------- /indices/optimize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text. 22 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/ 23 | func OptimizeIndices(args map[string]interface{}, indices ...string) (OptimizeResponse, error) { 24 | var retval OptimizeResponse 25 | var optimizeUrl string = "/_optimize" 26 | if len(indices) > 0 { 27 | optimizeUrl = fmt.Sprintf("/%s/%s", strings.Join(indices, ","), optimizeUrl) 28 | } 29 | 30 | body, err := api.DoCommand("POST", optimizeUrl, args, nil) 31 | if err != nil { 32 | return retval, err 33 | } 34 | if err == nil { 35 | // marshall into json 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | } 41 | return retval, err 42 | } 43 | 44 | type OptimizeResponse struct { 45 | Ok bool `json:"ok"` 46 | Shards api.Status `json:"_shards"` 47 | } 48 | -------------------------------------------------------------------------------- /indices/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // Snapshot allows to explicitly perform a snapshot through the gateway of one or more indices (backup them). 22 | // By default, each index gateway periodically snapshot changes, though it can be disabled and be controlled completely through this API. 23 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-gateway-snapshot/ 24 | func Snapshot(indices ...string) (api.ExtendedStatus, error) { 25 | var retval api.ExtendedStatus 26 | var url string 27 | if len(indices) > 0 { 28 | url = fmt.Sprintf("/%s/_gateway/snapshot", strings.Join(indices, ",")) 29 | 30 | } else { 31 | url = fmt.Sprintf("/_gateway/snapshot") 32 | } 33 | body, err := api.DoCommand("POST", url, nil, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/source.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: source 4 | # 5 | # Copyright 2012, Brian Flad, Fletcher Nichol 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | include_recipe "build-essential" 20 | 21 | pkgs = value_for_platform_family( 22 | ["rhel"] => %w{ expat-devel gettext-devel libcurl-devel openssl-devel zlib-devel } 23 | ) 24 | 25 | pkgs.each do |pkg| 26 | package pkg 27 | end 28 | 29 | remote_file "#{Chef::Config[:file_cache_path]}/git-#{node[:git][:version]}.tar.gz" do 30 | source node[:git][:url] 31 | checksum node[:git][:checksum] 32 | mode "0644" 33 | not_if "test -f #{Chef::Config[:file_cache_path]}/git-#{node[:git][:version]}.tar.gz" 34 | end 35 | 36 | execute "Extracting and Building Git #{node[:git][:version]} from Source" do 37 | cwd Chef::Config[:file_cache_path] 38 | command <<-COMMAND 39 | (mkdir git-#{node[:git][:version]} && tar -zxf git-#{node[:git][:version]}.tar.gz -C git-#{node[:git][:version]} --strip-components 1) 40 | (cd git-#{node[:git][:version]} && make prefix=#{node[:git][:prefix]} install) 41 | COMMAND 42 | creates "node[:git][:prefix]}/bin/git" 43 | not_if "git --version | grep #{node[:git][:version]}" 44 | end 45 | -------------------------------------------------------------------------------- /core/count.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | type CountResponse struct { 21 | Count int `json:"count"` 22 | Shard api.Status `json:"_shards"` 23 | } 24 | 25 | // Count allows the caller to easily execute a query and get the number of matches for that query. 26 | // It can be executed across one or more indices and across one or more types. 27 | // The query can either be provided using a simple query string as a parameter, 28 | // or using the Query DSL defined within the request body. 29 | // http://www.elasticsearch.org/guide/reference/api/count.html 30 | func Count(index string, _type string, args map[string]interface{}) (CountResponse, error) { 31 | var url string 32 | var retval CountResponse 33 | url = fmt.Sprintf("/%s/%s/_count", index, _type) 34 | body, err := api.DoCommand("GET", url, args, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | -------------------------------------------------------------------------------- /indices/refresh.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // Refresh explicitly refreshes one or more index, making all operations performed since 22 | // the last refresh available for search. The (near) real-time capabilities depend on the index engine used. 23 | // For example, the robin one requires refresh to be called, but by default a refresh is scheduled periodically. 24 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-refresh.html 25 | // TODO: add Shards to response 26 | func Refresh(indices ...string) (api.BaseResponse, error) { 27 | var url string 28 | var retval api.BaseResponse 29 | if len(indices) > 0 { 30 | url = fmt.Sprintf("/%s/_refresh", strings.Join(indices, ",")) 31 | } else { 32 | url = "/_refresh" 33 | } 34 | body, err := api.DoCommand("POST", url, nil, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | -------------------------------------------------------------------------------- /indices/flush.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // Flush flushes one or more indices through an API. The flush process of an index basically 22 | // frees memory from the index by flushing data to the index storage and clearing the internal transaction 23 | // log. By default, ElasticSearch uses memory heuristics in order to automatically trigger flush operations 24 | // as required in order to clear memory. 25 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-flush.html 26 | // TODO: add Shards to response 27 | func Flush(indices ...string) (api.BaseResponse, error) { 28 | var url string 29 | var retval api.BaseResponse 30 | if len(indices) > 0 { 31 | url = fmt.Sprintf("/%s/_flush", strings.Join(indices, ",")) 32 | } else { 33 | url = "/_flush" 34 | } 35 | body, err := api.DoCommand("POST", url, nil, nil) 36 | if err != nil { 37 | return retval, err 38 | } 39 | if err == nil { 40 | // marshall into json 41 | jsonErr := json.Unmarshal(body, &retval) 42 | if jsonErr != nil { 43 | return retval, jsonErr 44 | } 45 | } 46 | return retval, err 47 | } 48 | -------------------------------------------------------------------------------- /cookbooks/git/attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Author:: Jamie Winsor () 3 | # Cookbook Name:: git 4 | # Attributes:: default 5 | # 6 | # Copyright 2008-2012, Opscode, Inc. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | case platform_family 21 | when 'windows' 22 | set[:git][:version] = "1.7.9-preview20120201" 23 | set[:git][:url] = "http://msysgit.googlecode.com/files/Git-#{node[:git][:version]}.exe" 24 | set[:git][:checksum] = "0627394709375140d1e54e923983d259a60f9d8e" 25 | when "mac_os_x" 26 | default[:git][:osx_dmg][:app_name] = "git-1.7.9.4-intel-universal-snow-leopard" 27 | default[:git][:osx_dmg][:volumes_dir] = "Git 1.7.9.4 Snow Leopard Intel Universal" 28 | default[:git][:osx_dmg][:package_id] = "GitOSX.Installer.git1794.git.pkg" 29 | default[:git][:osx_dmg][:url] = "http://git-osx-installer.googlecode.com/files/git-1.7.9.4-intel-universal-snow-leopard.dmg" 30 | default[:git][:osx_dmg][:checksum] = "661c3fcf765572d3978df17c7636d59e" 31 | else 32 | default[:git][:prefix] = "/usr/local" 33 | default[:git][:version] = "1.7.11.4" 34 | default[:git][:url] = "https://github.com/git/git/tarball/v#{node[:git][:version]}" 35 | default[:git][:checksum] = "7a26d9bd0fd3384374bdc1afaae829f406bc123126817d994a460c49a3260ecc" 36 | end 37 | -------------------------------------------------------------------------------- /core/search_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/bmizerany/assert" 18 | "testing" 19 | ) 20 | 21 | func TestSearchRequest(t *testing.T) { 22 | qry := map[string]interface{}{ 23 | "query": map[string]interface{}{ 24 | "wildcard": map[string]string{"actor": "a*"}, 25 | }, 26 | } 27 | var args map[string]interface{} 28 | out, err := SearchRequest("github", "", args, qry) 29 | //log.Println(out) 30 | assert.T(t, &out != nil && err == nil, fmt.Sprintf("Should get docs")) 31 | assert.T(t, out.Hits.Len() == 10, fmt.Sprintf("Should have 10 docs but was %v", out.Hits.Len())) 32 | expectedHits := 621 33 | assert.T(t, CloseInt(out.Hits.Total, expectedHits), fmt.Sprintf("Should have %v hits but was %v", expectedHits, out.Hits.Total)) 34 | } 35 | 36 | func TestSearchResultToJSON(t *testing.T) { 37 | qry := map[string]interface{}{ 38 | "query": map[string]interface{}{ 39 | "wildcard": map[string]string{"actor": "a*"}, 40 | }, 41 | } 42 | var args map[string]interface{} 43 | out, err := SearchRequest("github", "", args, qry) 44 | 45 | if err != nil { 46 | t.Error(err) 47 | } 48 | _, err = json.Marshal(out.Hits.Hits) 49 | if err != nil { 50 | t.Error(err) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /core/validate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // Validate allows a user to validate a potentially expensive query without executing it. 21 | // see http://www.elasticsearch.org/guide/reference/api/validate.html 22 | func Validate(index string, _type string, args map[string]interface{}) (api.BaseResponse, error) { 23 | var url string 24 | var retval api.BaseResponse 25 | if len(_type) > 0 { 26 | url = fmt.Sprintf("/%s/%s/_validate/", index, _type) 27 | } else { 28 | url = fmt.Sprintf("/%s/_validate/", index) 29 | } 30 | body, err := api.DoCommand("GET", url, args, nil) 31 | if err != nil { 32 | return retval, err 33 | } 34 | if err == nil { 35 | // marshall into json 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | } 41 | return retval, err 42 | } 43 | 44 | type Validation struct { 45 | Valid bool `json:"valid"` 46 | Shards api.Status `json:"_shards"` 47 | Explainations []Explaination `json:"explanations,omitempty"` 48 | } 49 | 50 | type Explaination struct { 51 | Index string `json:"index"` 52 | Valid bool `json:"valid"` 53 | Error string `json:"error"` 54 | } 55 | -------------------------------------------------------------------------------- /cluster/nodesinfo_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | 14 | import ( 15 | "fmt" 16 | "github.com/bmizerany/assert" 17 | "testing" 18 | ) 19 | 20 | func TestGetAll(t *testing.T) { 21 | nodesInfo, err := AllNodesInfo() 22 | //log.Println(out) 23 | assert.T(t, err == nil, fmt.Sprintf("should not have gotten error, received :%v", err)) 24 | assert.T(t, nodesInfo.ClusterName == "elasticsearch", fmt.Sprintf("clustername should have been elasticsearch, received :%v", err)) 25 | for _, node := range nodesInfo.Nodes { 26 | assert.T(t, node.Settings != nil, fmt.Sprintf("Settings should not have been null")) 27 | assert.T(t, node.OS != nil, fmt.Sprintf("OS should not have been null")) 28 | assert.T(t, node.Process != nil, fmt.Sprintf("Process should not have been null")) 29 | assert.T(t, node.JVM != nil, fmt.Sprintf("JVM should not have been null")) 30 | assert.T(t, node.ThreadPool != nil, fmt.Sprintf("ThreadPool should not have been null")) 31 | assert.T(t, node.Network != nil, fmt.Sprintf("Network should not have been null")) 32 | assert.T(t, node.Transport != nil, fmt.Sprintf("Transport should not have been null")) 33 | assert.T(t, node.Http != nil, fmt.Sprintf("Http should not have been null")) 34 | assert.T(t, node.Plugins != nil, fmt.Sprintf("Plugins should not have been null")) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /indices/analyze.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | "github.com/buger/elastigo/api" 19 | ) 20 | 21 | // AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text. 22 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/ 23 | func AnalyzeIndices(index string, args map[string]interface{}) (AnalyzeResponse, error) { 24 | var retval AnalyzeResponse 25 | if len(args["text"].(string)) == 0 { 26 | return retval, errors.New("text to analyze must not be blank") 27 | } 28 | var analyzeUrl string = "/_analyze" 29 | if len(index) > 0 { 30 | analyzeUrl = fmt.Sprintf("/%s/%s", index, analyzeUrl) 31 | } 32 | 33 | body, err := api.DoCommand("GET", analyzeUrl, args, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | 47 | type AnalyzeResponse struct { 48 | Tokens []Token `json:"tokens"` 49 | } 50 | type Token struct { 51 | Name string `json:"token"` 52 | StartOffset int `json:"start_offset"` 53 | EndOffset int `json:"end_offset"` 54 | Type string `json:"type"` 55 | Position int `json:"position"` 56 | } 57 | -------------------------------------------------------------------------------- /cluster/updateSettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // UpdateSettings allows to update cluster wide specific settings. Defaults to Transient setting 21 | // Settings updated can either be persistent (applied cross restarts) or transient (will not survive a full cluster restart). 22 | // http://www.elasticsearch.org/guide/reference/api/admin-cluster-update-settings.html 23 | func UpdateSettings(settingType string, key string, value int) (ClusterSettingsResponse, error) { 24 | var retval ClusterSettingsResponse 25 | if settingType != "transient" && settingType != "persistent" { 26 | return retval, fmt.Errorf("settingType must be one of transient or persistent, you passed %s", settingType) 27 | } 28 | var url string = "/_cluster/state" 29 | m := map[string]map[string]int{settingType: map[string]int{key: value}} 30 | body, err := api.DoCommand("PUT", url, nil, m) 31 | if err != nil { 32 | return retval, err 33 | } 34 | if err == nil { 35 | // marshall into json 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | } 41 | return retval, err 42 | } 43 | 44 | type ClusterSettingsResponse struct { 45 | Transient map[string]int `json:"transient"` 46 | Persistent map[string]int `json:"persistent"` 47 | } 48 | -------------------------------------------------------------------------------- /tutorial/start_1.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package main 13 | 14 | import ( 15 | "flag" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "github.com/buger/elastigo/core" 19 | "log" 20 | "os" 21 | ) 22 | 23 | var ( 24 | host *string = flag.String("host", "localhost", "Elasticsearch Host") 25 | ) 26 | 27 | func main() { 28 | core.DebugRequests = true 29 | log.SetFlags(log.LstdFlags) 30 | flag.Parse() 31 | 32 | fmt.Println("host = ", *host) 33 | // Set the Elasticsearch Host to Connect to 34 | api.Domain = *host 35 | 36 | // Index a document 37 | _, err := core.Index("testindex", "user", "docid_1", nil, `{"name":"bob"}`) 38 | exitIfErr(err) 39 | 40 | // Index a doc using a map of values 41 | _, err = core.Index("testindex", "user", "docid_2", nil, map[string]string{"name": "venkatesh"}) 42 | exitIfErr(err) 43 | 44 | // Index a doc using Structs 45 | _, err = core.Index("testindex", "user", "docid_3", nil, MyUser{"wanda", 22}) 46 | exitIfErr(err) 47 | 48 | // Search Using Raw json String 49 | searchJson := `{ 50 | "query" : { 51 | "term" : { "Name" : "wanda" } 52 | } 53 | }` 54 | out, err := core.SearchRequest("testindex", "user", nil, searchJson) 55 | if len(out.Hits.Hits) == 1 { 56 | fmt.Println("%v", out.Hits.Hits[0].Source) 57 | } 58 | exitIfErr(err) 59 | 60 | } 61 | func exitIfErr(err error) { 62 | if err != nil { 63 | fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) 64 | os.Exit(1) 65 | } 66 | } 67 | 68 | type MyUser struct { 69 | Name string 70 | Age int 71 | } 72 | -------------------------------------------------------------------------------- /api/baseResponse.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package api 13 | 14 | import () 15 | 16 | type BaseResponse struct { 17 | Ok bool `json:"ok"` 18 | Index string `json:"_index,omitempty"` 19 | Type string `json:"_type,omitempty"` 20 | Id string `json:"_id,omitempty"` 21 | Source interface{} `json:"_source,omitempty"` // depends on the schema you've defined 22 | Version int `json:"_version,omitempty"` 23 | Found bool `json:"found,omitempty"` 24 | Exists bool `json:"exists,omitempty"` 25 | Matches []string `json:"matches,omitempty"` // percolate matches 26 | } 27 | 28 | type ExtendedStatus struct { 29 | Ok bool `json:"ok"` 30 | ShardsStatus Status `json:"_shards"` 31 | } 32 | 33 | type Status struct { 34 | Total int `json:"total"` 35 | Successful int `json:"successful"` 36 | Failed int `json:"failed"` 37 | } 38 | 39 | type Match struct { 40 | OK bool `json:"ok"` 41 | Matches []string `json:"matches"` 42 | Explanation Explanation `json:"explanation,omitempty"` 43 | } 44 | 45 | type Explanation struct { 46 | Value float32 `json:"value"` 47 | Description string `json:"description"` 48 | Details []Explanation `json:"details,omitempty"` 49 | } 50 | 51 | func Scroll(duration string) string { 52 | scrollString := "" 53 | if duration != "" { 54 | scrollString = "&scroll=" + duration 55 | } 56 | return scrollString 57 | } 58 | 59 | // http://www.elasticsearch.org/guide/reference/api/search/search-type/ 60 | -------------------------------------------------------------------------------- /core/deleteByQuery.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strings" 19 | ) 20 | 21 | // DeleteByQuery allows the caller to delete documents from one or more indices and one or more types based on a query. 22 | // The query can either be provided using a simple query string as a parameter, or using the Query DSL defined within 23 | // the request body. 24 | // see: http://www.elasticsearch.org/guide/reference/api/delete-by-query.html 25 | func DeleteByQuery(indices []string, types []string, args map[string]interface{}, query interface{}) (api.BaseResponse, error) { 26 | var url string 27 | var retval api.BaseResponse 28 | if len(indices) > 0 && len(types) > 0 { 29 | url = fmt.Sprintf("/%s/%s/_query", strings.Join(indices, ","), strings.Join(types, ",")) 30 | } else if len(indices) > 0 { 31 | url = fmt.Sprintf("/%s/_query", strings.Join(indices, ",")) 32 | } 33 | body, err := api.DoCommand("DELETE", url, args, query) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | 47 | func buildQuery() string { 48 | return "" 49 | } 50 | 51 | type DeleteByQueryResponse struct { 52 | Status bool `json:"ok"` 53 | Indicies map[string]IndexStatus `json:"_indices"` 54 | } 55 | 56 | type IndexStatus struct { 57 | Shards api.Status `json:"_shards"` 58 | } 59 | -------------------------------------------------------------------------------- /cookbooks/mercurial/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs mercurial 5 | 6 | Requirements 7 | ============ 8 | 9 | A package named "mercurial" must exist in the platform package 10 | management system. 11 | 12 | Usage 13 | ===== 14 | 15 | Install mercurial to make sure it is available to check out code from 16 | mercurial repositories. 17 | 18 | Resource/Provider 19 | ================= 20 | 21 | This cookbook includes LWRPs for managing: mercurial 22 | 23 | mercurial 24 | --------- 25 | 26 | ### Actions 27 | 28 | - :clone - this will simply issue a clone of the repository at the revision specified (default tip). 29 | - :sync - this will issue a clone of the repository if there is nothing at the path specified, otherwise a pull and update will be issued to bring the directory up-to-date. 30 | 31 | ### Parameter Attributes 32 | 33 | - `path` - **Name attribute** path where the repository is checked 34 | out. 35 | - `repository` - Repository to check out 36 | - `reference` - Reference in the repository 37 | - `key` - a private key on disk to use, for private repositories, must 38 | already exist. 39 | - `owner` - local user that the clone is run as 40 | - `group` - local group that the clone is run as 41 | - `mode` - permissions of the cloned repository 42 | 43 | ### Example 44 | 45 | mercurial "/home/site/checkouts/www" do 46 | repository "ssh://hg@bitbucket.org/niallsco/chef-hg" 47 | reference "tip" 48 | key "/home/site/.ssh/keyname" 49 | action :sync 50 | end 51 | 52 | License and Author 53 | ================== 54 | 55 | Author:: Joshua Timberman 56 | 57 | Copyright:: 2009, Opscode, Inc 58 | 59 | Licensed under the Apache License, Version 2.0 (the "License"); 60 | you may not use this file except in compliance with the License. 61 | You may obtain a copy of the License at 62 | 63 | http://www.apache.org/licenses/LICENSE-2.0 64 | 65 | Unless required by applicable law or agreed to in writing, software 66 | distributed under the License is distributed on an "AS IS" BASIS, 67 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 68 | See the License for the specific language governing permissions and 69 | limitations under the License. 70 | -------------------------------------------------------------------------------- /core/mget.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // MGet allows the caller to get multiple documents based on an index, type (optional) and id (and possibly routing). 21 | // The response includes a docs array with all the fetched documents, each element similar in structure to a document 22 | // provided by the get API. 23 | // see http://www.elasticsearch.org/guide/reference/api/multi-get.html 24 | func MGet(index string, _type string, mgetRequest MGetRequestContainer, args map[string]interface{}) (MGetResponseContainer, error) { 25 | var url string 26 | var retval MGetResponseContainer 27 | if len(index) <= 0 { 28 | url = fmt.Sprintf("/_mget") 29 | } 30 | if len(_type) > 0 && len(index) > 0 { 31 | url = fmt.Sprintf("/%s/%s/_mget", index, _type) 32 | } else if len(index) > 0 { 33 | url = fmt.Sprintf("/%s/_mget", index) 34 | } 35 | body, err := api.DoCommand("GET", url, args, mgetRequest) 36 | if err != nil { 37 | return retval, err 38 | } 39 | if err == nil { 40 | // marshall into json 41 | jsonErr := json.Unmarshal(body, &retval) 42 | if jsonErr != nil { 43 | return retval, jsonErr 44 | } 45 | } 46 | return retval, err 47 | } 48 | 49 | type MGetRequestContainer struct { 50 | Docs []MGetRequest `json:"docs"` 51 | } 52 | 53 | type MGetRequest struct { 54 | Index string `json:"_index"` 55 | Type string `json:"_type"` 56 | ID string `json:"_id"` 57 | IDS []string `json:"_ids,omitempty"` 58 | Fields []string `json:"fields,omitempty"` 59 | } 60 | 61 | type MGetResponseContainer struct { 62 | Docs []api.BaseResponse `json:"docs"` 63 | } 64 | -------------------------------------------------------------------------------- /core/moreLikeThis.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // MoreLikeThis allows the caller to get documents that are “like” a specified document. 21 | // http://www.elasticsearch.org/guide/reference/api/more-like-this.html 22 | func MoreLikeThis(index string, _type string, id string, args map[string]interface{}, query MoreLikeThisQuery) (api.BaseResponse, error) { 23 | var url string 24 | var retval api.BaseResponse 25 | url = fmt.Sprintf("/%s/%s/%s/_mlt", index, _type, id) 26 | body, err := api.DoCommand("GET", url, args, query) 27 | if err != nil { 28 | return retval, err 29 | } 30 | if err == nil { 31 | // marshall into json 32 | jsonErr := json.Unmarshal(body, &retval) 33 | if jsonErr != nil { 34 | return retval, jsonErr 35 | } 36 | } 37 | return retval, err 38 | } 39 | 40 | type MoreLikeThisQuery struct { 41 | MoreLikeThis MLT `json:"more_like_this"` 42 | } 43 | 44 | type MLT struct { 45 | Fields []string `json:"fields"` 46 | LikeText string `json:"like_text"` 47 | PercentTermsToMatch float32 `json:"percent_terms_to_match"` 48 | MinTermFrequency int `json:"min_term_freq"` 49 | MaxQueryTerms int `json:"max_query_terms"` 50 | StopWords []string `json:"stop_words"` 51 | MinDocFrequency int `json:"min_doc_freq"` 52 | MaxDocFrequency int `json:"max_doc_freq"` 53 | MinWordLength int `json:"min_word_len"` 54 | MaxWordLength int `json:"max_word_len"` 55 | BoostTerms int `json:"boost_terms"` 56 | Boost float32 `json:"boost"` 57 | Analyzer string `json:"analyzer"` 58 | } 59 | -------------------------------------------------------------------------------- /api/request_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // http://www.apache.org/licenses/LICENSE-2.0 5 | // Unless required by applicable law or agreed to in writing, software 6 | // distributed under the License is distributed on an "AS IS" BASIS, 7 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 8 | // See the License for the specific language governing permissions and 9 | // limitations under the License. 10 | 11 | package api 12 | 13 | import ( 14 | "fmt" 15 | "testing" 16 | 17 | "github.com/bmizerany/assert" 18 | ) 19 | 20 | func TestQueryString(t *testing.T) { 21 | // Test nil argument 22 | s, err := QueryString(nil) 23 | assert.T(t, s == "" && err == nil, fmt.Sprintf("Nil should not fail and yield empty string")) 24 | 25 | // Test single string argument 26 | s, err = QueryString(map[string]interface{}{"foo": "bar"}) 27 | exp := "foo=bar" 28 | assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s)) 29 | 30 | // Test single int argument 31 | s, err = QueryString(map[string]interface{}{"foo": 1}) 32 | exp = "foo=1" 33 | assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s)) 34 | 35 | // Test single float argument 36 | s, err = QueryString(map[string]interface{}{"foo": 3.141592}) 37 | exp = "foo=3.141592" 38 | assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s)) 39 | 40 | // Test single []string argument 41 | s, err = QueryString(map[string]interface{}{"foo": []string{"bar", "baz"}}) 42 | exp = "foo=bar%2Cbaz" 43 | assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s)) 44 | 45 | // Test combination of all arguments 46 | s, err = QueryString(map[string]interface{}{ 47 | "foo": "bar", 48 | "bar": 1, 49 | "baz": 3.141592, 50 | "test": []string{"a", "b"}, 51 | }) 52 | // url.Values also orders arguments alphabetically. 53 | exp = "bar=1&baz=3.141592&foo=bar&test=a%2Cb" 54 | assert.T(t, s == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, s)) 55 | 56 | // Test invalid datatype 57 | s, err = QueryString(map[string]interface{}{"foo": []int{}}) 58 | assert.T(t, err != nil, fmt.Sprintf("Expected err to not be nil")) 59 | } 60 | -------------------------------------------------------------------------------- /core/percolate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // RegisterPercolate allows the caller to register queries against an index, and then send percolate requests which include a doc, and 21 | // getting back the queries that match on that doc out of the set of registered queries. 22 | // Think of it as the reverse operation of indexing and then searching. Instead of sending docs, indexing them, 23 | // and then running queries. One sends queries, registers them, and then sends docs and finds out which queries 24 | // match that doc. 25 | // see http://www.elasticsearch.org/guide/reference/api/percolate.html 26 | func RegisterPercolate(index string, name string, args map[string]interface{}, query api.Query) (api.BaseResponse, error) { 27 | var url string 28 | var retval api.BaseResponse 29 | url = fmt.Sprintf("/_percolator/%s/%s", index, name) 30 | body, err := api.DoCommand("PUT", url, args, query) 31 | if err != nil { 32 | return retval, err 33 | } 34 | if err == nil { 35 | // marshall into json 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | } 41 | return retval, err 42 | } 43 | 44 | func Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (api.Match, error) { 45 | var url string 46 | var retval api.Match 47 | url = fmt.Sprintf("/%s/%s/_percolate", index, _type) 48 | body, err := api.DoCommand("GET", url, args, doc) 49 | if err != nil { 50 | return retval, err 51 | } 52 | if err == nil { 53 | // marshall into json 54 | jsonErr := json.Unmarshal(body, &retval) 55 | if jsonErr != nil { 56 | return retval, jsonErr 57 | } 58 | } 59 | return retval, err 60 | } 61 | -------------------------------------------------------------------------------- /cluster/clusterreroute.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package cluster 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | "github.com/buger/elastigo/api" 19 | ) 20 | 21 | // The cluster health API allows to get a very simple status on the health of the cluster. 22 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-health.html 23 | // information returned. Defaults to cluster.) 24 | func Reroute(dryRun bool, commands Commands) (api.ClusterHealthResponse, error) { 25 | var url string 26 | var retval api.ClusterHealthResponse 27 | 28 | if len(commands.Commands) > 0 { 29 | url = fmt.Sprintf("/_cluster/reroute%s&%s", dryRunOption(dryRun)) 30 | } else { 31 | return retval, errors.New("Must pass at least one command") 32 | } 33 | m := map[string]interface{}{"commands": commands.Commands} 34 | body, err := api.DoCommand("POST", url, m, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | 48 | func dryRunOption(isDryRun bool) string { 49 | if isDryRun { 50 | return "dry_run" 51 | } else { 52 | return "" 53 | } 54 | return "" 55 | } 56 | 57 | // supported commands are 58 | // move (index, shard, from_node, to_node) 59 | // cancel (index, shard, node, allow_primary) 60 | // allocate (index, shard, node, allow_primary) 61 | 62 | type Commands struct { 63 | Commands []interface{} `json:"commands"` 64 | } 65 | 66 | type MoveCommand struct { 67 | Index string `json:"index"` 68 | Shard string `json:"shard"` 69 | FromNode string `json:"from_node"` 70 | ToNode string `json:"to_node"` 71 | } 72 | 73 | type CancelCommand struct { 74 | Index string `json:"index"` 75 | Shard string `json:"shard"` 76 | Node string `json:"node"` 77 | AllowPrimary bool `json:"allow_primary,omitempty"` 78 | } 79 | type AllocateCommand struct { 80 | Index string `json:"index"` 81 | Shard string `json:"shard"` 82 | Node string `json:"node"` 83 | AllowPrimary bool `json:"allow_primary,omitempty"` 84 | } 85 | -------------------------------------------------------------------------------- /cookbooks/build-essential/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: build-essential 3 | # Recipe:: default 4 | # 5 | # Copyright 2008-2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | require 'chef/shell_out' 21 | 22 | compiletime = node['build_essential']['compiletime'] 23 | 24 | case node['os'] 25 | when "linux" 26 | 27 | # on apt-based platforms when first provisioning we need to force 28 | # apt-get update at compiletime if we are going to try to install at compiletime 29 | if node['platform_family'] == "debian" 30 | execute "apt-get update" do 31 | action :nothing 32 | # tip: to suppress this running every time, just use the apt cookbook 33 | not_if do 34 | ::File.exists?('/var/lib/apt/periodic/update-success-stamp') && 35 | ::File.mtime('/var/lib/apt/periodic/update-success-stamp') > Time.now - 86400*2 36 | end 37 | end.run_action(:run) if compiletime 38 | end 39 | 40 | packages = case node['platform_family'] 41 | when "debian" 42 | %w{build-essential binutils-doc} 43 | when "rhel", "fedora" 44 | %w{gcc gcc-c++ kernel-devel make} 45 | end 46 | 47 | packages.each do |pkg| 48 | r = package pkg do 49 | action ( compiletime ? :nothing : :install ) 50 | end 51 | r.run_action(:install) if compiletime 52 | end 53 | 54 | %w{autoconf flex bison}.each do |pkg| 55 | r = package pkg do 56 | action ( compiletime ? :nothing : :install ) 57 | end 58 | r.run_action(:install) if compiletime 59 | end 60 | when "darwin" 61 | result = Chef::ShellOut.new("pkgutil --pkgs").run_command 62 | installed = result.stdout.split("\n").include?("com.apple.pkg.gcc4.2Leo") 63 | pkg_filename = File.basename(node['build_essential']['osx']['gcc_installer_url']) 64 | pkg_path = "#{Chef::Config[:file_cache_path]}/#{pkg_filename}" 65 | 66 | r = remote_file pkg_path do 67 | source node['build_essential']['osx']['gcc_installer_url'] 68 | checksum node['build_essential']['osx']['gcc_installer_checksum'] 69 | action ( compiletime ? :nothing : :create ) 70 | not_if { installed } 71 | end 72 | r.run_action(:create) if compiletime 73 | 74 | r = execute "sudo installer -pkg \"#{pkg_path}\" -target /" do 75 | action ( compiletime ? :nothing : :run ) 76 | not_if { installed } 77 | end 78 | r.run_action(:run) if compiletime 79 | end 80 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Matthew Baird 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "encoding/json" 18 | "flag" 19 | "github.com/buger/elastigo/api" 20 | "github.com/buger/elastigo/cluster" 21 | "github.com/buger/elastigo/core" 22 | "github.com/buger/elastigo/indices" 23 | "log" 24 | "time" 25 | ) 26 | 27 | var ( 28 | eshost *string = flag.String("host", "localhost", "Elasticsearch Server Host Address") 29 | ) 30 | 31 | // for testing 32 | func main() { 33 | flag.Parse() 34 | log.SetFlags(log.Ltime | log.Lshortfile) 35 | api.Domain = *eshost 36 | core.VerboseLogging = true 37 | response, _ := core.Index("twitter", "tweet", "1", nil, NewTweet("kimchy", "Search is cool")) 38 | indices.Flush() 39 | log.Printf("Index OK: %v", response.Ok) 40 | searchresponse, err := core.SearchRequest("twitter", "tweet", nil, "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}") 41 | if err != nil { 42 | log.Println("error during search:" + err.Error()) 43 | log.Fatal(err) 44 | } 45 | // try marshalling to tweet type 46 | var t Tweet 47 | bytes, err := searchresponse.Hits.Hits[0].Source.MarshalJSON() 48 | if err != nil { 49 | log.Fatalf("err calling marshalJson:%v", err) 50 | } 51 | json.Unmarshal(bytes, t) 52 | log.Printf("Search Found: %s", t) 53 | response, _ = core.Get("twitter", "tweet", "1", nil) 54 | log.Printf("Get: %v", response.Exists) 55 | exists, _ := core.Exists("twitter", "tweet", "1", nil) 56 | log.Printf("Exists: %v", exists) 57 | indices.Flush() 58 | countResponse, _ := core.Count("twitter", "tweet", nil) 59 | log.Printf("Count: %v", countResponse.Count) 60 | response, _ = core.Delete("twitter", "tweet", "1", map[string]interface{}{"version": -1, "routing": ""}) 61 | log.Printf("Delete OK: %v", response.Ok) 62 | response, _ = core.Get("twitter", "tweet", "1", nil) 63 | log.Printf("Get: %v", response.Exists) 64 | 65 | healthResponse, _ := cluster.Health() 66 | log.Printf("Health: %v", healthResponse.Status) 67 | 68 | cluster.UpdateSettings("transient", "discovery.zen.minimum_master_nodes", 2) 69 | 70 | } 71 | 72 | // used in test suite, chosen to be similar to the documentation 73 | type Tweet struct { 74 | User string `json:"user"` 75 | PostDate time.Time `json:"postDate"` 76 | Message string `json:"message"` 77 | } 78 | 79 | func NewTweet(user string, message string) Tweet { 80 | return Tweet{User: user, PostDate: time.Now(), Message: message} 81 | } 82 | 83 | func (t *Tweet) String() string { 84 | b, _ := json.Marshal(t) 85 | return string(b) 86 | } 87 | -------------------------------------------------------------------------------- /cluster/health.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | package cluster 12 | 13 | import ( 14 | "encoding/json" 15 | "fmt" 16 | "github.com/buger/elastigo/api" 17 | "strings" 18 | ) 19 | 20 | // The cluster health API allows to get a very simple status on the health of the cluster. 21 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-health.html 22 | // TODO: implement wait_for_status, timeout, wait_for_relocating_shards, wait_for_nodes 23 | // TODO: implement level (Can be one of cluster, indices or shards. Controls the details level of the health 24 | // information returned. Defaults to cluster.) 25 | func Health(indices ...string) (api.ClusterHealthResponse, error) { 26 | var url string 27 | var retval api.ClusterHealthResponse 28 | if len(indices) > 0 { 29 | url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ",")) 30 | } else { 31 | url = "/_cluster/health" 32 | } 33 | body, err := api.DoCommand("GET", url, nil, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | //fmt.Println(body) 45 | return retval, err 46 | } 47 | 48 | type ClusterStateFilter struct { 49 | FilterNodes bool 50 | FilterRoutingTable bool 51 | FilterMetadata bool 52 | FilterBlocks bool 53 | FilterIndices []string 54 | } 55 | 56 | func (f ClusterStateFilter) Parameterize() []string { 57 | var parts []string 58 | 59 | if f.FilterNodes { 60 | parts = append(parts, "filter_nodes=true") 61 | } 62 | 63 | if f.FilterRoutingTable { 64 | parts = append(parts, "filter_routing_table=true") 65 | } 66 | 67 | if f.FilterMetadata { 68 | parts = append(parts, "filter_metadata=true") 69 | } 70 | 71 | if f.FilterBlocks { 72 | parts = append(parts, "filter_blocks=true") 73 | } 74 | 75 | if f.FilterIndices != nil && len(f.FilterIndices) > 0 { 76 | parts = append(parts, strings.Join([]string{"filter_indices=", strings.Join(f.FilterIndices, ",")}, "")) 77 | } 78 | 79 | return parts 80 | } 81 | 82 | func ClusterState(filter ClusterStateFilter) (api.ClusterStateResponse, error) { 83 | var parameters []string 84 | var url string 85 | var retval api.ClusterStateResponse 86 | 87 | parameters = filter.Parameterize() 88 | 89 | url = fmt.Sprintf("/_cluster/state?%s", strings.Join(parameters, "&")) 90 | 91 | body, err := api.DoCommand("GET", url, nil, nil) 92 | if err != nil { 93 | return retval, err 94 | } 95 | if err == nil { 96 | // marshall into json 97 | jsonErr := json.Unmarshal(body, &retval) 98 | if jsonErr != nil { 99 | return retval, jsonErr 100 | } 101 | } 102 | return retval, err 103 | 104 | } 105 | -------------------------------------------------------------------------------- /core/example_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core_test 13 | 14 | import ( 15 | "bytes" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "github.com/buger/elastigo/core" 19 | "strconv" 20 | "time" 21 | ) 22 | 23 | // The simplest usage of background bulk indexing 24 | func ExampleBulkIndexer_simple() { 25 | indexer := core.NewBulkIndexerErrors(10, 60) 26 | done := make(chan bool) 27 | indexer.Run(done) 28 | 29 | indexer.Index("twitter", "user", "1", "", nil, `{"name":"bob"}`, true) 30 | 31 | <-done // wait forever 32 | } 33 | 34 | // The simplest usage of background bulk indexing with error channel 35 | func ExampleBulkIndexer_errorchannel() { 36 | indexer := core.NewBulkIndexerErrors(10, 60) 37 | done := make(chan bool) 38 | indexer.Run(done) 39 | 40 | go func() { 41 | for errBuf := range indexer.ErrorChannel { 42 | // just blissfully print errors forever 43 | fmt.Println(errBuf.Err) 44 | } 45 | }() 46 | for i := 0; i < 20; i++ { 47 | indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true) 48 | } 49 | done <- true 50 | } 51 | 52 | // The simplest usage of background bulk indexing with error channel 53 | func ExampleBulkIndexer_errorsmarter() { 54 | indexer := core.NewBulkIndexerErrors(10, 60) 55 | done := make(chan bool) 56 | indexer.Run(done) 57 | 58 | errorCt := 0 // use sync.atomic or something if you need 59 | timer := time.NewTicker(time.Minute * 3) 60 | go func() { 61 | for { 62 | select { 63 | case _ = <-timer.C: 64 | if errorCt < 2 { 65 | errorCt = 0 66 | } 67 | case _ = <-done: 68 | return 69 | } 70 | } 71 | }() 72 | 73 | go func() { 74 | for errBuf := range indexer.ErrorChannel { 75 | errorCt++ 76 | fmt.Println(errBuf.Err) 77 | // log to disk? db? ???? Panic 78 | } 79 | }() 80 | for i := 0; i < 20; i++ { 81 | indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true) 82 | } 83 | done <- true // send shutdown signal 84 | } 85 | 86 | // The inspecting the response 87 | func ExampleBulkIndexer_responses() { 88 | indexer := core.NewBulkIndexer(10) 89 | // Create a custom Sender Func, to allow inspection of response/error 90 | indexer.BulkSender = func(buf *bytes.Buffer) error { 91 | // @buf is the buffer of docs about to be written 92 | respJson, err := api.DoCommand("POST", "/_bulk", nil, buf) 93 | if err != nil { 94 | // handle it better than this 95 | fmt.Println(string(respJson)) 96 | } 97 | return err 98 | } 99 | done := make(chan bool) 100 | indexer.Run(done) 101 | 102 | for i := 0; i < 20; i++ { 103 | indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, true) 104 | } 105 | done <- true // send shutdown signal 106 | } 107 | -------------------------------------------------------------------------------- /api/baseRequest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package api 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "io" 18 | "log" 19 | "time" 20 | ) 21 | 22 | func DoCommand(method string, url string, args map[string]interface{}, data interface{}) ([]byte, error) { 23 | var response map[string]interface{} 24 | var body []byte 25 | var httpStatusCode int 26 | 27 | query, err := QueryString(args) 28 | if err != nil { 29 | return nil, err 30 | } 31 | req, err := ElasticSearchRequest(method, url, query) 32 | if err != nil { 33 | return body, err 34 | } 35 | 36 | if data != nil { 37 | switch v := data.(type) { 38 | case string: 39 | req.SetBodyString(v) 40 | case io.Reader: 41 | req.SetBody(v) 42 | case []byte: 43 | req.SetBodyBytes(v) 44 | default: 45 | err = req.SetBodyJson(v) 46 | if err != nil { 47 | return body, err 48 | } 49 | } 50 | 51 | } 52 | httpStatusCode, body, err = req.Do(&response) 53 | 54 | if err != nil { 55 | return body, err 56 | } 57 | if httpStatusCode > 304 { 58 | 59 | jsonErr := json.Unmarshal(body, &response) 60 | if jsonErr == nil { 61 | if res_err, ok := response["error"]; ok { 62 | status, _ := response["status"] 63 | return body, ESError{time.Now(), fmt.Sprintf("Error [%s] Status [%v]", res_err, status), httpStatusCode} 64 | } 65 | } 66 | return body, jsonErr 67 | } 68 | return body, nil 69 | } 70 | 71 | // ESError is an error implementation that includes a time, message, and code. 72 | type ESError struct { 73 | When time.Time 74 | What string 75 | Code int 76 | } 77 | 78 | func (e ESError) Error() string { 79 | return fmt.Sprintf("%v: %v [%v]", e.When, e.What, e.Code) 80 | } 81 | 82 | // Exists allows the caller to check for the existance of a document using HEAD 83 | // This appears to be broken in the current version of elasticsearch 0.19.10, currently 84 | // returning nothing 85 | func Exists(index string, _type string, id string, args map[string]interface{}) (BaseResponse, error) { 86 | var response map[string]interface{} 87 | var body []byte 88 | var url string 89 | var retval BaseResponse 90 | var httpStatusCode int 91 | 92 | query, err := QueryString(args) 93 | if err != nil { 94 | return retval, err 95 | } 96 | 97 | if len(_type) > 0 { 98 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 99 | } else { 100 | url = fmt.Sprintf("/%s/%s", index, id) 101 | } 102 | req, err := ElasticSearchRequest("HEAD", url, query) 103 | if err != nil { 104 | // some sort of generic error handler 105 | } 106 | httpStatusCode, body, err = req.Do(&response) 107 | if httpStatusCode > 304 { 108 | if error, ok := response["error"]; ok { 109 | status, _ := response["status"] 110 | log.Printf("Error: %v (%v)\n", error, status) 111 | } 112 | } else { 113 | // marshall into json 114 | jsonErr := json.Unmarshal(body, &retval) 115 | if jsonErr != nil { 116 | log.Println(jsonErr) 117 | } 118 | } 119 | return retval, err 120 | } 121 | -------------------------------------------------------------------------------- /indices/putMapping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | api "github.com/buger/elastigo/api" 18 | "reflect" 19 | "strings" 20 | ) 21 | 22 | type Mapping map[string]MappingOptions 23 | 24 | type MappingOptions struct { 25 | Id IdOptions `json:"_id"` 26 | Timestamp TimestampOptions `json:"_timestamp"` 27 | Properties map[string]interface{} `json:"properties"` 28 | } 29 | 30 | type TimestampOptions struct { 31 | Enabled bool `json:"enabled"` 32 | } 33 | 34 | type IdOptions struct { 35 | Index string `json:"index,omitempty"` 36 | Path string `json:"path,omitempty"` 37 | } 38 | 39 | func (m_ Mapping) Options() MappingOptions { 40 | m := map[string]MappingOptions(m_) 41 | for _, v := range m { 42 | return v 43 | } 44 | panic(fmt.Errorf("Malformed input: %v", m_)) 45 | } 46 | 47 | func MappingForType(typeName string, opts MappingOptions) Mapping { 48 | return map[string]MappingOptions{typeName: opts} 49 | } 50 | 51 | func PutMapping(index string, typeName string, instance interface{}, opt MappingOptions) error { 52 | instanceType := reflect.TypeOf(instance) 53 | if instanceType.Kind() != reflect.Struct { 54 | return fmt.Errorf("instance kind was not struct") 55 | } 56 | 57 | if opt.Properties == nil { 58 | opt.Properties = make(map[string]interface{}) 59 | } 60 | getProperties(instanceType, opt.Properties) 61 | body, err := json.Marshal(MappingForType(typeName, opt)) 62 | if err != nil { 63 | return err 64 | } 65 | _, err = api.DoCommand("PUT", fmt.Sprintf("/%s/%s/_mapping", index, typeName), nil, string(body)) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func getProperties(t reflect.Type, prop map[string]interface{}) { 74 | n := t.NumField() 75 | for i := 0; i < n; i++ { 76 | field := t.Field(i) 77 | 78 | name := strings.Split(field.Tag.Get("json"), ",")[0] 79 | if name == "-" { 80 | continue 81 | } else if name == "" { 82 | name = field.Name 83 | } 84 | 85 | attrMap := make(map[string]string) 86 | tag := field.Tag.Get("elastic") 87 | if tag == "" { 88 | 89 | // We are looking for tags on any nested struct, independently of 90 | // whether the field is a struct, a pointer to struct, or a slice of structs 91 | targetType := field.Type 92 | if targetType.Kind() == reflect.Ptr || 93 | targetType.Kind() == reflect.Slice { 94 | targetType = field.Type.Elem() 95 | } 96 | 97 | if targetType.Kind() == reflect.Struct { 98 | if field.Anonymous { 99 | getProperties(targetType, prop) 100 | } else { 101 | nestedProp := make(map[string]interface{}) 102 | getProperties(targetType, nestedProp) 103 | prop[name] = map[string]interface{}{ 104 | "properties": nestedProp, 105 | } 106 | } 107 | } 108 | continue 109 | } 110 | attrs := strings.Split(tag, ",") 111 | for _, attr := range attrs { 112 | keyvalue := strings.Split(attr, ":") 113 | attrMap[keyvalue[0]] = keyvalue[1] 114 | } 115 | prop[name] = attrMap 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /search/facet.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "encoding/json" 16 | 17 | u "github.com/araddon/gou" 18 | ) 19 | 20 | var ( 21 | _ = u.DEBUG 22 | ) 23 | 24 | /* 25 | "facets": { 26 | "terms": { 27 | "terms": { 28 | "field": [ 29 | "@fields.category" 30 | ], 31 | "size": 25 32 | } 33 | } 34 | } 35 | 36 | 37 | "facets": { 38 | "actors": { "terms": {"field": ["actor"],"size": "10" }} 39 | , "langauge": { "terms": {"field": ["repository.language"],"size": "10" }} 40 | } 41 | 42 | */ 43 | func Facet() *FacetDsl { 44 | return &FacetDsl{} 45 | } 46 | 47 | func FacetRange(field string) *RangeDsl { 48 | out := &RangeDsl{&RangeDef{}, nil} 49 | out.RangeDef.Field = field 50 | return out 51 | } 52 | 53 | type FacetDsl struct { 54 | size string 55 | Terms map[string]*Term `json:"terms,omitempty"` 56 | Ranges map[string]*RangeDsl `json:"terms,omitempty"` 57 | } 58 | 59 | type RangeDsl struct { 60 | RangeDef *RangeDef `json:"range,omitempty"` 61 | FilterVal *FilterWrap `json:"facet_filter,omitempty"` 62 | } 63 | 64 | type RangeDef struct { 65 | Field string `json:"field,omitempty"` 66 | Values []*RangeVal `json:"ranges,omitempty"` 67 | } 68 | 69 | type RangeVal struct { 70 | From string `json:"from,omitempty"` 71 | To string `json:"to,omitempty"` 72 | } 73 | 74 | func (m *RangeDsl) Range(from, to string) *RangeDsl { 75 | if len(m.RangeDef.Values) == 0 { 76 | m.RangeDef.Values = make([]*RangeVal, 0) 77 | } 78 | 79 | m.RangeDef.Values = append(m.RangeDef.Values, &RangeVal{From: from, To: to}) 80 | return m 81 | } 82 | 83 | func (s *RangeDsl) Filter(fl ...interface{}) *RangeDsl { 84 | if s.FilterVal == nil { 85 | s.FilterVal = NewFilterWrap() 86 | } 87 | 88 | s.FilterVal.addFilters(fl) 89 | return s 90 | } 91 | 92 | func (m *FacetDsl) Size(size string) *FacetDsl { 93 | m.size = size 94 | return m 95 | } 96 | 97 | func (m *FacetDsl) Fields(fields ...string) *FacetDsl { 98 | if len(fields) < 1 { 99 | return m 100 | } 101 | if len(m.Terms) == 0 { 102 | m.Terms = make(map[string]*Term) 103 | } 104 | m.Terms[fields[0]] = &Term{Terms{Fields: fields}, nil} 105 | return m 106 | } 107 | 108 | func (m *FacetDsl) Regex(field, match string) *FacetDsl { 109 | if len(m.Terms) == 0 { 110 | m.Terms = make(map[string]*Term) 111 | } 112 | m.Terms[field] = &Term{Terms{Fields: []string{field}, Regex: match}, nil} 113 | return m 114 | } 115 | 116 | func (m *FacetDsl) Term(t *Term) *FacetDsl { 117 | if len(m.Terms) == 0 { 118 | m.Terms = make(map[string]*Term) 119 | } 120 | m.Terms[t.Terms.Fields[0]] = t 121 | return m 122 | } 123 | 124 | func (m *FacetDsl) Range(r *RangeDsl) *FacetDsl { 125 | if len(m.Ranges) == 0 { 126 | m.Ranges = make(map[string]*RangeDsl) 127 | } 128 | m.Ranges[r.RangeDef.Field] = r 129 | return m 130 | } 131 | 132 | func (m *FacetDsl) MarshalJSON() ([]byte, error) { 133 | data := map[string]interface{}{} 134 | for key, t := range m.Terms { 135 | t.Terms.Size = m.size 136 | data[key] = t 137 | } 138 | for key, r := range m.Ranges { 139 | data[key] = r 140 | } 141 | return json.Marshal(&data) 142 | } 143 | -------------------------------------------------------------------------------- /core/get.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "net/http" 19 | ) 20 | 21 | // Get allows caller to get a typed JSON document from the index based on its id. 22 | // GET - retrieves the doc 23 | // HEAD - checks for existence of the doc 24 | // http://www.elasticsearch.org/guide/reference/api/get.html 25 | // TODO: make this implement an interface 26 | func get(index string, _type string, id string, args map[string]interface{}, source interface{}) (api.BaseResponse, error) { 27 | var url string 28 | retval := api.BaseResponse{Source: source} 29 | if len(_type) > 0 { 30 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 31 | } else { 32 | url = fmt.Sprintf("/%s/%s", index, id) 33 | } 34 | body, err := api.DoCommand("GET", url, args, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | 48 | // The get API allows to get a typed JSON document from the index based on its id. 49 | // GET - retrieves the doc 50 | // HEAD - checks for existence of the doc 51 | // http://www.elasticsearch.org/guide/reference/api/get.html 52 | // TODO: make this implement an interface 53 | func Get(index string, _type string, id string, args map[string]interface{}) (api.BaseResponse, error) { 54 | return get(index, _type, id, args, nil) 55 | } 56 | 57 | // Same as Get but with custom source type. 58 | func GetCustom(index string, _type string, id string, args map[string]interface{}, source interface{}) (api.BaseResponse, error) { 59 | return get(index, _type, id, args, source) 60 | } 61 | 62 | // GetSource retrieves the document by id and converts it to provided interface 63 | func GetSource(index string, _type string, id string, args map[string]interface{}, source interface{}) error { 64 | url := fmt.Sprintf("/%s/%s/%s/_source", index, _type, id) 65 | body, err := api.DoCommand("GET", url, args, nil) 66 | if err == nil { 67 | err = json.Unmarshal(body, &source) 68 | } 69 | return err 70 | } 71 | 72 | // Exists allows caller to check for the existance of a document using HEAD 73 | func Exists(index string, _type string, id string, args map[string]interface{}) (bool, error) { 74 | 75 | var url string 76 | 77 | query, err := api.QueryString(args) 78 | if err != nil { 79 | return false, err 80 | } 81 | 82 | if len(_type) > 0 { 83 | url = fmt.Sprintf("/%s/%s/%s?fields=_id", index, _type, id) 84 | } else { 85 | url = fmt.Sprintf("/%s/%s?fields=_id", index, id) 86 | } 87 | 88 | req, err := api.ElasticSearchRequest("HEAD", url, query) 89 | 90 | if err != nil { 91 | return false, err 92 | } 93 | 94 | httpStatusCode, _, err := req.Do(nil) 95 | 96 | if err != nil { 97 | return false, err 98 | } 99 | if httpStatusCode == http.StatusOK { 100 | return true, err 101 | } 102 | return false, err 103 | } 104 | 105 | // ExistsIndex allows caller to check for the existance of an index or a type using HEAD 106 | func ExistsIndex(index string, _type string, args map[string]interface{}) (bool, error) { 107 | var url string 108 | 109 | query, err := api.QueryString(args) 110 | if err != nil { 111 | return false, err 112 | } 113 | 114 | if len(_type) > 0 { 115 | url = fmt.Sprintf("/%s/%s", index, _type) 116 | } else { 117 | url = fmt.Sprintf("/%s", index) 118 | } 119 | req, err := api.ElasticSearchRequest("HEAD", url, query) 120 | httpStatusCode, _, err := req.Do(nil) 121 | 122 | if err != nil { 123 | return false, err 124 | } 125 | if httpStatusCode == http.StatusOK { 126 | return true, err 127 | } 128 | return false, err 129 | } 130 | -------------------------------------------------------------------------------- /core/update.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | ) 19 | 20 | // Update updates a document based on a script provided. The operation gets the document 21 | // (collocated with the shard) from the index, runs the script (with optional script language and parameters), 22 | // and index back the result (also allows to delete, or ignore the operation). It uses versioning to make sure 23 | // no updates have happened during the “get” and “reindex”. (available from 0.19 onwards). 24 | // Note, this operation still means full reindex of the document, it just removes some network roundtrips 25 | // and reduces chances of version conflicts between the get and the index. The _source field need to be enabled 26 | // for this feature to work. 27 | // 28 | // http://www.elasticsearch.org/guide/reference/api/update.html 29 | // TODO: finish this, it's fairly complex 30 | func Update(index string, _type string, id string, args map[string]interface{}, data interface{}) (api.BaseResponse, error) { 31 | var url string 32 | var retval api.BaseResponse 33 | 34 | url = fmt.Sprintf("/%s/%s/%s/_update", index, _type, id) 35 | body, err := api.DoCommand("POST", url, args, data) 36 | if err != nil { 37 | return retval, err 38 | } 39 | if err == nil { 40 | // marshall into json 41 | jsonErr := json.Unmarshal(body, &retval) 42 | if jsonErr != nil { 43 | return retval, jsonErr 44 | } 45 | } 46 | return retval, err 47 | } 48 | 49 | // UpdateWithPartialDoc updates a document based on partial document provided. The update API also 50 | // support passing a partial document (since 0.20), which will be merged into the existing 51 | // document (simple recursive merge, inner merging of objects, replacing core "keys/values" and arrays). 52 | // If both doc and script is specified, then doc is ignored. Best is to put your field pairs of the partial 53 | // document in the script itself. 54 | // 55 | // http://www.elasticsearch.org/guide/reference/api/update.html 56 | func UpdateWithPartialDoc(index string, _type string, id string, args map[string]interface{}, doc interface{}, upsert bool) (api.BaseResponse, error) { 57 | switch v := doc.(type) { 58 | case string: 59 | upsertStr := "" 60 | if upsert { 61 | upsertStr = ", \"doc_as_upsert\":true" 62 | } 63 | content := fmt.Sprintf("{\"doc\":%s %s}", v, upsertStr) 64 | return Update(index, _type, id, args, content) 65 | } 66 | var data map[string]interface{} = make(map[string]interface{}) 67 | data["doc"] = doc 68 | if upsert { 69 | data["doc_as_upsert"] = true 70 | } 71 | return Update(index, _type, id, args, data) 72 | } 73 | 74 | // UpdateWithScript updates a document based on a script provided. 75 | // The operation gets the document (collocated with the shard) from the index, runs the script 76 | // (with optional script language and parameters), and index back the result (also allows to 77 | // delete, or ignore the operation). It uses versioning to make sure no updates have happened 78 | // during the "get" and "reindex". (available from 0.19 onwards). 79 | // 80 | // Note, this operation still means full reindex of the document, it just removes some network 81 | // roundtrips and reduces chances of version conflicts between the get and the index. The _source 82 | // field need to be enabled for this feature to work. 83 | // http://www.elasticsearch.org/guide/reference/api/update.html 84 | func UpdateWithScript(index string, _type string, id string, args map[string]interface{}, script string, params interface{}) (api.BaseResponse, error) { 85 | switch v := params.(type) { 86 | case string: 87 | paramsPart := fmt.Sprintf("{\"params\":%s}", v) 88 | data := fmt.Sprintf("{\"script\":\"%s\", \"params\":%s}", script, paramsPart) 89 | return Update(index, _type, id, args, data) 90 | } 91 | var data map[string]interface{} = make(map[string]interface{}) 92 | data["params"] = params 93 | data["script"] = script 94 | return Update(index, _type, id, args, data) 95 | } 96 | -------------------------------------------------------------------------------- /search/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "fmt" 16 | //"github.com/araddon/gou" 17 | "github.com/bmizerany/assert" 18 | "testing" 19 | ) 20 | 21 | func TestFilters(t *testing.T) { 22 | // search for docs that are missing repository.name 23 | qry := Search("github").Filter( 24 | Filter().Exists("repository.name"), 25 | ) 26 | out, err := qry.Result() 27 | assert.T(t, err == nil, t, "should not have error") 28 | expectedDocs := 10 29 | expectedHits := 7695 30 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 31 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 32 | qry = Search("github").Filter( 33 | Filter().Missing("repository.name"), 34 | ) 35 | expectedHits = 389 36 | out, _ = qry.Result() 37 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 38 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 39 | 40 | //actor_attributes: {type: "User", 41 | qry = Search("github").Filter( 42 | Filter().Terms("actor_attributes.location", "portland"), 43 | ) 44 | out, _ = qry.Result() 45 | expectedDocs = 10 46 | expectedHits = 71 47 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 48 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 49 | 50 | /* 51 | Should this be an AND by default? 52 | */ 53 | qry = Search("github").Filter( 54 | Filter().Terms("actor_attributes.location", "portland"), 55 | Filter().Terms("repository.has_wiki", true), 56 | ) 57 | out, err = qry.Result() 58 | expectedDocs = 10 59 | expectedHits = 44 60 | assert.T(t, err == nil, t, "should not have error") 61 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 62 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 63 | 64 | // NOW, lets try with two query calls instead of one 65 | qry = Search("github").Filter( 66 | Filter().Terms("actor_attributes.location", "portland"), 67 | ) 68 | qry.Filter( 69 | Filter().Terms("repository.has_wiki", true), 70 | ) 71 | out, err = qry.Result() 72 | //gou.Debug(out) 73 | assert.T(t, err == nil, t, "should not have error") 74 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 75 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 76 | 77 | qry = Search("github").Filter( 78 | "or", 79 | Filter().Terms("actor_attributes.location", "portland"), 80 | Filter().Terms("repository.has_wiki", true), 81 | ) 82 | out, err = qry.Result() 83 | expectedHits = 6676 84 | assert.T(t, err == nil, t, "should not have error") 85 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 86 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have %v total got %v", expectedHits, out.Hits.Total)) 87 | } 88 | 89 | func TestFilterRange(t *testing.T) { 90 | 91 | // now lets filter range for repositories with more than 100 forks 92 | out, _ := Search("github").Size("25").Filter( 93 | Range().Field("repository.forks").From("100"), 94 | ).Result() 95 | if out == nil || &out.Hits == nil { 96 | t.Fail() 97 | return 98 | } 99 | expectedDocs := 25 100 | expectedHits := 725 101 | 102 | assert.T(t, out.Hits.Len() == expectedDocs, fmt.Sprintf("Should have %v docs got %v", expectedDocs, out.Hits.Len())) 103 | assert.T(t, out.Hits.Total == expectedHits, fmt.Sprintf("Should have total %v got %v", expectedHits, out.Hits.Total)) 104 | } 105 | -------------------------------------------------------------------------------- /cookbooks/build-essential/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs packages required for compiling C software from source. Use 5 | this cookbook if you wish to compile C programs, or install RubyGems 6 | with native extensions. 7 | 8 | Requirements 9 | ============ 10 | 11 | ## Platform 12 | 13 | Supported platforms by platform family: 14 | 15 | * Linux (debian, rhel, fedora) 16 | * Darwin (`mac_os_x` 10.6+) 17 | 18 | Attributes 19 | ========== 20 | 21 | * `node['build_essential']['compiletime']` - Whether the resources in 22 | the default recipe should be configured at the "Compile" phase of the 23 | Chef run. Defaults to false, see __Usage__ for more information. 24 | * `node['build_essential']['osx']['gcc_installer_url']` - The URL of 25 | the OS X GCC package installer (.pkg). 26 | * `node['build_essential']['osx']['gcc_installer_checksum']` - The 27 | SHA256 checksum of the OS X GCC installer. 28 | 29 | Recipes 30 | ======= 31 | 32 | This cookbook has one recipe, default. 33 | 34 | On Linux platforms (see __Platform__ above for a supported list of 35 | families), packages required to build C source projects are installed. 36 | This includes GCC, make, autconf and others. On Debian-family 37 | distributions, the apt-cache may need to be updated, especially during 38 | compile time installation. See __Usage__ for further information. 39 | 40 | On Mac OS X, the GCC standalone installer by Kenneth Reitz is 41 | installed. Note that this is *not* the Xcode CLI package, as that does 42 | not include all programs and headers required to build some common 43 | GNU-style C projects, such as those that are available from projects 44 | such as MacPorts or Homebrew. Changing the attributes for the GCC 45 | installer URL and checksum to the Xcode values may work, but this is 46 | untested. 47 | 48 | Usage 49 | ===== 50 | 51 | Simply include the `build-essential` and the required tools will be 52 | installed to the system, and later recipes will be able to compile 53 | software from C source code. 54 | 55 | For RubyGems that include native C extensions you wish to use with 56 | Chef, you should do two things. 57 | 58 | 0. Ensure that the C libraries, include files and other assorted "dev" 59 | type packages are installed. You should do this in the compile phase 60 | after the build-essential recipe. 61 | 1. Use the `chef_gem` resource in your recipes. This requires Chef version 0.10.10+. 62 | 2. Set the `compiletime` attribute in roles where such recipes are 63 | required. This will ensure that the build tools are available to 64 | compile the RubyGems' extensions, as `chef_gem` happens during the 65 | compile phase, too. 66 | 67 | Example installation of a devel package at compile-time in a recipe: 68 | 69 | package "mypackage-dev" do 70 | action :nothing 71 | end.run_action(:install) 72 | 73 | Example use of `chef_gem`: 74 | 75 | chef_gem "mygem" 76 | 77 | Example role: 78 | 79 | name "myapp" 80 | run_list( 81 | "recipe[build-essential]", 82 | "recipe[myapp]" 83 | ) 84 | default_attributes( 85 | "build_essential" => { 86 | "compiletime" => true 87 | } 88 | ) 89 | 90 | The compile time option (via the attribute) is to ensure that the 91 | proper packages are available at the right time in the Chef run. It is 92 | recommended that the build-essential recipe appear early in the run 93 | list. 94 | 95 | The Chef wiki has documentation on 96 | [the anatomy of a chef run](http://wiki.opscode.com/display/chef/Anatomy+of+a+Chef+Run). 97 | 98 | Limitations 99 | =========== 100 | 101 | It is not in the scope of this cookbook to handle installing the 102 | required headers for individual software projects in order to compile 103 | them, or to compile RubyGems with native C extensions. You should 104 | create a cookbook for handling that. 105 | 106 | License and Author 107 | ================== 108 | 109 | Author:: Joshua Timberman () 110 | Author:: Seth Chisamore () 111 | 112 | Copyright 2009-2011, Opscode, Inc. () 113 | 114 | Licensed under the Apache License, Version 2.0 (the "License"); 115 | you may not use this file except in compliance with the License. 116 | You may obtain a copy of the License at 117 | 118 | http://www.apache.org/licenses/LICENSE-2.0 119 | 120 | Unless required by applicable law or agreed to in writing, software 121 | distributed under the License is distributed on an "AS IS" BASIS, 122 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 123 | See the License for the specific language governing permissions and 124 | limitations under the License. 125 | -------------------------------------------------------------------------------- /search/aggregate_test.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | // Test all aggregate types and nested aggregations 10 | func TestAggregateDsl(t *testing.T) { 11 | 12 | min := Aggregate("min_price").Min("price") 13 | max := Aggregate("max_price").Max("price") 14 | sum := Aggregate("sum_price").Sum("price") 15 | avg := Aggregate("avg_price").Avg("price") 16 | stats := Aggregate("stats_price").Stats("price") 17 | extendedStats := Aggregate("extended_stats_price").ExtendedStats("price") 18 | valueCount := Aggregate("value_count_price").ValueCount("price") 19 | percentiles := Aggregate("percentiles_price").Percentiles("price") 20 | cardinality := Aggregate("cardinality_price").Cardinality("price", true, 50) 21 | global := Aggregate("global").Global() 22 | missing := Aggregate("missing_price").Missing("price") 23 | terms := Aggregate("terms_price").Terms("price") 24 | significantTerms := Aggregate("significant_terms_price").SignificantTerms("price") 25 | histogram := Aggregate("histogram_price").Histogram("price", 50) 26 | 27 | dateAgg := Aggregate("articles_over_time").DateHistogram("date", "month") 28 | dateAgg.Aggregates( 29 | min, 30 | max, 31 | sum, 32 | avg, 33 | stats, 34 | extendedStats, 35 | valueCount, 36 | percentiles, 37 | cardinality, 38 | global, 39 | missing, 40 | terms, 41 | significantTerms, 42 | histogram, 43 | ) 44 | 45 | qry := Search("github").Aggregates(dateAgg) 46 | 47 | marshaled, err := json.MarshalIndent(qry.AggregatesVal, "", " ") 48 | if err != nil { 49 | t.Errorf("Failed to marshal AggregatesVal: %s", err.Error()) 50 | return 51 | } 52 | 53 | assertJsonMatch( 54 | t, 55 | marshaled, 56 | []byte(` 57 | { 58 | "articles_over_time": { 59 | "date_histogram" : { 60 | "field" : "date", 61 | "interval" : "month" 62 | }, 63 | "aggregations": { 64 | "min_price":{ 65 | "min": { "field": "price" } 66 | }, 67 | "max_price":{ 68 | "max": { "field": "price" } 69 | }, 70 | "sum_price":{ 71 | "sum": { "field": "price" } 72 | }, 73 | "avg_price": { 74 | "avg": { "field": "price" } 75 | }, 76 | "stats_price":{ 77 | "stats": { "field": "price" } 78 | }, 79 | "extended_stats_price":{ 80 | "extended_stats": { "field": "price" } 81 | }, 82 | "value_count_price":{ 83 | "value_count": { "field": "price" } 84 | }, 85 | "percentiles_price":{ 86 | "percentiles": { "field": "price" } 87 | }, 88 | "cardinality_price":{ 89 | "cardinality": { "field": "price", "precision_threshold": 50 } 90 | }, 91 | "global":{ 92 | "global": {} 93 | }, 94 | "missing_price":{ 95 | "missing": { "field": "price" } 96 | }, 97 | "terms_price":{ 98 | "terms": { "field": "price" } 99 | }, 100 | "significant_terms_price":{ 101 | "significant_terms": { "field": "price" } 102 | }, 103 | "histogram_price":{ 104 | "histogram": { "field": "price", "interval": 50 } 105 | } 106 | } 107 | } 108 | } 109 | `), 110 | ) 111 | 112 | } 113 | 114 | func TestAggregateFilter(t *testing.T) { 115 | 116 | avg := Aggregate("avg_price").Avg("price") 117 | 118 | dateAgg := Aggregate("in_stock_products").Filter( 119 | Range().Field("stock").Gt(0), 120 | ) 121 | 122 | dateAgg.Aggregates( 123 | avg, 124 | ) 125 | 126 | qry := Search("github").Aggregates(dateAgg) 127 | 128 | marshaled, err := json.MarshalIndent(qry.AggregatesVal, "", " ") 129 | if err != nil { 130 | t.Errorf("Failed to marshal AggregatesVal: %s", err.Error()) 131 | return 132 | } 133 | 134 | assertJsonMatch( 135 | t, 136 | marshaled, 137 | []byte(` 138 | { 139 | "in_stock_products" : { 140 | "filter" : { 141 | "range" : { "stock" : { "gt" : 0 } } 142 | }, 143 | "aggregations" : { 144 | "avg_price" : { "avg" : { "field" : "price" } } 145 | } 146 | } 147 | } 148 | `), 149 | ) 150 | } 151 | 152 | func assertJsonMatch(t *testing.T, match, expected []byte) { 153 | var m interface{} 154 | var e interface{} 155 | 156 | err := json.Unmarshal(expected, &e) 157 | if err != nil { 158 | t.Errorf("Failed to unmarshal expectation: %s", err.Error()) 159 | return 160 | } 161 | err = json.Unmarshal(match, &m) 162 | if err != nil { 163 | t.Errorf("Failed to unmarshal match: %s", err.Error()) 164 | return 165 | } 166 | 167 | if !reflect.DeepEqual(m, e) { 168 | t.Errorf("Expected %s but got %s", string(expected), string(match)) 169 | return 170 | } 171 | 172 | } 173 | -------------------------------------------------------------------------------- /core/index.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | "log" 19 | "net/url" 20 | "strconv" 21 | 22 | "github.com/buger/elastigo/api" 23 | ) 24 | 25 | // VerboseLogging controls whether elastigo will log more information 26 | // about its actions. Set it to false for less logging. 27 | var VerboseLogging bool = true; 28 | 29 | // Index adds or updates a typed JSON document in a specific index, making it searchable, creating an index 30 | // if it did not exist. 31 | // if id is omited, op_type 'create' will be passed and http method will default to "POST" 32 | // _type is optional 33 | // id is optional 34 | // parentId is optional 35 | // version is optional 36 | // op_type is optional 37 | // routing is optional 38 | // timestamp is optional 39 | // ttl is optional 40 | // percolate is optional 41 | // timeout is optional 42 | // http://www.elasticsearch.org/guide/reference/api/index_.html 43 | func Index(index string, _type string, id string, args map[string]interface{}, data interface{}) (api.BaseResponse, error) { 44 | return IndexWithParameters(index, _type, id, "", 0, "", "", "", 0, "", "", false, args, data) 45 | } 46 | 47 | // IndexWithParameters takes all the potential parameters available 48 | func IndexWithParameters(index string, _type string, id string, parentId string, version int, op_type string, 49 | routing string, timestamp string, ttl int, percolate string, timeout string, refresh bool, 50 | args map[string]interface{}, data interface{}) (api.BaseResponse, error) { 51 | var url string 52 | var retval api.BaseResponse 53 | url, err := GetIndexUrl(index, _type, id, parentId, version, op_type, routing, timestamp, ttl, percolate, timeout, refresh) 54 | if err != nil { 55 | return retval, err 56 | } 57 | var method string 58 | if len(id) == 0 { 59 | method = "POST" 60 | } else { 61 | method = "PUT" 62 | } 63 | if VerboseLogging { 64 | log.Printf("about to :%v %v %s", url, args, data) 65 | } 66 | body, err := api.DoCommand(method, url, args, data) 67 | if err != nil { 68 | return retval, err 69 | } 70 | if err == nil { 71 | // marshall into json 72 | jsonErr := json.Unmarshal(body, &retval) 73 | if jsonErr != nil { 74 | return retval, jsonErr 75 | } 76 | } 77 | return retval, err 78 | } 79 | 80 | func GetIndexUrl(index string, _type string, id string, parentId string, version int, op_type string, 81 | routing string, timestamp string, ttl int, percolate string, timeout string, refresh bool) (retval string, e error) { 82 | 83 | if len(index) == 0 { 84 | return "", errors.New("index can not be blank") 85 | } 86 | var partialURL string 87 | var values url.Values = url.Values{} 88 | if len(_type) == 0 && len(id) > 0 { 89 | e = errors.New("Can't specify id when _type is blank") 90 | return 91 | } 92 | if len(_type) > 0 && len(id) > 0 { 93 | partialURL = fmt.Sprintf("/%s/%s/%s", index, _type, id) 94 | } else if len(_type) > 0 { 95 | partialURL = fmt.Sprintf("/%s/%s", index, _type) 96 | } else { 97 | partialURL = fmt.Sprintf("/%s", index) 98 | } 99 | // A child document can be indexed by specifying it’s parent when indexing. 100 | if len(parentId) > 0 { 101 | values.Add("parent", parentId) 102 | } 103 | // versions start at 1, so if greater than 0 104 | if version > 0 { 105 | values.Add("version", strconv.Itoa(version)) 106 | } 107 | if len(op_type) > 0 { 108 | if len(id) == 0 { 109 | //if id is omited, op_type defaults to 'create' 110 | values.Add("op_type", "create") 111 | } else { 112 | values.Add("op_type", op_type) 113 | } 114 | } 115 | if len(routing) > 0 { 116 | values.Add("routing", routing) 117 | } 118 | // A document can be indexed with a timestamp associated with it. 119 | // The timestamp value of a document can be set using the timestamp parameter. 120 | if len(timestamp) > 0 { 121 | values.Add("timestamp", timestamp) 122 | } 123 | // A document can be indexed with a ttl (time to live) associated with it. Expired documents 124 | // will be expunged automatically. 125 | if ttl > 0 { 126 | values.Add("ttl", strconv.Itoa(ttl)) 127 | } 128 | if len(percolate) > 0 { 129 | values.Add("percolate", percolate) 130 | } 131 | // example 5m 132 | if len(timeout) > 0 { 133 | values.Add("timeout", timeout) 134 | } 135 | 136 | if refresh { 137 | values.Add("refresh", "true") 138 | } 139 | 140 | partialURL += "?" + values.Encode() 141 | return partialURL, nil 142 | } 143 | -------------------------------------------------------------------------------- /indices/putMapping_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package indices 13 | 14 | import ( 15 | "encoding/json" 16 | api "github.com/buger/elastigo/api" 17 | "io/ioutil" 18 | "net/http" 19 | "net/http/httptest" 20 | "net/url" 21 | "strings" 22 | "testing" 23 | ) 24 | 25 | var ( 26 | mux *http.ServeMux 27 | server *httptest.Server 28 | ) 29 | 30 | func setup(t *testing.T) { 31 | mux = http.NewServeMux() 32 | server = httptest.NewServer(mux) 33 | 34 | serverURL, err := url.Parse(server.URL) 35 | if err != nil { 36 | t.Fatalf("Error: %v", err) 37 | } 38 | 39 | api.Domain = strings.Split(serverURL.Host, ":")[0] 40 | api.Port = strings.Split(serverURL.Host, ":")[1] 41 | } 42 | 43 | func teardown() { 44 | server.Close() 45 | } 46 | 47 | type TestStruct struct { 48 | Id string `json:"id" elastic:"index:not_analyzed"` 49 | DontIndex string `json:"dontIndex" elastic:"index:no"` 50 | Number int `json:"number" elastic:"type:integer,index:analyzed"` 51 | Omitted string `json:"-"` 52 | NoJson string `elastic:"type:string"` 53 | unexported string 54 | JsonOmitEmpty string `json:"jsonOmitEmpty,omitempty" elastic:"type:string"` 55 | Embedded 56 | Nested NestedStruct `json:"nested"` 57 | NestedP *NestedStruct `json:"pointer_to_nested"` 58 | NestedS []NestedStruct `json:"slice_of_nested"` 59 | MultiAnalyze string `json:"multi_analyze"` 60 | } 61 | 62 | type Embedded struct { 63 | EmbeddedField string `json:"embeddedField" elastic:"type:string"` 64 | } 65 | 66 | type NestedStruct struct { 67 | NestedField string `json:"nestedField" elastic:"type:date"` 68 | } 69 | 70 | func TestPutMapping(t *testing.T) { 71 | setup(t) 72 | defer teardown() 73 | 74 | options := MappingOptions{ 75 | Timestamp: TimestampOptions{Enabled: true}, 76 | Id: IdOptions{Index: "analyzed", Path: "id"}, 77 | Properties: map[string]interface{}{ 78 | // special properties that can't be expressed as tags 79 | "multi_analyze": map[string]interface{}{ 80 | "type": "multi_field", 81 | "fields": map[string]map[string]string{ 82 | "ma_analyzed": {"type": "string", "index": "analyzed"}, 83 | "ma_unanalyzed": {"type": "string", "index": "un_analyzed"}, 84 | }, 85 | }, 86 | }, 87 | } 88 | expValue := MappingForType("myType", MappingOptions{ 89 | Timestamp: TimestampOptions{Enabled: true}, 90 | Id: IdOptions{Index: "analyzed", Path: "id"}, 91 | Properties: map[string]interface{}{ 92 | "NoJson": map[string]string{"type": "string"}, 93 | "dontIndex": map[string]string{"index": "no"}, 94 | "embeddedField": map[string]string{"type": "string"}, 95 | "id": map[string]string{"index": "not_analyzed"}, 96 | "jsonOmitEmpty": map[string]string{"type": "string"}, 97 | "number": map[string]string{"index": "analyzed", "type": "integer"}, 98 | "multi_analyze": map[string]interface{}{ 99 | "type": "multi_field", 100 | "fields": map[string]map[string]string{ 101 | "ma_analyzed": {"type": "string", "index": "analyzed"}, 102 | "ma_unanalyzed": {"type": "string", "index": "un_analyzed"}, 103 | }, 104 | }, 105 | "nested": map[string]map[string]map[string]string{ 106 | "properties": { 107 | "nestedField": {"type": "date"}, 108 | }, 109 | }, 110 | "pointer_to_nested": map[string]map[string]map[string]string{ 111 | "properties": { 112 | "nestedField": {"type": "date"}, 113 | }, 114 | }, 115 | "slice_of_nested": map[string]map[string]map[string]string{ 116 | "properties": { 117 | "nestedField": {"type": "date"}, 118 | }, 119 | }, 120 | }, 121 | }) 122 | 123 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 124 | var value map[string]interface{} 125 | bd, err := ioutil.ReadAll(r.Body) 126 | json.NewDecoder(strings.NewReader(string(bd))).Decode(&value) 127 | expValJson, err := json.MarshalIndent(expValue, "", " ") 128 | if err != nil { 129 | t.Errorf("Got error: %v", err) 130 | } 131 | valJson, err := json.MarshalIndent(value, "", " ") 132 | if err != nil { 133 | t.Errorf("Got error: %v", err) 134 | } 135 | 136 | if string(expValJson) != string(valJson) { 137 | t.Errorf("Expected %s but got %s", string(expValJson), string(valJson)) 138 | } 139 | }) 140 | 141 | err := PutMapping("myIndex", "myType", TestStruct{}, options) 142 | if err != nil { 143 | t.Errorf("Error: %v", err) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /core/test_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "bufio" 16 | "bytes" 17 | "compress/gzip" 18 | "crypto/md5" 19 | "encoding/json" 20 | "flag" 21 | "fmt" 22 | "github.com/buger/elastigo/api" 23 | "io" 24 | "log" 25 | "net/http" 26 | "os" 27 | "testing" 28 | "time" 29 | ) 30 | 31 | /* 32 | 33 | usage: 34 | 35 | test -v -host eshost -loaddata 36 | 37 | */ 38 | 39 | var ( 40 | _ = os.ModeDir 41 | bulkStarted bool 42 | hasLoadedData bool 43 | hasStartedTesting bool 44 | eshost *string = flag.String("host", "localhost", "Elasticsearch Server Host Address") 45 | loadData *bool = flag.Bool("loaddata", false, "This loads a bunch of test data into elasticsearch for testing") 46 | ) 47 | 48 | func init() { 49 | 50 | } 51 | func InitTests(startIndexer bool) { 52 | if !hasStartedTesting { 53 | flag.Parse() 54 | hasStartedTesting = true 55 | log.SetFlags(log.Ltime | log.Lshortfile) 56 | api.Domain = *eshost 57 | } 58 | if startIndexer && !bulkStarted { 59 | BulkDelaySeconds = 1 60 | bulkStarted = true 61 | BulkIndexerGlobalRun(100, make(chan bool)) 62 | if *loadData && !hasLoadedData { 63 | log.Println("loading test data ") 64 | hasLoadedData = true 65 | LoadTestData() 66 | } 67 | } 68 | } 69 | 70 | // dumb simple assert for testing, printing 71 | // Assert(len(items) == 9, t, "Should be 9 but was %d", len(items)) 72 | func Assert(is bool, t *testing.T, format string, args ...interface{}) { 73 | if is == false { 74 | log.Printf(format, args...) 75 | t.Fail() 76 | } 77 | } 78 | 79 | // Wait for condition (defined by func) to be true, a utility to create a ticker 80 | // checking every 100 ms to see if something (the supplied check func) is done 81 | // 82 | // WaitFor(func() bool { 83 | // return ctr.Ct == 0 84 | // },10) 85 | // 86 | // @timeout (in seconds) is the last arg 87 | func WaitFor(check func() bool, timeoutSecs int) { 88 | timer := time.NewTicker(100 * time.Millisecond) 89 | tryct := 0 90 | for _ = range timer.C { 91 | if check() { 92 | timer.Stop() 93 | break 94 | } 95 | if tryct >= timeoutSecs*10 { 96 | timer.Stop() 97 | break 98 | } 99 | tryct++ 100 | } 101 | } 102 | 103 | func TestFake(t *testing.T) { 104 | 105 | } 106 | 107 | type GithubEvent struct { 108 | Url string 109 | Created time.Time `json:"created_at"` 110 | Type string 111 | } 112 | 113 | // This loads test data from github archives (~6700 docs) 114 | func LoadTestData() { 115 | docCt := 0 116 | errCt := 0 117 | indexer := NewBulkIndexer(1) 118 | indexer.BulkSender = func(buf *bytes.Buffer) error { 119 | // log.Printf("Sent %d bytes total %d docs sent", buf.Len(), docCt) 120 | req, err := api.ElasticSearchRequest("POST", "/_bulk", "") 121 | if err != nil { 122 | errCt += 1 123 | log.Fatalf("ERROR: ", err) 124 | return err 125 | } 126 | req.SetBody(buf) 127 | // res, err := http.DefaultClient.Do(*(api.Request(req))) 128 | var response map[string]interface{} 129 | httpStatusCode, _, err := req.Do(&response) 130 | if err != nil { 131 | errCt += 1 132 | log.Fatalf("ERROR: %v", err) 133 | return err 134 | } 135 | if httpStatusCode != 200 { 136 | log.Fatalf("Not 200! %d \n", httpStatusCode) 137 | } 138 | return err 139 | } 140 | done := make(chan bool) 141 | indexer.Run(done) 142 | resp, err := http.Get("http://data.githubarchive.org/2012-12-10-15.json.gz") 143 | if err != nil || resp == nil { 144 | panic("Could not download data") 145 | } 146 | defer resp.Body.Close() 147 | if err != nil { 148 | log.Println(err) 149 | return 150 | } 151 | gzReader, err := gzip.NewReader(resp.Body) 152 | defer gzReader.Close() 153 | if err != nil { 154 | panic(err) 155 | } 156 | r := bufio.NewReader(gzReader) 157 | var ge GithubEvent 158 | docsm := make(map[string]bool) 159 | h := md5.New() 160 | for { 161 | line, err := r.ReadBytes('\n') 162 | if err != nil && err != io.EOF { 163 | log.Println("FATAL: could not read line? ", err) 164 | } else if err != nil { 165 | indexer.Flush() 166 | break 167 | } 168 | if err := json.Unmarshal(line, &ge); err == nil { 169 | // create an "ID" 170 | h.Write(line) 171 | id := fmt.Sprintf("%x", h.Sum(nil)) 172 | if _, ok := docsm[id]; ok { 173 | log.Println("HM, already exists? ", ge.Url) 174 | } 175 | docsm[id] = true 176 | indexer.Index("github", ge.Type, id, "", &ge.Created, line, true) 177 | docCt++ 178 | } else { 179 | log.Println("ERROR? ", string(line)) 180 | } 181 | } 182 | if errCt != 0 { 183 | log.Println("FATAL, could not load ", errCt) 184 | } 185 | // lets wait a bit to ensure that elasticsearch finishes? 186 | time.Sleep(time.Second * 5) 187 | if len(docsm) != docCt { 188 | panic(fmt.Sprintf("Docs didn't match? %d:%d", len(docsm), docCt)) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /search/aggregate.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import "encoding/json" 4 | 5 | func Aggregate(name string) *AggregateDsl { 6 | return &AggregateDsl{Name: name} 7 | } 8 | 9 | type AggregateDsl struct { 10 | Name string 11 | TypeName string 12 | Type interface{} 13 | Filters *FilterWrap `json:"filters,omitempty"` 14 | AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"` 15 | } 16 | 17 | type FieldAggregate struct { 18 | Field string `json:"field"` 19 | } 20 | 21 | /** 22 | * Aggregates accepts n "sub-aggregates" to be applied to this aggregate 23 | * 24 | * agg := Aggregate("user").Term("user_id") 25 | * agg.Aggregates( 26 | * Aggregate("total_spent").Sum("price"), 27 | * Aggregate("total_saved").Sum("discount"), 28 | * ) 29 | */ 30 | func (d *AggregateDsl) Aggregates(aggs ...*AggregateDsl) *AggregateDsl { 31 | if len(aggs) < 1 { 32 | return d 33 | } 34 | if len(d.AggregatesVal) == 0 { 35 | d.AggregatesVal = make(map[string]*AggregateDsl) 36 | } 37 | 38 | for _, agg := range aggs { 39 | d.AggregatesVal[agg.Name] = agg 40 | } 41 | return d 42 | } 43 | 44 | func (d *AggregateDsl) Min(field string) *AggregateDsl { 45 | d.Type = FieldAggregate{Field: field} 46 | d.TypeName = "min" 47 | return d 48 | } 49 | 50 | func (d *AggregateDsl) Max(field string) *AggregateDsl { 51 | d.Type = FieldAggregate{Field: field} 52 | d.TypeName = "max" 53 | return d 54 | } 55 | 56 | func (d *AggregateDsl) Sum(field string) *AggregateDsl { 57 | d.Type = FieldAggregate{Field: field} 58 | d.TypeName = "sum" 59 | return d 60 | } 61 | 62 | func (d *AggregateDsl) Avg(field string) *AggregateDsl { 63 | d.Type = FieldAggregate{Field: field} 64 | d.TypeName = "avg" 65 | return d 66 | } 67 | 68 | func (d *AggregateDsl) Stats(field string) *AggregateDsl { 69 | d.Type = FieldAggregate{Field: field} 70 | d.TypeName = "stats" 71 | return d 72 | } 73 | 74 | func (d *AggregateDsl) ExtendedStats(field string) *AggregateDsl { 75 | d.Type = FieldAggregate{Field: field} 76 | d.TypeName = "extended_stats" 77 | return d 78 | } 79 | 80 | func (d *AggregateDsl) ValueCount(field string) *AggregateDsl { 81 | d.Type = FieldAggregate{Field: field} 82 | d.TypeName = "value_count" 83 | return d 84 | } 85 | 86 | func (d *AggregateDsl) Percentiles(field string) *AggregateDsl { 87 | d.Type = FieldAggregate{Field: field} 88 | d.TypeName = "percentiles" 89 | return d 90 | } 91 | 92 | type Cardinality struct { 93 | Field string `json:"field"` 94 | PrecisionThreshold float64 `json:"precision_threshold,omitempty"` 95 | Rehash bool `json:"rehash,omitempty"` 96 | } 97 | 98 | /** 99 | * Cardinality( 100 | * "field_name", 101 | * true, 102 | * 0, 103 | * ) 104 | */ 105 | func (d *AggregateDsl) Cardinality(field string, rehash bool, threshold int) *AggregateDsl { 106 | c := Cardinality{Field: field} 107 | 108 | // Only set if it's false, since the default is true 109 | if !rehash { 110 | c.Rehash = false 111 | } 112 | 113 | if threshold > 0 { 114 | c.PrecisionThreshold = float64(threshold) 115 | } 116 | d.Type = c 117 | d.TypeName = "cardinality" 118 | return d 119 | } 120 | 121 | func (d *AggregateDsl) Global() *AggregateDsl { 122 | d.Type = struct{}{} 123 | d.TypeName = "global" 124 | return d 125 | } 126 | 127 | func (d *AggregateDsl) Filter(filters ...interface{}) *AggregateDsl { 128 | 129 | if len(filters) == 0 { 130 | return d 131 | } 132 | 133 | if d.Filters == nil { 134 | d.Filters = NewFilterWrap() 135 | } 136 | 137 | d.Filters.addFilters(filters) 138 | return d 139 | } 140 | 141 | func (d *AggregateDsl) Missing(field string) *AggregateDsl { 142 | d.Type = FieldAggregate{Field: field} 143 | d.TypeName = "missing" 144 | return d 145 | } 146 | 147 | func (d *AggregateDsl) Terms(field string) *AggregateDsl { 148 | d.Type = FieldAggregate{Field: field} 149 | d.TypeName = "terms" 150 | return d 151 | } 152 | 153 | func (d *AggregateDsl) SignificantTerms(field string) *AggregateDsl { 154 | d.Type = FieldAggregate{Field: field} 155 | d.TypeName = "significant_terms" 156 | return d 157 | } 158 | 159 | type Histogram struct { 160 | Field string `json:"field"` 161 | Interval float64 `json:"interval"` 162 | } 163 | 164 | func (d *AggregateDsl) Histogram(field string, interval int) *AggregateDsl { 165 | d.Type = Histogram{ 166 | Field: field, 167 | Interval: float64(interval), 168 | } 169 | d.TypeName = "histogram" 170 | return d 171 | } 172 | 173 | type DateHistogram struct { 174 | Field string `json:"field"` 175 | Interval string `json:"interval"` 176 | } 177 | 178 | func (d *AggregateDsl) DateHistogram(field, interval string) *AggregateDsl { 179 | d.Type = DateHistogram{ 180 | Field: field, 181 | Interval: interval, 182 | } 183 | d.TypeName = "date_histogram" 184 | return d 185 | } 186 | 187 | func (d *AggregateDsl) MarshalJSON() ([]byte, error) { 188 | return json.Marshal(d.toMap()) 189 | } 190 | 191 | func (d *AggregateDsl) toMap() map[string]interface{} { 192 | root := map[string]interface{}{} 193 | 194 | if d.Type != nil { 195 | root[d.TypeName] = d.Type 196 | } 197 | aggregates := d.aggregatesMap() 198 | 199 | if d.Filters != nil { 200 | root["filter"] = d.Filters 201 | } 202 | 203 | if len(aggregates) > 0 { 204 | root["aggregations"] = aggregates 205 | } 206 | return root 207 | 208 | } 209 | func (d *AggregateDsl) aggregatesMap() map[string]interface{} { 210 | root := map[string]interface{}{} 211 | 212 | if len(d.AggregatesVal) > 0 { 213 | for _, agg := range d.AggregatesVal { 214 | root[agg.Name] = agg.toMap() 215 | } 216 | } 217 | return root 218 | } 219 | -------------------------------------------------------------------------------- /search/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | 18 | . "github.com/araddon/gou" 19 | ) 20 | 21 | var ( 22 | _ = DEBUG 23 | ) 24 | 25 | // A bool (and/or) clause 26 | type BoolClause string 27 | 28 | // Filter clause is either a boolClause or FilterOp 29 | type FilterClause interface { 30 | String() string 31 | } 32 | 33 | // A wrapper to allow for custom serialization 34 | type FilterWrap struct { 35 | boolClause string 36 | filters []interface{} 37 | } 38 | 39 | func NewFilterWrap() *FilterWrap { 40 | return &FilterWrap{filters: make([]interface{}, 0), boolClause: "and"} 41 | } 42 | 43 | func (f *FilterWrap) String() string { 44 | return fmt.Sprintf(`fopv: %d:%v`, len(f.filters), f.filters) 45 | } 46 | 47 | // Custom marshalling to support the query dsl 48 | func (f *FilterWrap) addFilters(fl []interface{}) { 49 | if len(fl) > 1 { 50 | fc := fl[0] 51 | switch fc.(type) { 52 | case BoolClause, string: 53 | f.boolClause = fc.(string) 54 | fl = fl[1:] 55 | } 56 | } 57 | f.filters = append(f.filters, fl...) 58 | } 59 | 60 | // Custom marshalling to support the query dsl 61 | func (f *FilterWrap) MarshalJSON() ([]byte, error) { 62 | var root interface{} 63 | if len(f.filters) > 1 { 64 | root = map[string]interface{}{f.boolClause: f.filters} 65 | } else if len(f.filters) == 1 { 66 | root = f.filters[0] 67 | } 68 | return json.Marshal(root) 69 | } 70 | 71 | /* 72 | "filter": { 73 | "range": { 74 | "@timestamp": { 75 | "from": "2012-12-29T16:52:48+00:00", 76 | "to": "2012-12-29T17:52:48+00:00" 77 | } 78 | } 79 | } 80 | "filter": { 81 | "missing": { 82 | "field": "repository.name" 83 | } 84 | } 85 | 86 | "filter" : { 87 | "terms" : { 88 | "user" : ["kimchy", "elasticsearch"], 89 | "execution" : "bool", 90 | "_cache": true 91 | } 92 | } 93 | 94 | "filter" : { 95 | "term" : { "user" : "kimchy"} 96 | } 97 | 98 | "filter" : { 99 | "and" : [ 100 | { 101 | "range" : { 102 | "postDate" : { 103 | "from" : "2010-03-01", 104 | "to" : "2010-04-01" 105 | } 106 | } 107 | }, 108 | { 109 | "prefix" : { "name.second" : "ba" } 110 | } 111 | ] 112 | } 113 | 114 | */ 115 | 116 | // Filter Operation 117 | // 118 | // Filter().Term("user","kimchy") 119 | // 120 | // // we use variadics to allow n arguments, first is the "field" rest are values 121 | // Filter().Terms("user", "kimchy", "elasticsearch") 122 | // 123 | // Filter().Exists("repository.name") 124 | // 125 | func Filter() *FilterOp { 126 | return &FilterOp{} 127 | } 128 | 129 | func CompoundFilter(fl ...interface{}) *FilterWrap { 130 | FilterVal := NewFilterWrap() 131 | FilterVal.addFilters(fl) 132 | return FilterVal 133 | } 134 | 135 | type FilterOp struct { 136 | curField string 137 | TermsMap map[string][]interface{} `json:"terms,omitempty"` 138 | Range map[string]map[string]interface{} `json:"range,omitempty"` 139 | Exist map[string]string `json:"exists,omitempty"` 140 | MisssingVal map[string]string `json:"missing,omitempty"` 141 | } 142 | 143 | // A range is a special type of Filter operation 144 | // 145 | // Range().Exists("repository.name") 146 | func Range() *FilterOp { 147 | return &FilterOp{Range: make(map[string]map[string]interface{})} 148 | } 149 | 150 | func (f *FilterOp) Field(fld string) *FilterOp { 151 | f.curField = fld 152 | if _, ok := f.Range[fld]; !ok { 153 | m := make(map[string]interface{}) 154 | f.Range[fld] = m 155 | } 156 | return f 157 | } 158 | 159 | // Filter Terms 160 | // 161 | // Filter().Terms("user","kimchy") 162 | // 163 | // // we use variadics to allow n arguments, first is the "field" rest are values 164 | // Filter().Terms("user", "kimchy", "elasticsearch") 165 | // 166 | func (f *FilterOp) Terms(field string, values ...interface{}) *FilterOp { 167 | if len(f.TermsMap) == 0 { 168 | f.TermsMap = make(map[string][]interface{}) 169 | } 170 | for _, val := range values { 171 | f.TermsMap[field] = append(f.TermsMap[field], val) 172 | } 173 | 174 | return f 175 | } 176 | func (f *FilterOp) From(from string) *FilterOp { 177 | f.Range[f.curField]["from"] = from 178 | return f 179 | } 180 | func (f *FilterOp) To(to string) *FilterOp { 181 | f.Range[f.curField]["to"] = to 182 | return f 183 | } 184 | func (f *FilterOp) Gt(gt int) *FilterOp { 185 | f.Range[f.curField]["gt"] = float64(gt) 186 | return f 187 | } 188 | func (f *FilterOp) Exists(name string) *FilterOp { 189 | f.Exist = map[string]string{"field": name} 190 | return f 191 | } 192 | func (f *FilterOp) Missing(name string) *FilterOp { 193 | f.MisssingVal = map[string]string{"field": name} 194 | return f 195 | } 196 | 197 | // Add another Filterop, "combines" two filter ops into one 198 | func (f *FilterOp) Add(fop *FilterOp) *FilterOp { 199 | // TODO, this is invalid, refactor 200 | if len(fop.Exist) > 0 { 201 | f.Exist = fop.Exist 202 | } 203 | if len(fop.MisssingVal) > 0 { 204 | f.MisssingVal = fop.MisssingVal 205 | } 206 | if len(fop.Range) > 0 { 207 | f.Range = fop.Range 208 | } 209 | return f 210 | } 211 | -------------------------------------------------------------------------------- /search/search.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | u "github.com/araddon/gou" 18 | "github.com/buger/elastigo/api" 19 | "github.com/buger/elastigo/core" 20 | "log" 21 | "strconv" 22 | "strings" 23 | ) 24 | 25 | var ( 26 | _ = u.DEBUG 27 | ) 28 | 29 | // Search is the entry point to the SearchDsl, it is a chainable set of utilities 30 | // to create searches. 31 | // 32 | // params 33 | // @index = elasticsearch index to search 34 | // 35 | // out, err := Search("github").Type("Issues").Pretty().Query( 36 | // Query().Range( 37 | // Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), 38 | // ).Search("add"), 39 | // ).Result() 40 | func Search(index string) *SearchDsl { 41 | return &SearchDsl{Index: index, args: map[string]interface{}{}} 42 | } 43 | 44 | type SearchDsl struct { 45 | args map[string]interface{} 46 | types []string 47 | FromVal int `json:"from,omitempty"` 48 | SizeVal int `json:"size,omitempty"` 49 | Index string `json:"-"` 50 | FacetVal *FacetDsl `json:"facets,omitempty"` 51 | QueryVal *QueryDsl `json:"query,omitempty"` 52 | SortBody []*SortDsl `json:"sort,omitempty"` 53 | FilterVal *FilterWrap `json:"filter,omitempty"` 54 | AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"` 55 | } 56 | 57 | func (s *SearchDsl) Bytes() ([]byte, error) { 58 | return api.DoCommand("POST", s.url(), s.args, s) 59 | } 60 | 61 | func (s *SearchDsl) Result() (*core.SearchResult, error) { 62 | var retval core.SearchResult 63 | if core.DebugRequests { 64 | sb, _ := json.MarshalIndent(s, " ", " ") 65 | log.Println(s.url()) 66 | log.Println(string(sb)) 67 | } 68 | body, err := s.Bytes() 69 | if err != nil { 70 | u.Errorf("%v", err) 71 | return nil, err 72 | } 73 | jsonErr := json.Unmarshal(body, &retval) 74 | if jsonErr != nil { 75 | u.Errorf("%v \n\t%s", jsonErr, string(body)) 76 | } 77 | //Debug(string(body)) 78 | return &retval, jsonErr 79 | } 80 | 81 | func (s *SearchDsl) url() string { 82 | url := fmt.Sprintf("/%s%s/_search", s.Index, s.getType()) 83 | return url 84 | } 85 | 86 | func (s *SearchDsl) Pretty() *SearchDsl { 87 | s.args["pretty"] = "1" 88 | return s 89 | } 90 | 91 | // Type is the elasticsearch *Type* within a specific index 92 | func (s *SearchDsl) Type(indexType string) *SearchDsl { 93 | if len(s.types) == 0 { 94 | s.types = make([]string, 0) 95 | } 96 | s.types = append(s.types, indexType) 97 | return s 98 | } 99 | 100 | func (s *SearchDsl) getType() string { 101 | if len(s.types) > 0 { 102 | return "/" + strings.Join(s.types, ",") 103 | } 104 | return "" 105 | } 106 | 107 | func (s *SearchDsl) From(from string) *SearchDsl { 108 | s.args["from"] = from 109 | return s 110 | } 111 | 112 | // Search is a simple interface to search, doesn't have the power of query 113 | // but uses a simple query_string search 114 | func (s *SearchDsl) Search(srch string) *SearchDsl { 115 | s.QueryVal = Query().Search(srch) 116 | return s 117 | } 118 | 119 | func (s *SearchDsl) Size(size string) *SearchDsl { 120 | s.args["size"] = size 121 | return s 122 | } 123 | 124 | func (s *SearchDsl) Fields(fields ...string) *SearchDsl { 125 | s.args["fields"] = strings.Join(fields, ",") 126 | return s 127 | } 128 | 129 | func (s *SearchDsl) Source(returnSource bool) *SearchDsl { 130 | s.args["_source"] = strconv.FormatBool(returnSource) 131 | return s 132 | } 133 | 134 | // Facet passes a Query expression to this search 135 | // 136 | // qry := Search("github").Size("0").Facet( 137 | // Facet().Regex("repository.name", "no.*").Size("8"), 138 | // ) 139 | // 140 | // qry := Search("github").Pretty().Facet( 141 | // Facet().Fields("type").Size("25"), 142 | // ) 143 | func (s *SearchDsl) Facet(f *FacetDsl) *SearchDsl { 144 | s.FacetVal = f 145 | return s 146 | } 147 | 148 | func (s *SearchDsl) Aggregates(aggs ...*AggregateDsl) *SearchDsl { 149 | if len(aggs) < 1 { 150 | return s 151 | } 152 | if len(s.AggregatesVal) == 0 { 153 | s.AggregatesVal = make(map[string]*AggregateDsl) 154 | } 155 | 156 | for _, agg := range aggs { 157 | s.AggregatesVal[agg.Name] = agg 158 | } 159 | return s 160 | } 161 | 162 | func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl { 163 | s.QueryVal = q 164 | return s 165 | } 166 | 167 | // Filter adds a Filter Clause with optional Boolean Clause. This accepts n number of 168 | // filter clauses. If more than one, and missing Boolean Clause it assumes "and" 169 | // 170 | // qry := Search("github").Filter( 171 | // Filter().Exists("repository.name"), 172 | // ) 173 | // 174 | // qry := Search("github").Filter( 175 | // "or", 176 | // Filter().Exists("repository.name"), 177 | // Filter().Terms("actor_attributes.location", "portland"), 178 | // ) 179 | // 180 | // qry := Search("github").Filter( 181 | // Filter().Exists("repository.name"), 182 | // Filter().Terms("repository.has_wiki", true) 183 | // ) 184 | func (s *SearchDsl) Filter(fl ...interface{}) *SearchDsl { 185 | if s.FilterVal == nil { 186 | s.FilterVal = NewFilterWrap() 187 | } 188 | 189 | s.FilterVal.addFilters(fl) 190 | return s 191 | } 192 | 193 | func (s *SearchDsl) Sort(sort ...*SortDsl) *SearchDsl { 194 | if s.SortBody == nil { 195 | s.SortBody = make([]*SortDsl, 0) 196 | } 197 | s.SortBody = append(s.SortBody, sort...) 198 | return s 199 | } 200 | 201 | func (s *SearchDsl) Scroll(duration string) *SearchDsl { 202 | s.args["scroll"] = duration 203 | return s 204 | } 205 | -------------------------------------------------------------------------------- /cluster/stats.go: -------------------------------------------------------------------------------- 1 | package cluster 2 | 3 | type NodeStatsReponse struct { 4 | ClusterName string `json:"cluster_name"` 5 | Nodes map[string]NodeStatsNodeResponse 6 | } 7 | type NodeStatsNodeResponse struct { 8 | Name string `json:"name"` 9 | Timestamp int64 `json:"timestamp"` 10 | TransportAddress string `json:"transport_address"` 11 | Hostname string `json:"hostname"` 12 | Indices NodeStatsIndicesResponse `json:"indices"` 13 | OS NodeStatsOSResponse `json:"os"` 14 | Network NodeStatsNetworkResponse `json:"network"` 15 | ThreadPool map[string]NodeStatsThreadPoolPoolResponse `json:"thread_pool"` 16 | } 17 | 18 | type NodeStatsNetworkResponse struct { 19 | TCP NodeStatsTCPResponse `json:"tcp"` 20 | } 21 | 22 | type NodeStatsTransportResponse struct { 23 | ServerOpen int64 `json:"server_open"` 24 | RxCount int64 `json:"rx_count"` 25 | RxSize int64 `json:"rx_size_in_bytes"` 26 | TxCount int64 `json:"tx_count"` 27 | TxSize int64 `json:"tx_size_in_bytes"` 28 | } 29 | 30 | type NodeStatsThreadPoolPoolResponse struct { 31 | Threads int64 `json:"threads"` 32 | Queue int64 `json:"queue"` 33 | Active int64 `json:"active"` 34 | Rejected int64 `json:"rejected"` 35 | Largest int64 `json:"largest"` 36 | Completed int64 `json:"completed"` 37 | } 38 | 39 | type NodeStatsTCPResponse struct { 40 | ActiveOpens int64 `json:"active_opens"` 41 | PassiveOpens int64 `json:"passive_opens"` 42 | CurrEstab int64 `json:"curr_estab"` 43 | InSegs int64 `json:"in_segs"` 44 | OutSegs int64 `json:"out_segs"` 45 | RetransSegs int64 `json:"retrans_segs"` 46 | EstabResets int64 `json:"estab_resets"` 47 | AttemptFails int64 `json:"attempt_fails"` 48 | InErrs int64 `json:"in_errs"` 49 | OutRsts int64 `json:"out_rsts"` 50 | } 51 | 52 | type NodeStatsIndicesResponse struct { 53 | Docs NodeStatsIndicesDocsResponse 54 | Store NodeStatsIndicesStoreResponse 55 | Indexing NodeStatsIndicesIndexingResponse 56 | Get NodeStatsIndicesGetResponse 57 | Search NodeStatsIndicesStoreResponse 58 | } 59 | 60 | type NodeStatsIndicesDocsResponse struct { 61 | Count int64 `json:"count"` 62 | Deleted int64 `json:"deleted"` 63 | } 64 | 65 | type NodeStatsIndicesStoreResponse struct { 66 | Size int64 `json:"size_in_bytes"` 67 | ThrottleTime int64 `json:"throttle_time_in_millis"` 68 | } 69 | 70 | type NodeStatsIndicesIndexingResponse struct { 71 | IndexTotal int64 `json:"index_total"` 72 | IndexTime int64 `json:"index_time_in_millis"` 73 | IndexCurrent int64 `json:"index_current"` 74 | DeleteTotal int64 `json:"delete_total"` 75 | DeleteTime int64 `json:"delete_time_in_millis"` 76 | DeleteCurrent int64 `json:"delete_current"` 77 | } 78 | 79 | type NodeStatsIndicesGetResponse struct { 80 | Total int64 `json:"total"` 81 | Time int64 `json:"time_in_millis"` 82 | ExistsTotal int64 `json:"exists_total"` 83 | ExistsTime int64 `json:"exists_time_in_millis"` 84 | MissingTotal int64 `json:"missing_total"` 85 | MissingTime int64 `json:"missing_time_in_millis"` 86 | Current int64 `json:"current"` 87 | } 88 | 89 | type NodeStatsIndicesSearchResponse struct { 90 | OpenContext int64 `json:"open_contexts"` 91 | QueryTotal int64 `json:"query_total"` 92 | QueryTime int64 `json:"query_time_in_millis"` 93 | QueryCurrent int64 `json:"query_current"` 94 | FetchTotal int64 `json:"fetch_total"` 95 | FetchTime int64 `json:"fetch_time_in_millis"` 96 | FetchCurrent int64 `json:"fetch_current"` 97 | } 98 | 99 | type NodeStatsOSResponse struct { 100 | Timestamp int64 `json:"timestamp"` 101 | Uptime int64 `json:"uptime_in_millis"` 102 | LoadAvg []float64 `json:"load_average"` 103 | CPU NodeStatsOSCPUResponse `json:"cpu"` 104 | Mem NodeStatsOSMemResponse `json:"mem"` 105 | Swap NodeStatsOSSwapResponse `json:"swap"` 106 | } 107 | 108 | type NodeStatsOSMemResponse struct { 109 | Free int64 `json:"free_in_bytes"` 110 | Used int64 `json:"used_in_bytes"` 111 | ActualFree int64 `json:"actual_free_in_bytes"` 112 | ActualUsed int64 `json:"actual_used_in_bytes"` 113 | } 114 | 115 | type NodeStatsOSSwapResponse struct { 116 | Used int64 `json:"used_in_bytes"` 117 | Free int64 `json:"free_in_bytes"` 118 | } 119 | 120 | type NodeStatsOSCPUResponse struct { 121 | Sys int64 `json:"sys"` 122 | User int64 `json:"user"` 123 | Idle int64 `json:"idle"` 124 | Steal int64 `json:"stolen"` 125 | } 126 | 127 | type NodeStatsProcessResponse struct { 128 | Timestamp int64 `json:"timestamp"` 129 | OpenFD int64 `json:"open_file_descriptors"` 130 | CPU NodeStatsProcessCPUResponse `json:"cpu"` 131 | Memory NodeStatsProcessMemResponse `json:"mem"` 132 | } 133 | 134 | type NodeStatsProcessMemResponse struct { 135 | Resident int64 `json:"resident_in_bytes"` 136 | Share int64 `json:"share_in_bytes"` 137 | TotalVirtual int64 `json:"total_virtual_in_bytes"` 138 | } 139 | 140 | type NodeStatsProcessCPUResponse struct { 141 | Percent int64 `json:"percent"` 142 | Sys int64 `json:"sys_in_millis"` 143 | User int64 `json:"user_in_millis"` 144 | Total int64 `json:"total_in_millis"` 145 | } 146 | 147 | type NodeStatsHTTPResponse struct { 148 | CurrentOpen int64 `json:"current_open"` 149 | TotalOpen int64 `json:"total_open"` 150 | } 151 | 152 | type NodeStatsFSResponse struct { 153 | Timestamp int64 `json:"timestamp"` 154 | Data map[string]NodeStatsFSDataResponse `json:"data"` 155 | } 156 | 157 | type NodeStatsFSDataResponse struct { 158 | Path string `json:"path"` 159 | Mount string `json:"mount"` 160 | Device string `json:"dev"` 161 | Total int64 `json:"total_in_bytes"` 162 | Free int64 `json:"free_in_bytes"` 163 | Available int64 `json:"available_in_bytes"` 164 | DiskReads int64 `json:"disk_reads"` 165 | DiskWrites int64 `json:"disk_writes"` 166 | DiskReadSize int64 `json:"disk_read_size_in_bytes"` 167 | DiskWriteSize int64 `json:"disk_write_size_in_bytes"` 168 | } 169 | -------------------------------------------------------------------------------- /api/clusterstatresponses.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type NodeStatsReponse struct { 4 | ClusterName string `json:"cluster_name"` 5 | Nodes map[string]NodeStatsNodeResponse 6 | } 7 | type NodeStatsNodeResponse struct { 8 | Name string `json:"name"` 9 | Timestamp int64 `json:"timestamp"` 10 | TransportAddress string `json:"transport_address"` 11 | Hostname string `json:"hostname"` 12 | Indices NodeStatsIndicesResponse `json:"indices"` 13 | OS NodeStatsOSResponse `json:"os"` 14 | Network NodeStatsNetworkResponse `json:"network"` 15 | ThreadPool map[string]NodeStatsThreadPoolPoolResponse `json:"thread_pool"` 16 | } 17 | 18 | type NodeStatsNetworkResponse struct { 19 | TCP NodeStatsTCPResponse `json:"tcp"` 20 | } 21 | 22 | type NodeStatsTransportResponse struct { 23 | ServerOpen int64 `json:"server_open"` 24 | RxCount int64 `json:"rx_count"` 25 | RxSize int64 `json:"rx_size_in_bytes"` 26 | TxCount int64 `json:"tx_count"` 27 | TxSize int64 `json:"tx_size_in_bytes"` 28 | } 29 | 30 | type NodeStatsThreadPoolPoolResponse struct { 31 | Threads int64 `json:"threads"` 32 | Queue int64 `json:"queue"` 33 | Active int64 `json:"active"` 34 | Rejected int64 `json:"rejected"` 35 | Largest int64 `json:"largest"` 36 | Completed int64 `json:"completed"` 37 | } 38 | 39 | type NodeStatsTCPResponse struct { 40 | ActiveOpens int64 `json:"active_opens"` 41 | PassiveOpens int64 `json:"passive_opens"` 42 | CurrEstab int64 `json:"curr_estab"` 43 | InSegs int64 `json:"in_segs"` 44 | OutSegs int64 `json:"out_segs"` 45 | RetransSegs int64 `json:"retrans_segs"` 46 | EstabResets int64 `json:"estab_resets"` 47 | AttemptFails int64 `json:"attempt_fails"` 48 | InErrs int64 `json:"in_errs"` 49 | OutRsts int64 `json:"out_rsts"` 50 | } 51 | 52 | type NodeStatsIndicesResponse struct { 53 | Docs NodeStatsIndicesDocsResponse 54 | Store NodeStatsIndicesStoreResponse 55 | Indexing NodeStatsIndicesIndexingResponse 56 | Get NodeStatsIndicesGetResponse 57 | Search NodeStatsIndicesSearchResponse 58 | } 59 | 60 | type NodeStatsIndicesDocsResponse struct { 61 | Count int64 `json:"count"` 62 | Deleted int64 `json:"deleted"` 63 | } 64 | 65 | type NodeStatsIndicesStoreResponse struct { 66 | Size int64 `json:"size_in_bytes"` 67 | ThrottleTime int64 `json:"throttle_time_in_millis"` 68 | } 69 | 70 | type NodeStatsIndicesIndexingResponse struct { 71 | IndexTotal int64 `json:"index_total"` 72 | IndexTime int64 `json:"index_time_in_millis"` 73 | IndexCurrent int64 `json:"index_current"` 74 | DeleteTotal int64 `json:"delete_total"` 75 | DeleteTime int64 `json:"delete_time_in_millis"` 76 | DeleteCurrent int64 `json:"delete_current"` 77 | } 78 | 79 | type NodeStatsIndicesGetResponse struct { 80 | Total int64 `json:"total"` 81 | Time int64 `json:"time_in_millis"` 82 | ExistsTotal int64 `json:"exists_total"` 83 | ExistsTime int64 `json:"exists_time_in_millis"` 84 | MissingTotal int64 `json:"missing_total"` 85 | MissingTime int64 `json:"missing_time_in_millis"` 86 | Current int64 `json:"current"` 87 | } 88 | 89 | type NodeStatsIndicesSearchResponse struct { 90 | OpenContext int64 `json:"open_contexts"` 91 | QueryTotal int64 `json:"query_total"` 92 | QueryTime int64 `json:"query_time_in_millis"` 93 | QueryCurrent int64 `json:"query_current"` 94 | FetchTotal int64 `json:"fetch_total"` 95 | FetchTime int64 `json:"fetch_time_in_millis"` 96 | FetchCurrent int64 `json:"fetch_current"` 97 | } 98 | 99 | type NodeStatsOSResponse struct { 100 | Timestamp int64 `json:"timestamp"` 101 | Uptime int64 `json:"uptime_in_millis"` 102 | LoadAvg []float64 `json:"load_average"` 103 | CPU NodeStatsOSCPUResponse `json:"cpu"` 104 | Mem NodeStatsOSMemResponse `json:"mem"` 105 | Swap NodeStatsOSSwapResponse `json:"swap"` 106 | } 107 | 108 | type NodeStatsOSMemResponse struct { 109 | Free int64 `json:"free_in_bytes"` 110 | Used int64 `json:"used_in_bytes"` 111 | ActualFree int64 `json:"actual_free_in_bytes"` 112 | ActualUsed int64 `json:"actual_used_in_bytes"` 113 | } 114 | 115 | type NodeStatsOSSwapResponse struct { 116 | Used int64 `json:"used_in_bytes"` 117 | Free int64 `json:"free_in_bytes"` 118 | } 119 | 120 | type NodeStatsOSCPUResponse struct { 121 | Sys int64 `json:"sys"` 122 | User int64 `json:"user"` 123 | Idle int64 `json:"idle"` 124 | Steal int64 `json:"stolen"` 125 | } 126 | 127 | type NodeStatsProcessResponse struct { 128 | Timestamp int64 `json:"timestamp"` 129 | OpenFD int64 `json:"open_file_descriptors"` 130 | CPU NodeStatsProcessCPUResponse `json:"cpu"` 131 | Memory NodeStatsProcessMemResponse `json:"mem"` 132 | } 133 | 134 | type NodeStatsProcessMemResponse struct { 135 | Resident int64 `json:"resident_in_bytes"` 136 | Share int64 `json:"share_in_bytes"` 137 | TotalVirtual int64 `json:"total_virtual_in_bytes"` 138 | } 139 | 140 | type NodeStatsProcessCPUResponse struct { 141 | Percent int64 `json:"percent"` 142 | Sys int64 `json:"sys_in_millis"` 143 | User int64 `json:"user_in_millis"` 144 | Total int64 `json:"total_in_millis"` 145 | } 146 | 147 | type NodeStatsHTTPResponse struct { 148 | CurrentOpen int64 `json:"current_open"` 149 | TotalOpen int64 `json:"total_open"` 150 | } 151 | 152 | type NodeStatsFSResponse struct { 153 | Timestamp int64 `json:"timestamp"` 154 | Data map[string]NodeStatsFSDataResponse `json:"data"` 155 | } 156 | 157 | type NodeStatsFSDataResponse struct { 158 | Path string `json:"path"` 159 | Mount string `json:"mount"` 160 | Device string `json:"dev"` 161 | Total int64 `json:"total_in_bytes"` 162 | Free int64 `json:"free_in_bytes"` 163 | Available int64 `json:"available_in_bytes"` 164 | DiskReads int64 `json:"disk_reads"` 165 | DiskWrites int64 `json:"disk_writes"` 166 | DiskReadSize int64 `json:"disk_read_size_in_bytes"` 167 | DiskWriteSize int64 `json:"disk_write_size_in_bytes"` 168 | } 169 | -------------------------------------------------------------------------------- /search/query.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package search 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | //"log" 18 | "strings" 19 | ) 20 | 21 | // QueryDsl creates a new Query Dsl 22 | func Query() *QueryDsl { 23 | return &QueryDsl{} 24 | } 25 | 26 | /* 27 | 28 | some ways to serialize 29 | "query": { 30 | "filtered": { 31 | "query": { 32 | "query_string": { 33 | "default_operator": "OR", 34 | "default_field": "_all", 35 | "query": " actor:\"bob\" AND type:\"EventType\"" 36 | } 37 | }, 38 | "filter": { 39 | "range": { 40 | "@timestamp": { 41 | "from": "2012-12-29T16:52:48+00:00", 42 | "to": "2012-12-29T17:52:48+00:00" 43 | } 44 | } 45 | } 46 | } 47 | }, 48 | 49 | "query" : { 50 | "term" : { "user" : "kimchy" } 51 | } 52 | 53 | "query" : { 54 | "match_all" : {} 55 | }, 56 | */ 57 | type QueryDsl struct { 58 | QueryEmbed 59 | FilterVal *FilterOp `json:"filter,omitempty"` 60 | } 61 | 62 | // The core Query Syntax can be embedded as a child of a variety of different parents 63 | type QueryEmbed struct { 64 | MatchAll *MatchAll `json:"match_all,omitempty"` 65 | Terms map[string]string `json:"term,omitempty"` 66 | Qs *QueryString `json:"query_string,omitempty"` 67 | //Exist string `json:"_exists_,omitempty"` 68 | } 69 | 70 | // MarshalJSON provides custom marshalling to support the query dsl which is a conditional 71 | // json format, not always the same parent/children 72 | func (qd *QueryDsl) MarshalJSON() ([]byte, error) { 73 | q := qd.QueryEmbed 74 | hasQuery := false 75 | if q.Qs != nil || len(q.Terms) > 0 || q.MatchAll != nil { 76 | hasQuery = true 77 | } 78 | // If a query has a 79 | if qd.FilterVal != nil && hasQuery { 80 | queryB, err := json.Marshal(q) 81 | if err != nil { 82 | return queryB, err 83 | } 84 | filterB, err := json.Marshal(qd.FilterVal) 85 | if err != nil { 86 | return filterB, err 87 | } 88 | return []byte(fmt.Sprintf(`{"filtered":{"query":%s,"filter":%s}}`, queryB, filterB)), nil 89 | } 90 | return json.Marshal(q) 91 | } 92 | 93 | // get all 94 | func (q *QueryDsl) All() *QueryDsl { 95 | q.MatchAll = &MatchAll{""} 96 | return q 97 | } 98 | 99 | // Limit the query to this range 100 | func (q *QueryDsl) Range(fop *FilterOp) *QueryDsl { 101 | if q.FilterVal == nil { 102 | q.FilterVal = fop 103 | return q 104 | } 105 | // TODO: this is not valid, refactor 106 | q.FilterVal.Add(fop) 107 | return q 108 | } 109 | 110 | // Add a term search for a specific field 111 | // Term("user","kimchy") 112 | func (q *QueryDsl) Term(name, value string) *QueryDsl { 113 | if len(q.Terms) == 0 { 114 | q.Terms = make(map[string]string) 115 | } 116 | q.Terms[name] = value 117 | return q 118 | } 119 | 120 | // The raw search strings (lucene valid) 121 | func (q *QueryDsl) Search(searchFor string) *QueryDsl { 122 | //I don't think this is right, it is not a filter.query, it should be q query? 123 | qs := NewQueryString("", "") 124 | q.QueryEmbed.Qs = &qs 125 | q.QueryEmbed.Qs.Query = searchFor 126 | return q 127 | } 128 | 129 | // Querystring operations 130 | func (q *QueryDsl) Qs(qs *QueryString) *QueryDsl { 131 | q.QueryEmbed.Qs = qs 132 | return q 133 | } 134 | 135 | // Fields in query_string search 136 | // Fields("fieldname","search_for","","") 137 | // 138 | // Fields("fieldname,field2,field3","search_for","","") 139 | // 140 | // Fields("fieldname,field2,field3","search_for","field_exists","") 141 | func (q *QueryDsl) Fields(fields, search, exists, missing string) *QueryDsl { 142 | fieldList := strings.Split(fields, ",") 143 | qs := NewQueryString("", "") 144 | q.QueryEmbed.Qs = &qs 145 | q.QueryEmbed.Qs.Query = search 146 | if len(fieldList) == 1 { 147 | q.QueryEmbed.Qs.DefaultField = fields 148 | } else { 149 | q.QueryEmbed.Qs.Fields = fieldList 150 | } 151 | q.QueryEmbed.Qs.Exists = exists 152 | q.QueryEmbed.Qs.Missing = missing 153 | return q 154 | } 155 | 156 | // Filter this query 157 | func (q *QueryDsl) Filter(f *FilterOp) *QueryDsl { 158 | q.FilterVal = f 159 | return q 160 | } 161 | 162 | type MatchAll struct { 163 | All string `json:"-"` 164 | } 165 | 166 | // should we reuse QueryDsl here? 167 | type QueryWrap struct { 168 | Qs QueryString `json:"query_string,omitempty"` 169 | } 170 | 171 | // QueryString based search 172 | func NewQueryString(field, query string) QueryString { 173 | return QueryString{"", field, query, "", "", nil} 174 | } 175 | 176 | type QueryString struct { 177 | DefaultOperator string `json:"default_operator,omitempty"` 178 | DefaultField string `json:"default_field,omitempty"` 179 | Query string `json:"query,omitempty"` 180 | Exists string `json:"_exists_,omitempty"` 181 | Missing string `json:"_missing_,omitempty"` 182 | Fields []string `json:"fields,omitempty"` 183 | //_exists_:field1, 184 | //_missing_:field1, 185 | } 186 | 187 | // Generic Term based (used in query, facet, filter) 188 | type Term struct { 189 | Terms Terms `json:"terms,omitempty"` 190 | FilterVal *FilterWrap `json:"facet_filter,omitempty"` 191 | } 192 | 193 | type Terms struct { 194 | Fields []string `json:"field,omitempty"` 195 | Size string `json:"size,omitempty"` 196 | Regex string `json:"regex,omitempty"` 197 | } 198 | 199 | func NewTerm(fields ...string) *Term { 200 | m := &Term{Terms{Fields: fields}, nil} 201 | return m 202 | } 203 | 204 | func (s *Term) Filter(fl ...interface{}) *Term { 205 | if s.FilterVal == nil { 206 | s.FilterVal = NewFilterWrap() 207 | } 208 | 209 | s.FilterVal.addFilters(fl) 210 | return s 211 | } 212 | 213 | // Custom marshalling 214 | func (t *Terms) MarshalJSON() ([]byte, error) { 215 | m := make(map[string]interface{}) 216 | // TODO: this isn't getting called!? 217 | if len(t.Fields) == 1 { 218 | m["field"] = t.Fields[0] 219 | } else if len(t.Fields) > 1 { 220 | m["fields"] = t.Fields 221 | } 222 | if len(t.Regex) > 0 { 223 | m["regex"] = t.Regex 224 | } 225 | if len(t.Size) > 0 { 226 | m["size"] = t.Size 227 | } 228 | return json.Marshal(m) 229 | } 230 | -------------------------------------------------------------------------------- /core/search.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package core 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "github.com/buger/elastigo/api" 18 | "strconv" 19 | "strings" 20 | ) 21 | 22 | var ( 23 | DebugRequests = false 24 | ) 25 | 26 | // SearchRequest performs a very basic search on an index via the request URI API. 27 | // 28 | // params: 29 | // @index: the elasticsearch index 30 | // @_type: optional ("" if not used) search specific type in this index 31 | // @args: a map of URL parameters. Allows all the URI-request parameters allowed by ElasticSearch. 32 | // @query: this can be one of 3 types: 33 | // 1) string value that is valid elasticsearch 34 | // 2) io.Reader that can be set in body (also valid elasticsearch string syntax..) 35 | // 3) other type marshalable to json (also valid elasticsearch json) 36 | // 37 | // out, err := SearchRequest(true, "github", map[string]interface{} {"from" : 10}, qryType) 38 | // 39 | // http://www.elasticsearch.org/guide/reference/api/search/uri-request.html 40 | func SearchRequest(index string, _type string, args map[string]interface{}, query interface{}) (SearchResult, error) { 41 | var uriVal string 42 | var retval SearchResult 43 | if len(_type) > 0 && _type != "*" { 44 | uriVal = fmt.Sprintf("/%s/%s/_search", index, _type) 45 | } else { 46 | uriVal = fmt.Sprintf("/%s/_search", index) 47 | } 48 | body, err := api.DoCommand("POST", uriVal, args, query) 49 | if err != nil { 50 | return retval, err 51 | } 52 | if err == nil { 53 | // marshall into json 54 | jsonErr := json.Unmarshal([]byte(body), &retval) 55 | if jsonErr != nil { 56 | return retval, jsonErr 57 | } 58 | } 59 | return retval, err 60 | } 61 | 62 | // SearchUri performs the simplest possible query in url string 63 | // params: 64 | // @index: the elasticsearch index 65 | // @_type: optional ("" if not used) search specific type in this index 66 | // @args: a map of URL parameters. Most important one is q 67 | // 68 | // out, err := SearchUri("github","", map[string]interface{} { "q" : `user:kimchy`}) 69 | // 70 | // produces a request like this: host:9200/github/_search?q=user:kimchy" 71 | // 72 | // http://www.elasticsearch.org/guide/reference/api/search/uri-request.html 73 | func SearchUri(index, _type string, args map[string]interface{}) (SearchResult, error) { 74 | var uriVal string 75 | var retval SearchResult 76 | if len(_type) > 0 && _type != "*" { 77 | uriVal = fmt.Sprintf("/%s/%s/_search", index, _type) 78 | } else { 79 | uriVal = fmt.Sprintf("/%s/_search", index) 80 | } 81 | //log.Println(uriVal) 82 | body, err := api.DoCommand("GET", uriVal, args, nil) 83 | if err != nil { 84 | return retval, err 85 | } 86 | if err == nil { 87 | // marshall into json 88 | jsonErr := json.Unmarshal([]byte(body), &retval) 89 | if jsonErr != nil { 90 | return retval, jsonErr 91 | } 92 | } 93 | return retval, err 94 | } 95 | 96 | func Scroll(args map[string]interface{}, scroll_id string) (SearchResult, error) { 97 | var url string 98 | var retval SearchResult 99 | 100 | if _, ok := args["scroll"]; !ok { 101 | return retval, fmt.Errorf("Cannot call scroll without 'scroll' in arguments") 102 | } 103 | 104 | url = "/_search/scroll" 105 | 106 | body, err := api.DoCommand("POST", url, args, scroll_id) 107 | if err != nil { 108 | return retval, err 109 | } 110 | if err == nil { 111 | // marshall into json 112 | jsonErr := json.Unmarshal([]byte(body), &retval) 113 | if jsonErr != nil { 114 | return retval, jsonErr 115 | } 116 | } 117 | return retval, err 118 | } 119 | 120 | type SearchResult struct { 121 | Took int `json:"took"` 122 | TimedOut bool `json:"timed_out"` 123 | ShardStatus api.Status `json:"_shards"` 124 | Hits Hits `json:"hits"` 125 | Facets json.RawMessage `json:"facets,omitempty"` // structure varies on query 126 | ScrollId string `json:"_scroll_id,omitempty"` 127 | Aggregations json.RawMessage `json:"aggregations,omitempty"` // structure varies on query 128 | } 129 | 130 | func (s *SearchResult) String() string { 131 | return fmt.Sprintf("", s.Took, s.TimedOut, s.Hits.Total) 132 | } 133 | 134 | type Hits struct { 135 | Total int `json:"total"` 136 | // MaxScore float32 `json:"max_score"` 137 | Hits []Hit `json:"hits"` 138 | } 139 | 140 | func (h *Hits) Len() int { 141 | return len(h.Hits) 142 | } 143 | 144 | type Hit struct { 145 | Index string `json:"_index"` 146 | Type string `json:"_type,omitempty"` 147 | Id string `json:"_id"` 148 | Score Float32Nullable `json:"_score,omitempty"` // Filters (no query) dont have score, so is null 149 | Source *json.RawMessage `json:"_source"` // marshalling left to consumer 150 | Fields *json.RawMessage `json:"fields"` // when a field arg is passed to ES, instead of _source it returns fields 151 | Explanation *Explanation `json:"_explanation,omitempty"` 152 | } 153 | 154 | type Explanation struct { 155 | Value float32 `json:"value"` 156 | Description string `json:"description"` 157 | Details []*Explanation `json:"details,omitempty"` 158 | } 159 | 160 | func (e *Explanation) String(indent string) string { 161 | if len(e.Details) == 0 { 162 | return fmt.Sprintf("%s>>> %v = %s", indent, e.Value, strings.Replace(e.Description, "\n", "", -1)) 163 | } else { 164 | detailStrs := make([]string, 0) 165 | for _, detail := range e.Details { 166 | detailStrs = append(detailStrs, fmt.Sprintf("%s", detail.String(indent+"| "))) 167 | } 168 | return fmt.Sprintf("%s%v = %s(\n%s\n%s)", indent, e.Value, strings.Replace(e.Description, "\n", "", -1), strings.Join(detailStrs, "\n"), indent) 169 | } 170 | } 171 | 172 | // Elasticsearch returns some invalid (according to go) json, with floats having... 173 | // 174 | // json: cannot unmarshal null into Go value of type float32 (see last field.) 175 | // 176 | // "hits":{"total":6808,"max_score":null, 177 | // "hits":[{"_index":"10user","_type":"user","_id":"751820","_score":null, 178 | type Float32Nullable float32 179 | 180 | func (i *Float32Nullable) UnmarshalJSON(data []byte) error { 181 | if len(data) == 0 || string(data) == "null" { 182 | return nil 183 | } 184 | 185 | if in, err := strconv.ParseFloat(string(data), 32); err != nil { 186 | return err 187 | } else { 188 | *i = Float32Nullable(in) 189 | } 190 | return nil 191 | } 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | elastigo 2 | -------- 3 | [![Total views](https://sourcegraph.com/api/repos/github.com/mattbaird/elastigo/counters/views.png)](https://sourcegraph.com/github.com/mattbaird/elastigo) 4 | 5 | [![Build Status][1]][2] 6 | 7 | [1]: https://drone.io/github.com/mattbaird/elastigo/status.png 8 | [2]: https://drone.io/github.com/mattbaird/elastigo/latest 9 | 10 | 11 | A Go (Golang) based Elasticsearch client, implements core api for Indexing and searching. GoDoc http://godoc.org/github.com/mattbaird/elastigo 12 | 13 | To get the Chef based Vagrantfile working, be sure to pull like so:: 14 | 15 | # This will pull submodules. 16 | git clone --recursive git@github.com:mattbaird/elastigo.git 17 | 18 | It's easier to use the ElasticSearch provided Docker image found here: https://github.com/dockerfile/elasticsearch 19 | 20 | Non-persistent usage is: 21 | ```bash 22 | docker run -d -p 9200:9200 -p 9300:9300 dockerfile/elasticsearch 23 | ``` 24 | 25 | Quick Start with Docker 26 | ======================= 27 | Make sure docker is installed. If you are running docker on a mac, you must expose ports 9200 and 9300. Shut down docker: 28 | ```bash 29 | boot2docker stop 30 | ``` 31 | and run 32 | ```bash 33 | for i in {9200..9300}; do 34 | VBoxManage modifyvm "boot2docker-vm" --natpf1 "tcp-port$i,tcp,,$i,,$i"; 35 | VBoxManage modifyvm "boot2docker-vm" --natpf1 "udp-port$i,udp,,$i,,$i"; 36 | done 37 | ``` 38 | The following will allow you to get the code, and run the tests against your docker based non-persistent elasticsearch: 39 | 40 | ```bash 41 | docker run -d -p 9200:9200 -p 9300:9300 dockerfile/elasticsearch 42 | git clone git@github.com:mattbaird/elastigo.git 43 | cd elastigo 44 | go get -u ./... 45 | cd core 46 | go test -v -host localhost -loaddata 47 | cd .. 48 | go test -v ./... 49 | ``` 50 | 51 | status updates 52 | ======================== 53 | 54 | * *2014-5-21* Note: Drone.io tests are failing, I don't know why because the build and tests are working fine for me on my ubuntu box running the docker elasticsearch image. It's possible there is a timing issue. Any Ideas? 55 | * *2013-9-27* Fleshing out cluster and indices APIs, updated vagrant image to 0.90.3 56 | * *2013-7-10* Improvements/changes to bulk indexer (includes breaking changes to support TTL), 57 | Search dsl supports And/Or/Not 58 | * *SearchDsl* should still be considered beta at this 59 | point, there will be minor breaking changes as more of the 60 | elasticsearch feature set is implemented. 61 | * *2013-1-26* expansion of search dsl for greater coverage 62 | * *2012-12-30* new bulk indexing and search dsl 63 | * *2012-10-12* early in development, not ready for production yet. 64 | 65 | 66 | Adding content to Elasticsearch 67 | ---------------------------------------------- 68 | 69 | examples: 70 | ```go 71 | import "github.com/buger/elastigo/api" 72 | import "github.com/buger/elastigo/core" 73 | 74 | type Tweet struct { 75 | User string `json:"user"` 76 | Message string `json:"message"` 77 | } 78 | 79 | // Set the Elasticsearch Host to Connect to 80 | api.Domain = "localhost" 81 | // api.Port = "9300" 82 | 83 | // add single go struct entity 84 | response, _ := core.Index("twitter", "tweet", "1", nil, Tweet{"kimchy", "Search is cool"}) 85 | 86 | // you have bytes 87 | tw := Tweet{"kimchy", "Search is cool part 2"} 88 | bytesLine, err := json.Marshal(tw) 89 | response, _ := core.Index("twitter", "tweet", "2", nil, bytesLine) 90 | 91 | // Bulk Indexing 92 | t := time.Now() 93 | core.IndexBulk("twitter", "tweet", "3", &t, Tweet{"kimchy", "Search is now cooler"}) 94 | 95 | // Search Using Raw json String 96 | searchJson := `{ 97 | "query" : { 98 | "term" : { "user" : "kimchy" } 99 | } 100 | }` 101 | out, err := core.SearchRequest(true, "twitter", "tweet", searchJson, "") 102 | if len(out.Hits.Hits) == 1 { 103 | fmt.Println(string(out.Hits.Hits[0].Source)) 104 | } 105 | ``` 106 | 107 | Search DSL Examples 108 | ------------------------- 109 | 110 | A Faceted, ranged Search using the `Search DSL` : 111 | 112 | ```go 113 | import "github.com/buger/elastigo/api" 114 | import "github.com/buger/elastigo/core" 115 | 116 | // Set the Elasticsearch Host to Connect to 117 | api.Domain = "localhost" 118 | // api.Port = "9300" 119 | 120 | out, err := Search("github").Size("1").Facet( 121 | Facet().Fields("actor").Size("500"), 122 | ).Query( 123 | Query().Range( 124 | Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), 125 | ).Search("add"), 126 | ).Result() 127 | ``` 128 | 129 | A Ranged Search using the `Search DSL` : 130 | 131 | ```go 132 | out, err := Search("github").Type("Issues").Pretty().Query( 133 | Query().Range( 134 | Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), 135 | ).Search("add"), 136 | ).Result() 137 | ``` 138 | 139 | A Simple Search using the `Search DSL` : 140 | 141 | ```go 142 | out, err := Search("github").Type("Issues").Size("100").Search("add").Result() 143 | ``` 144 | 145 | A Direct Search using the api : 146 | 147 | ```go 148 | qry := map[string]interface{}{ 149 | "query":map[string]interface{}{ 150 | "term":map[string]string{"user:"kimchy"}, 151 | }, 152 | } 153 | core.SearchRequest(true, "github", "Issues", qry, "", 0) 154 | ``` 155 | 156 | A Direct Search using the query string Api : 157 | 158 | ```go 159 | core.SearchUri("github", "Issues", "user:kimchy", "", 0) 160 | ``` 161 | 162 | A Filtered search `Search DSL` : 163 | 164 | ```go 165 | out, err := Search("github").Filter( 166 | Filter().Exists("repository.name"), 167 | ).Result() 168 | ``` 169 | 170 | Adding content to Elasticsearch in Bulk 171 | ---------------------------------------------- 172 | 173 | example: 174 | 175 | ```go 176 | import "github.com/buger/elastigo/api" 177 | import "github.com/buger/elastigo/core" 178 | 179 | // Set the Elasticsearch Host to Connect to 180 | api.Domain = "localhost" 181 | // api.Port = "9300" 182 | 183 | indexer := core.NewBulkIndexerErrors(10, 60) 184 | done := make(chan bool) 185 | indexer.Run(done) 186 | 187 | go func() { 188 | for errBuf := range indexer.ErrorChannel { 189 | // just blissfully print errors forever 190 | fmt.Println(errBuf.Err) 191 | } 192 | }() 193 | for i := 0; i < 20; i++ { 194 | indexer.Index("twitter", "user", strconv.Itoa(i), "", nil, `{"name":"bob"}`, false) 195 | } 196 | done <- true 197 | // Indexing might take a while. So make sure the program runs 198 | // a little longer when trying this in main. 199 | ``` 200 | 201 | license 202 | ======= 203 | Copyright 2012 Matthew Baird, Aaron Raddon, and more! 204 | 205 | Licensed under the Apache License, Version 2.0 (the "License"); 206 | you may not use this file except in compliance with the License. 207 | You may obtain a copy of the License at 208 | 209 | http://www.apache.org/licenses/LICENSE-2.0 210 | 211 | Unless required by applicable law or agreed to in writing, software 212 | distributed under the License is distributed on an "AS IS" BASIS, 213 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 214 | See the License for the specific language governing permissions and 215 | limitations under the License. 216 | -------------------------------------------------------------------------------- /cluster/nodesInfo.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | package cluster 12 | 13 | import ( 14 | "encoding/json" 15 | "fmt" 16 | "github.com/buger/elastigo/api" 17 | "strings" 18 | ) 19 | 20 | // The cluster nodes info API allows to retrieve one or more (or all) of the cluster nodes information. 21 | // informatino can be one of jvm, process 22 | func AllNodesInfo() (NodeInfo, error) { 23 | return NodesInfo([]string{"_all"}, "_all") 24 | } 25 | 26 | func NodesInfo(information []string, nodes ...string) (NodeInfo, error) { 27 | var url string 28 | var retval NodeInfo 29 | url = fmt.Sprintf("/_nodes/%s/%s", strings.Join(nodes, ","), strings.Join(information, ",")) 30 | body, err := api.DoCommand("GET", url, nil, nil) 31 | if err != nil { 32 | return retval, err 33 | } 34 | if err == nil { 35 | // marshall into json 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | } 41 | return retval, err 42 | } 43 | 44 | type NodeInfo struct { 45 | ClusterName string `json:"cluster_name"` 46 | Nodes map[string]Node `json:"nodes"` // node name is random string 47 | } 48 | 49 | type Node struct { 50 | Name string `json:"name,omitempty"` 51 | TransportAddress string `json:"transport_address,omitempty"` 52 | Host string `json:"host,omitempty"` 53 | Ip string `json:"ip,omitempty"` 54 | Version string `json:"version,omitempty"` 55 | Build string `json:"build,omitempty"` 56 | Hostname string `json:"hostname,omitempty"` 57 | HttpAddress string `json:"http_address,omitempty"` 58 | Settings *Settings `json:"settings,omitempty"` 59 | OS *OS `json:"os,omitempty"` 60 | Process *Process `json:"process,omitempty"` 61 | JVM *JVM `json:"jvm,omitempty"` 62 | ThreadPool *ThreadPool `json:"thread_pool,omitempty"` 63 | Network *Network `json:"network,omitempty"` 64 | Transport *Transport `json:"transport,omitempty"` 65 | Http *Http `json:"http,omitempty"` 66 | Plugins []*Plugin `json:"plugins,omitempty"` 67 | } 68 | 69 | type Settings struct { 70 | Path *Path `json:"path,omitempty"` 71 | Foreground string `json:"foreground,omitempty"` 72 | Name string `json:"name,omitempty"` 73 | } 74 | 75 | type Path struct { 76 | Logs string `json:"logs,omitempty"` 77 | home string `json:"home,omitempty"` 78 | } 79 | 80 | type Cluster struct { 81 | Name string `json:"name"` 82 | } 83 | 84 | type OS struct { 85 | RefreshInterval int `json:"refresh_interval,omitempty"` 86 | AvailableProcessors int `json:"available_processors,omitempty"` 87 | } 88 | 89 | type CPU struct { 90 | Vendor string `json:"vendor,omitempty"` 91 | Model string `json:"model,omitempty"` 92 | Mhz int `json:"mhz,omitempty"` 93 | TotalCores int `json:"total_cores,omitempty"` 94 | TotalSockets int `json:"total_sockets,omitempty"` 95 | CoresPerSocket int `json:"cores_per_socket,omitempty"` 96 | CacheSizeInBytes int `json:"cache_size_in_bytes,omitempty"` 97 | } 98 | 99 | type MEM struct { 100 | TotalInBytes int `json:"total_in_bytes,omitempty"` 101 | } 102 | 103 | type SWAP struct { 104 | TotalInBytes int `json:"total_in_bytes,omitempty"` 105 | } 106 | 107 | type Process struct { 108 | RefreshInterval int `json:"refresh_interval,omitempty"` 109 | Id int `json:"id,omitempty"` 110 | MaxFileDescriptors int `json:"max_file_descriptors,omitempty"` 111 | Mlockall bool `json:"mlockall,omitempty"` 112 | } 113 | 114 | type JVM struct { 115 | Pid int `json:"pid,omitempty"` 116 | Version string `json:"version,omitempty"` 117 | VMName string `json:"vm_name,omitempty"` 118 | VMVersion string `json:"vm_version,omitempty"` 119 | VMVendor string `json:"vm_vendor,omitempty"` 120 | StartTime int `json:"start_time,omitempty"` 121 | Mem *JvmMem `json:"mem,omitempty"` 122 | GcCollectors []string `json:"gc_collectors,omitempty"` 123 | MemoryPools []string `json:"memory_pools,omitempty"` 124 | } 125 | 126 | type JvmMem struct { 127 | HeapInitInBytes int `json:"heap_init_in_bytes,omitempty"` 128 | HeapMaxInBytes int `json:"heap_max_in_bytes,omitempty"` 129 | NonHeapInitInBytes int `json:"non_heap_init_in_bytes,omitempty"` 130 | NonHeapMaxInBytes int `json:"non_heap_max_in_bytes,omitempty"` 131 | DirectMaxInBytes int `json:"direct_max_in_bytes,omitempty"` 132 | } 133 | 134 | type ThreadPool struct { 135 | Generic *ThreadPoolConfig `json:"generic,omitempty"` 136 | Index *ThreadPoolConfig `json:"index,omitempty"` 137 | Get *ThreadPoolConfig `json:"get,omitempty"` 138 | Snapshot *ThreadPoolConfig `json:"snapshot,omitempty"` 139 | Merge *ThreadPoolConfig `json:"merge,omitempty"` 140 | Suggest *ThreadPoolConfig `json:"suggest,omitempty"` 141 | Bulk *ThreadPoolConfig `json:"bulk,omitempty"` 142 | Optimize *ThreadPoolConfig `json:"optimize,omitempty"` 143 | Warmer *ThreadPoolConfig `json:"warmer,omitempty"` 144 | Flush *ThreadPoolConfig `json:"flush,omitempty"` 145 | Search *ThreadPoolConfig `json:"search,omitempty"` 146 | Percolate *ThreadPoolConfig `json:"percolate,omitempty"` 147 | Management *ThreadPoolConfig `json:"management,omitempty"` 148 | Refresh *ThreadPoolConfig `json:"refresh,omitempty"` 149 | } 150 | 151 | type ThreadPoolConfig struct { 152 | Type string `json:"type,omitempty"` 153 | Min int `json:"min,omitempty"` 154 | Max int `json:"max,omitempty"` 155 | QueueSize string `json:"queue_size,omitempty"` 156 | KeepAlive string `json:"keep_alive,omitempty"` 157 | } 158 | 159 | type Network struct { 160 | RefreshInterval int `json:"refresh_interval,omitempty"` 161 | PrimaryInterface *Interface `json:"primary_interface,omitempty"` 162 | } 163 | 164 | type Interface struct { 165 | Address string `json:"address,omitempty"` 166 | Name string `json:"name,omitempty"` 167 | MacAddress string `json:"mac_address,omitempty"` 168 | } 169 | 170 | type Transport struct { 171 | BoundAddress string `json:"bound_address,omitempty"` 172 | PublishAddress string `json:"publish_address,omitempty"` 173 | } 174 | 175 | type Http struct { 176 | BoundAddress string `json:"bound_address,omitempty"` 177 | PublishAddress string `json:"publish_address,omitempty"` 178 | } 179 | 180 | type Plugin struct { 181 | Name string `json:"name,omitempty"` 182 | Description string `json:"description,omitempty"` 183 | Site bool `json:"site,omitempty"` 184 | Jvm bool `json:"jvm,omitempty"` 185 | Url string `json:"url,omitempty"` 186 | } 187 | --------------------------------------------------------------------------------