├── .drone.yml ├── .gitignore ├── .gitmodules ├── .travis.yml ├── HACKING.md ├── LICENSE ├── README.md ├── Vagrantfile ├── client.go ├── cookbooks ├── build-essential │ ├── CHANGELOG.md │ ├── CONTRIBUTING │ ├── LICENSE │ ├── README.md │ ├── attributes │ │ └── default.rb │ ├── metadata.rb │ └── recipes │ │ └── default.rb ├── git │ ├── CHANGELOG.md │ ├── CONTRIBUTING │ ├── LICENSE │ ├── README.md │ ├── attributes │ │ └── default.rb │ ├── metadata.rb │ ├── recipes │ │ ├── default.rb │ │ ├── server.rb │ │ └── source.rb │ └── templates │ │ └── default │ │ ├── sv-git-daemon-log-run.erb │ │ └── sv-git-daemon-run.erb ├── golang │ ├── README.md │ ├── attributes │ │ └── default.rb │ ├── metadata.rb │ └── recipes │ │ ├── default.rb │ │ └── ppa.rb └── mercurial │ ├── CHANGELOG.md │ ├── CONTRIBUTING │ ├── LICENSE │ ├── README.md │ ├── metadata.rb │ ├── providers │ └── default.rb │ ├── recipes │ └── default.rb │ └── resources │ └── default.rb ├── doc.go ├── lib ├── baserequest.go ├── baseresponse.go ├── cataliasinfo.go ├── cataliasinfo_test.go ├── catindexinfo.go ├── catindexinfo_test.go ├── catnodeinfo.go ├── catnodeinfo_test.go ├── catresponses.go ├── catshardinfo.go ├── catshardinfo_test.go ├── clusterhealth.go ├── clusterhealthresponses.go ├── clusternodeshotthreads.go ├── clusternodesinfo.go ├── clusternodesinfo_test.go ├── clusternodesshutdown.go ├── clusternodesstats.go ├── clusterreroute.go ├── clusterstate.go ├── clusterstatresponses.go ├── clusterupdatesettings.go ├── connection.go ├── connection_test.go ├── corebulk.go ├── corebulk_test.go ├── corebulkudp.go ├── corecount.go ├── coredelete.go ├── coredeletebyquery.go ├── coreexample_test.go ├── coreexplain.go ├── coreget.go ├── coreindex.go ├── coremget.go ├── coremorelikethis.go ├── coremsearch.go ├── corepercolate.go ├── corepercolate_test.go ├── coresearch.go ├── coresearch_test.go ├── coretest_test.go ├── coreupdate.go ├── corevalidate.go ├── error.go ├── indicesaliases.go ├── indicesanalyze.go ├── indicesclearcache.go ├── indicescreateindex.go ├── indicesdeleteindex.go ├── indicesdeletemapping.go ├── indicesdeletemapping_test.go ├── indicesdoc.go ├── indicesflush.go ├── indicesgetsettings.go ├── indicesindicesexists.go ├── indicesopencloseindex.go ├── indicesoptimize.go ├── indicesputmapping.go ├── indicesputmapping_test.go ├── indicesputsettings.go ├── indicesrefresh.go ├── indicessegments.go ├── indicessnapshot.go ├── indicesstats.go ├── indicesstatus.go ├── indicestemplates.go ├── indicesupdatesettings.go ├── request.go ├── request_test.go ├── searchaggregate.go ├── searchaggregate_test.go ├── searchdsl.go ├── searchfacet.go ├── searchfacet_test.go ├── searchfilter.go ├── searchfilter_test.go ├── searchhighlight.go ├── searchhighlight_test.go ├── searchquery.go ├── searchreadme ├── searchsearch.go ├── searchsearch_test.go ├── searchsort.go ├── setup_test.go ├── shared.go ├── shared_test.go └── snapshot.go └── tutorial ├── README.md └── start_1.go /.drone.yml: -------------------------------------------------------------------------------- 1 | script: 2 | - go get -t ./... 3 | - go get github.com/bmizerany/assert 4 | - cd lib 5 | - go build 6 | - go test -v -loaddata 7 | - go test -v 8 | 9 | services: 10 | - dockerfile/elasticsearch 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | .DS_Store 10 | .vagrant 11 | 12 | # Architecture specific extensions/prefixes 13 | *.[568vq] 14 | [568vq].out 15 | 16 | *.cgo1.go 17 | *.cgo2.c 18 | _cgo_defun.c 19 | _cgo_gotypes.go 20 | _cgo_export.* 21 | 22 | _testmain.go 23 | 24 | *.exe 25 | 26 | elastigo 27 | 28 | # IDE 29 | 30 | *.iml 31 | .idea/ 32 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "java"] 2 | path = java 3 | url = https://github.com/cookbooks/java.git 4 | [submodule "cookbooks/java"] 5 | path = cookbooks/java 6 | url = https://github.com/cookbooks/java.git 7 | [submodule "cookbooks/apt"] 8 | path = cookbooks/apt 9 | url = https://github.com/opscode-cookbooks/apt.git 10 | [submodule "cookbooks/elasticsearch"] 11 | path = cookbooks/elasticsearch 12 | url = https://github.com/mattbaird/elasticsearch-chef.git 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | services: 2 | - elasticsearch 3 | 4 | language: go 5 | 6 | go: 7 | - 1.6 8 | 9 | script: 10 | - go get -t ./... 11 | - go build ./lib/ 12 | - cd ./lib/ && go test -v -host localhost -loaddata 13 | - go install 14 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | 2 | Testing 3 | ----------------- 4 | 5 | To run tests, this library loads data into an elasticsearch server and tests against that. 6 | 7 | See api/coretest_test.go. The data set should remain the same as it pulls a known set of github archive data. 8 | 9 | usage: 10 | 11 | $cd core 12 | 13 | $go test -v -host eshost -loaddata # load the data 14 | 15 | $go test -v -host eshost # without load data, which only needs to run once 16 | 17 | Clean out the Elasticsearch index: 18 | 19 | http -v DELETE http://localhost:9200/github 20 | or 21 | curl -XDELETE http://localhost:9200/github 22 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure("2") do |config| 5 | config.vm.box = "lucid64" 6 | config.vm.box_url = "http://files.vagrantup.com/lucid64.box" 7 | config.vm.network :forwarded_port, guest: 9300, host: 9300, auto_correct: true 8 | config.vm.provision :shell, :inline => "gem install chef --version 10.26.0 --no-rdoc --no-ri --conservative" 9 | 10 | config.vm.provider :virtualbox do |vb| 11 | vb.gui = false 12 | vb.customize ["modifyvm", :id, "--memory", "1024"] 13 | vb.customize ["modifyvm", :id, "--cpus", "1"] 14 | # This allows symlinks to be created within the /vagrant root directory, 15 | # which is something librarian-puppet needs to be able to do. This might 16 | # be enabled by default depending on what version of VirtualBox is used. 17 | vb.customize ["setextradata", :id, "VBoxInternal2/SharedFoldersEnableSymlinksCreate/v-root", "1"] 18 | end 19 | config.vm.provision :chef_solo do |chef| 20 | chef.cookbooks_path = "cookbooks" 21 | chef.add_recipe("apt") 22 | chef.add_recipe("java") 23 | chef.add_recipe("elasticsearch") 24 | chef.add_recipe("git") 25 | chef.add_recipe("mercurial") 26 | chef.add_recipe("build-essential") 27 | chef.add_recipe("golang") 28 | end 29 | end -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Matthew Baird 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | package main 15 | 16 | import ( 17 | "encoding/json" 18 | "flag" 19 | "log" 20 | "time" 21 | 22 | elastigo "github.com/mattbaird/elastigo/lib" 23 | ) 24 | 25 | var ( 26 | eshost *string = flag.String("host", "localhost", "Elasticsearch Server Host Address") 27 | ) 28 | 29 | // for testing 30 | func main() { 31 | flag.Parse() 32 | log.SetFlags(log.Ltime | log.Lshortfile) 33 | 34 | c := elastigo.NewConn() 35 | c.Domain = *eshost 36 | response, _ := c.Index("twitter", "tweet", "1", nil, NewTweet("kimchy", "Search is cool")) 37 | c.Flush() 38 | log.Printf("Index OK: %v", response.Ok) 39 | searchresponse, err := c.Search("twitter", "tweet", nil, "{\"query\" : {\"term\" : { \"user\" : \"kimchy\" }}}") 40 | if err != nil { 41 | log.Println("error during search:" + err.Error()) 42 | log.Fatal(err) 43 | } 44 | // try marshalling to tweet type 45 | var t Tweet 46 | bytes, err := searchresponse.Hits.Hits[0].Source.MarshalJSON() 47 | if err != nil { 48 | log.Fatalf("err calling marshalJson:%v", err) 49 | } 50 | json.Unmarshal(bytes, &t) 51 | log.Printf("Search Found: %s", t) 52 | response, _ = c.Get("twitter", "tweet", "1", nil) 53 | log.Printf("Get: %v", response.Exists) 54 | exists, _ := c.Exists("twitter", "tweet", "1", nil) 55 | log.Printf("Exists: %v", exists) 56 | c.Flush() 57 | countResponse, _ := c.Count("twitter", "tweet", nil, nil) 58 | 59 | log.Printf("Count: %v", countResponse.Count) 60 | response, _ = c.Delete("twitter", "tweet", "1", map[string]interface{}{"version": -1, "routing": ""}) 61 | log.Printf("Delete OK: %v", response.Ok) 62 | response, _ = c.Get("twitter", "tweet", "1", nil) 63 | log.Printf("Get: %v", response.Exists) 64 | 65 | healthResponse, _ := c.Health() 66 | log.Printf("Health: %v", healthResponse.Status) 67 | 68 | c.UpdateSettings("transient", "discovery.zen.minimum_master_nodes", 2) 69 | } 70 | 71 | // used in test suite, chosen to be similar to the documentation 72 | type Tweet struct { 73 | User string `json:"user"` 74 | PostDate time.Time `json:"postDate"` 75 | Message string `json:"message"` 76 | } 77 | 78 | func NewTweet(user string, message string) Tweet { 79 | return Tweet{User: user, PostDate: time.Now(), Message: message} 80 | } 81 | 82 | func (t *Tweet) String() string { 83 | b, _ := json.Marshal(t) 84 | return string(b) 85 | } 86 | -------------------------------------------------------------------------------- /cookbooks/build-essential/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.1.2: 2 | 3 | * [COOK-1620] - support OS X 10.8 4 | 5 | ## v1.1.0: 6 | 7 | * [COOK-1098] - support amazon linux 8 | * [COOK-1149] - support Mac OS X 9 | * [COOK-1296] - allow for compile-time installation of packages 10 | through an attribute (see README) 11 | 12 | ## v1.0.2: 13 | 14 | * [COOK-1098] - Add Amazon Linux platform support 15 | * [COOK-1149] - Add OS X platform support 16 | -------------------------------------------------------------------------------- /cookbooks/build-essential/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /cookbooks/build-essential/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs packages required for compiling C software from source. Use 5 | this cookbook if you wish to compile C programs, or install RubyGems 6 | with native extensions. 7 | 8 | Requirements 9 | ============ 10 | 11 | ## Platform 12 | 13 | Supported platforms by platform family: 14 | 15 | * Linux (debian, rhel, fedora) 16 | * Darwin (`mac_os_x` 10.6+) 17 | 18 | Attributes 19 | ========== 20 | 21 | * `node['build_essential']['compiletime']` - Whether the resources in 22 | the default recipe should be configured at the "Compile" phase of the 23 | Chef run. Defaults to false, see __Usage__ for more information. 24 | * `node['build_essential']['osx']['gcc_installer_url']` - The URL of 25 | the OS X GCC package installer (.pkg). 26 | * `node['build_essential']['osx']['gcc_installer_checksum']` - The 27 | SHA256 checksum of the OS X GCC installer. 28 | 29 | Recipes 30 | ======= 31 | 32 | This cookbook has one recipe, default. 33 | 34 | On Linux platforms (see __Platform__ above for a supported list of 35 | families), packages required to build C source projects are installed. 36 | This includes GCC, make, autconf and others. On Debian-family 37 | distributions, the apt-cache may need to be updated, especially during 38 | compile time installation. See __Usage__ for further information. 39 | 40 | On Mac OS X, the GCC standalone installer by Kenneth Reitz is 41 | installed. Note that this is *not* the Xcode CLI package, as that does 42 | not include all programs and headers required to build some common 43 | GNU-style C projects, such as those that are available from projects 44 | such as MacPorts or Homebrew. Changing the attributes for the GCC 45 | installer URL and checksum to the Xcode values may work, but this is 46 | untested. 47 | 48 | Usage 49 | ===== 50 | 51 | Simply include the `build-essential` and the required tools will be 52 | installed to the system, and later recipes will be able to compile 53 | software from C source code. 54 | 55 | For RubyGems that include native C extensions you wish to use with 56 | Chef, you should do two things. 57 | 58 | 0. Ensure that the C libraries, include files and other assorted "dev" 59 | type packages are installed. You should do this in the compile phase 60 | after the build-essential recipe. 61 | 1. Use the `chef_gem` resource in your recipes. This requires Chef version 0.10.10+. 62 | 2. Set the `compiletime` attribute in roles where such recipes are 63 | required. This will ensure that the build tools are available to 64 | compile the RubyGems' extensions, as `chef_gem` happens during the 65 | compile phase, too. 66 | 67 | Example installation of a devel package at compile-time in a recipe: 68 | 69 | package "mypackage-dev" do 70 | action :nothing 71 | end.run_action(:install) 72 | 73 | Example use of `chef_gem`: 74 | 75 | chef_gem "mygem" 76 | 77 | Example role: 78 | 79 | name "myapp" 80 | run_list( 81 | "recipe[build-essential]", 82 | "recipe[myapp]" 83 | ) 84 | default_attributes( 85 | "build_essential" => { 86 | "compiletime" => true 87 | } 88 | ) 89 | 90 | The compile time option (via the attribute) is to ensure that the 91 | proper packages are available at the right time in the Chef run. It is 92 | recommended that the build-essential recipe appear early in the run 93 | list. 94 | 95 | The Chef wiki has documentation on 96 | [the anatomy of a chef run](http://wiki.opscode.com/display/chef/Anatomy+of+a+Chef+Run). 97 | 98 | Limitations 99 | =========== 100 | 101 | It is not in the scope of this cookbook to handle installing the 102 | required headers for individual software projects in order to compile 103 | them, or to compile RubyGems with native C extensions. You should 104 | create a cookbook for handling that. 105 | 106 | License and Author 107 | ================== 108 | 109 | Author:: Joshua Timberman () 110 | Author:: Seth Chisamore () 111 | 112 | Copyright 2009-2011, Opscode, Inc. () 113 | 114 | Licensed under the Apache License, Version 2.0 (the "License"); 115 | you may not use this file except in compliance with the License. 116 | You may obtain a copy of the License at 117 | 118 | http://www.apache.org/licenses/LICENSE-2.0 119 | 120 | Unless required by applicable law or agreed to in writing, software 121 | distributed under the License is distributed on an "AS IS" BASIS, 122 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 123 | See the License for the specific language governing permissions and 124 | limitations under the License. 125 | -------------------------------------------------------------------------------- /cookbooks/build-essential/attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: build-essential 3 | # Attributes:: default 4 | # 5 | # Copyright 2008-2012, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | default['build_essential']['compiletime'] = false 21 | 22 | case platform 23 | when "mac_os_x" 24 | case 25 | when Chef::VersionConstraint.new("~> 10.7.0").include?(platform_version), 26 | Chef::VersionConstraint.new("~> 10.8.0").include?(platform_version) 27 | default['build_essential']['osx']['gcc_installer_url'] = "https://github.com/downloads/kennethreitz/osx-gcc-installer/GCC-10.7-v2.pkg" 28 | default['build_essential']['osx']['gcc_installer_checksum'] = "df36aa87606feb99d0db9ac9a492819e" 29 | when Chef::VersionConstraint.new("~> 10.6.0").include?(platform_version) 30 | default['build_essential']['osx']['gcc_installer_url'] = "https://github.com/downloads/kennethreitz/osx-gcc-installer/GCC-10.6.pkg" 31 | default['build_essential']['osx']['gcc_installer_checksum'] = "d1db5bab6a3f6b9f3b5577a130baeefa" 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /cookbooks/build-essential/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs C compiler / build tools" 5 | version "1.1.2" 6 | recipe "build-essential", "Installs packages required for compiling C software from source." 7 | 8 | %w{ fedora redhat centos ubuntu debian amazon }.each do |os| 9 | supports os 10 | end 11 | 12 | supports "mac_os_x", ">= 10.6.0" 13 | -------------------------------------------------------------------------------- /cookbooks/build-essential/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: build-essential 3 | # Recipe:: default 4 | # 5 | # Copyright 2008-2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | require 'chef/shell_out' 21 | 22 | compiletime = node['build_essential']['compiletime'] 23 | 24 | case node['os'] 25 | when "linux" 26 | 27 | # on apt-based platforms when first provisioning we need to force 28 | # apt-get update at compiletime if we are going to try to install at compiletime 29 | if node['platform_family'] == "debian" 30 | execute "apt-get update" do 31 | action :nothing 32 | # tip: to suppress this running every time, just use the apt cookbook 33 | not_if do 34 | ::File.exists?('/var/lib/apt/periodic/update-success-stamp') && 35 | ::File.mtime('/var/lib/apt/periodic/update-success-stamp') > Time.now - 86400*2 36 | end 37 | end.run_action(:run) if compiletime 38 | end 39 | 40 | packages = case node['platform_family'] 41 | when "debian" 42 | %w{build-essential binutils-doc} 43 | when "rhel", "fedora" 44 | %w{gcc gcc-c++ kernel-devel make} 45 | end 46 | 47 | packages.each do |pkg| 48 | r = package pkg do 49 | action ( compiletime ? :nothing : :install ) 50 | end 51 | r.run_action(:install) if compiletime 52 | end 53 | 54 | %w{autoconf flex bison}.each do |pkg| 55 | r = package pkg do 56 | action ( compiletime ? :nothing : :install ) 57 | end 58 | r.run_action(:install) if compiletime 59 | end 60 | when "darwin" 61 | result = Chef::ShellOut.new("pkgutil --pkgs").run_command 62 | installed = result.stdout.split("\n").include?("com.apple.pkg.gcc4.2Leo") 63 | pkg_filename = File.basename(node['build_essential']['osx']['gcc_installer_url']) 64 | pkg_path = "#{Chef::Config[:file_cache_path]}/#{pkg_filename}" 65 | 66 | r = remote_file pkg_path do 67 | source node['build_essential']['osx']['gcc_installer_url'] 68 | checksum node['build_essential']['osx']['gcc_installer_checksum'] 69 | action ( compiletime ? :nothing : :create ) 70 | not_if { installed } 71 | end 72 | r.run_action(:create) if compiletime 73 | 74 | r = execute "sudo installer -pkg \"#{pkg_path}\" -target /" do 75 | action ( compiletime ? :nothing : :run ) 76 | not_if { installed } 77 | end 78 | r.run_action(:run) if compiletime 79 | end 80 | -------------------------------------------------------------------------------- /cookbooks/git/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.0.2: 2 | 3 | * [COOK-1537] - add recipe for source installation 4 | 5 | ## v1.0.0: 6 | 7 | * [COOK-1152] - Add support for Mac OS X 8 | * [COOK-1112] - Add support for Windows 9 | 10 | ## v0.10.0: 11 | 12 | * [COOK-853] - Git client installation on CentOS 13 | 14 | ## v0.9.0: 15 | 16 | * Current public release. 17 | -------------------------------------------------------------------------------- /cookbooks/git/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /cookbooks/git/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs git and optionally sets up a git server as a daemon under runit. 5 | 6 | Requirements 7 | ============ 8 | 9 | ## Platform: 10 | 11 | * Debian/Ubuntu 12 | * ArchLinux 13 | 14 | ## Cookbooks: 15 | 16 | * runit 17 | 18 | Recipes 19 | ======= 20 | 21 | ## default 22 | 23 | Installs base git packages based on platform. 24 | 25 | ## server 26 | 27 | Sets up a git daemon to provide a server. 28 | 29 | ## source 30 | 31 | Installs git from source. 32 | 33 | Usage 34 | ===== 35 | 36 | This cookbook primarily installs git core packages. It can also be 37 | used to serve git repositories. 38 | 39 | include_recipe "git::server" 40 | 41 | This creates the directory /srv/git and starts a git daemon, exporting 42 | all repositories found. Repositories need to be added manually, but 43 | will be available once they are created. 44 | 45 | License and Author 46 | ================== 47 | 48 | Author:: Joshua Timberman () 49 | 50 | Copyright:: 2009-2012, Opscode, Inc. 51 | 52 | Licensed under the Apache License, Version 2.0 (the "License"); 53 | you may not use this file except in compliance with the License. 54 | You may obtain a copy of the License at 55 | 56 | http://www.apache.org/licenses/LICENSE-2.0 57 | 58 | Unless required by applicable law or agreed to in writing, software 59 | distributed under the License is distributed on an "AS IS" BASIS, 60 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 61 | See the License for the specific language governing permissions and 62 | limitations under the License. 63 | -------------------------------------------------------------------------------- /cookbooks/git/attributes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Author:: Jamie Winsor () 3 | # Cookbook Name:: git 4 | # Attributes:: default 5 | # 6 | # Copyright 2008-2012, Opscode, Inc. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | case platform_family 21 | when 'windows' 22 | set[:git][:version] = "1.7.9-preview20120201" 23 | set[:git][:url] = "http://msysgit.googlecode.com/files/Git-#{node[:git][:version]}.exe" 24 | set[:git][:checksum] = "0627394709375140d1e54e923983d259a60f9d8e" 25 | when "mac_os_x" 26 | default[:git][:osx_dmg][:app_name] = "git-1.7.9.4-intel-universal-snow-leopard" 27 | default[:git][:osx_dmg][:volumes_dir] = "Git 1.7.9.4 Snow Leopard Intel Universal" 28 | default[:git][:osx_dmg][:package_id] = "GitOSX.Installer.git1794.git.pkg" 29 | default[:git][:osx_dmg][:url] = "http://git-osx-installer.googlecode.com/files/git-1.7.9.4-intel-universal-snow-leopard.dmg" 30 | default[:git][:osx_dmg][:checksum] = "661c3fcf765572d3978df17c7636d59e" 31 | else 32 | default[:git][:prefix] = "/usr/local" 33 | default[:git][:version] = "1.7.11.4" 34 | default[:git][:url] = "https://github.com/git/git/tarball/v#{node[:git][:version]}" 35 | default[:git][:checksum] = "7a26d9bd0fd3384374bdc1afaae829f406bc123126817d994a460c49a3260ecc" 36 | end 37 | -------------------------------------------------------------------------------- /cookbooks/git/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs git and/or sets up a Git server daemon" 5 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 6 | version "1.0.2" 7 | recipe "git", "Installs git" 8 | recipe "git::server", "Sets up a runit_service for git daemon" 9 | recipe "git::source", "Installs git from source" 10 | 11 | %w{ amazon arch centos debian fedora redhat scientific ubuntu windows }.each do |os| 12 | supports os 13 | end 14 | 15 | supports "mac_os_x", ">= 10.6.0" 16 | 17 | %w{ build-essential dmg runit yum }.each do |cb| 18 | depends cb 19 | end 20 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: default 4 | # 5 | # Copyright 2008-2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | case node[:platform] 20 | when "debian", "ubuntu" 21 | package "git-core" 22 | when "centos","redhat","scientific","fedora" 23 | case node[:platform_version].to_i 24 | when 5 25 | include_recipe "yum::epel" 26 | end 27 | package "git" 28 | when "windows" 29 | windows_package "git" do 30 | source node[:git][:url] 31 | checksum node[:git][:checksum] 32 | action :install 33 | not_if { File.exists? 'C:\Program Files (x86)\Git\bin\git.exe' } 34 | end 35 | when "mac_os_x" 36 | dmg_package "GitOSX-Installer" do 37 | app node[:git][:osx_dmg][:app_name] 38 | package_id node[:git][:osx_dmg][:package_id] 39 | volumes_dir node[:git][:osx_dmg][:volumes_dir] 40 | source node[:git][:osx_dmg][:url] 41 | checksum node[:git][:osx_dmg][:checksum] 42 | type "pkg" 43 | action :install 44 | end 45 | else 46 | package "git" 47 | end 48 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/server.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: server 4 | # 5 | # Copyright 2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | include_recipe "git" 20 | 21 | directory "/srv/git" do 22 | owner "root" 23 | group "root" 24 | mode 0755 25 | end 26 | 27 | case node[:platform] 28 | when "debian", "ubuntu" 29 | include_recipe "runit" 30 | runit_service "git-daemon" 31 | else 32 | log "Platform requires setting up a git daemon service script." 33 | log "Hint: /usr/bin/git daemon --export-all --user=nobody --group=daemon --base-path=/srv/git" 34 | end 35 | -------------------------------------------------------------------------------- /cookbooks/git/recipes/source.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: git 3 | # Recipe:: source 4 | # 5 | # Copyright 2012, Brian Flad, Fletcher Nichol 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | 19 | include_recipe "build-essential" 20 | 21 | pkgs = value_for_platform_family( 22 | ["rhel"] => %w{ expat-devel gettext-devel libcurl-devel openssl-devel zlib-devel } 23 | ) 24 | 25 | pkgs.each do |pkg| 26 | package pkg 27 | end 28 | 29 | remote_file "#{Chef::Config[:file_cache_path]}/git-#{node[:git][:version]}.tar.gz" do 30 | source node[:git][:url] 31 | checksum node[:git][:checksum] 32 | mode "0644" 33 | not_if "test -f #{Chef::Config[:file_cache_path]}/git-#{node[:git][:version]}.tar.gz" 34 | end 35 | 36 | execute "Extracting and Building Git #{node[:git][:version]} from Source" do 37 | cwd Chef::Config[:file_cache_path] 38 | command <<-COMMAND 39 | (mkdir git-#{node[:git][:version]} && tar -zxf git-#{node[:git][:version]}.tar.gz -C git-#{node[:git][:version]} --strip-components 1) 40 | (cd git-#{node[:git][:version]} && make prefix=#{node[:git][:prefix]} install) 41 | COMMAND 42 | creates "node[:git][:prefix]}/bin/git" 43 | not_if "git --version | grep #{node[:git][:version]}" 44 | end 45 | -------------------------------------------------------------------------------- /cookbooks/git/templates/default/sv-git-daemon-log-run.erb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec svlogd -tt ./main 3 | -------------------------------------------------------------------------------- /cookbooks/git/templates/default/sv-git-daemon-run.erb: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | exec 2>&1 3 | exec /usr/bin/git daemon --export-all --user=nobody --group=daemon --base-path=/srv/git /srv/git 4 | -------------------------------------------------------------------------------- /cookbooks/golang/README.md: -------------------------------------------------------------------------------- 1 | # Go Language Chef Cookbook 2 | 3 | This is an OpsCode Chef cookbook for [Go, the programming language](http://golang.org). 4 | 5 | It uses the ["Todd Vierling" Ubuntu PPA](https://launchpad.net/~duh/+archive/golang) 6 | and allows you to tweak version using Chef node attributes. 7 | 8 | It is released under the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 9 | 10 | 11 | ## Recipes 12 | 13 | Main recipe is `golang::default`. 14 | 15 | ## Supported OSes 16 | 17 | Ubuntu 10.10 to 12.04, will likely work just as well on Debian unstable. 18 | 19 | 20 | ## Dependencies 21 | 22 | None. 23 | 24 | 25 | ## Copyright & License 26 | 27 | Matthew Baird, 2013. 28 | 29 | Released under the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 30 | -------------------------------------------------------------------------------- /cookbooks/golang/attributes/default.rb: -------------------------------------------------------------------------------- 1 | default[:golang] = { 2 | # can be "stable" or "tip" 3 | :version => "stable", 4 | :multi => { 5 | :versions => %w(go1.0.3 go1.1.1), 6 | :default_version => "go1.1.1", 7 | :aliases => { 8 | "go1" => "go1.1.1" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /cookbooks/golang/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Matthew Baird" 2 | maintainer_email "mattbaird@gmail.com" 3 | license "Apache 2.0" 4 | description "Installs go language from duh's Ubuntu PPA" 5 | long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) 6 | version "1.0.0" 7 | recipe "golang", "Installs go" 8 | 9 | depends "apt" 10 | supports "ubuntu" 11 | -------------------------------------------------------------------------------- /cookbooks/golang/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: golang 3 | # Recipe:: default 4 | # 5 | # Copyright 2012, Michael S. Klishin, Travis CI Development Team 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | include_recipe "golang::ppa" 21 | # echo 'export GOBIN=#{node['golang']['gobin']}' >> /home/vagrant/.bash_golang 22 | # echo 'export GOROOT=/usr/local/go/' >> /home/vagrant/.bash_golang 23 | 24 | bash "Export ENV Vars" do 25 | code <<-EOC 26 | mkdir -p /home/vagrant/code/go/ 27 | chown vagrant /home/vagrant/code/go/ 28 | echo 'export GOPATH=/home/vagrant/code/go/' >> /home/vagrant/.bash_golang 29 | echo 'export GOROOT=/usr/lib/go/' >> /home/vagrant/.bash_golang 30 | echo 'export PATH=$PATH:$GOBIN' >> /home/vagrant/.bash_golang 31 | echo 'source /home/vagrant/.bash_golang' >> /home/vagrant/.bashrc 32 | source /home/vagrant/.bashrc 33 | EOC 34 | creates "/home/vagrant/.bash_golang" 35 | end 36 | -------------------------------------------------------------------------------- /cookbooks/golang/recipes/ppa.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: golang 3 | # Recipe:: ppa 4 | # 5 | # Copyright 2012, Michael S. Klishin, Travis CI Development Team 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | apt_repository "duh-ppa" do 21 | uri "http://ppa.launchpad.net/duh/golang/ubuntu" 22 | distribution node['lsb']['codename'] 23 | components ["main"] 24 | key "60480472" 25 | keyserver "keyserver.ubuntu.com" 26 | action :add 27 | notifies :run, "execute[apt-get update]", :immediately 28 | end 29 | 30 | package "golang" do 31 | # version "1.1.1" 32 | action :install 33 | end 34 | -------------------------------------------------------------------------------- /cookbooks/mercurial/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## v1.0.0: 2 | 3 | * [COOK-1373] - README example correction 4 | * [COOK-1179] - LWRP for repo management 5 | 6 | For further discussion about possible changes to the LWRP, see 7 | COOK-879, whereby it may become a fully fledged provider for chef's 8 | built in scm_repo resource. 9 | 10 | ## v0.7.1: 11 | 12 | * Current public release 13 | -------------------------------------------------------------------------------- /cookbooks/mercurial/CONTRIBUTING: -------------------------------------------------------------------------------- 1 | If you would like to contribute, please open a ticket in JIRA: 2 | 3 | * http://tickets.opscode.com 4 | 5 | Create the ticket in the COOK project and use the cookbook name as the 6 | component. 7 | 8 | For all code contributions, we ask that contributors sign a 9 | contributor license agreement (CLA). Instructions may be found here: 10 | 11 | * http://wiki.opscode.com/display/chef/How+to+Contribute 12 | 13 | When contributing changes to individual cookbooks, please do not 14 | modify the version number in the metadata.rb. Also please do not 15 | update the CHANGELOG.md for a new version. Not all changes to a 16 | cookbook may be merged and released in the same versions. Opscode will 17 | handle the version updates during the release process. You are welcome 18 | to correct typos or otherwise make updates to documentation in the 19 | README. 20 | 21 | If a contribution adds new platforms or platform versions, indicate 22 | such in the body of the commit message(s), and update the relevant 23 | COOK ticket. When writing commit messages, it is helpful for others if 24 | you indicate the COOK ticket. For example: 25 | 26 | git commit -m '[COOK-1041] Updated pool resource to correctly delete.' 27 | 28 | In the ticket itself, it is also helpful if you include log output of 29 | a successful Chef run, but this is not absolutely required. 30 | -------------------------------------------------------------------------------- /cookbooks/mercurial/README.md: -------------------------------------------------------------------------------- 1 | Description 2 | =========== 3 | 4 | Installs mercurial 5 | 6 | Requirements 7 | ============ 8 | 9 | A package named "mercurial" must exist in the platform package 10 | management system. 11 | 12 | Usage 13 | ===== 14 | 15 | Install mercurial to make sure it is available to check out code from 16 | mercurial repositories. 17 | 18 | Resource/Provider 19 | ================= 20 | 21 | This cookbook includes LWRPs for managing: mercurial 22 | 23 | mercurial 24 | --------- 25 | 26 | ### Actions 27 | 28 | - :clone - this will simply issue a clone of the repository at the revision specified (default tip). 29 | - :sync - this will issue a clone of the repository if there is nothing at the path specified, otherwise a pull and update will be issued to bring the directory up-to-date. 30 | 31 | ### Parameter Attributes 32 | 33 | - `path` - **Name attribute** path where the repository is checked 34 | out. 35 | - `repository` - Repository to check out 36 | - `reference` - Reference in the repository 37 | - `key` - a private key on disk to use, for private repositories, must 38 | already exist. 39 | - `owner` - local user that the clone is run as 40 | - `group` - local group that the clone is run as 41 | - `mode` - permissions of the cloned repository 42 | 43 | ### Example 44 | 45 | mercurial "/home/site/checkouts/www" do 46 | repository "ssh://hg@bitbucket.org/niallsco/chef-hg" 47 | reference "tip" 48 | key "/home/site/.ssh/keyname" 49 | action :sync 50 | end 51 | 52 | License and Author 53 | ================== 54 | 55 | Author:: Joshua Timberman 56 | 57 | Copyright:: 2009, Opscode, Inc 58 | 59 | Licensed under the Apache License, Version 2.0 (the "License"); 60 | you may not use this file except in compliance with the License. 61 | You may obtain a copy of the License at 62 | 63 | http://www.apache.org/licenses/LICENSE-2.0 64 | 65 | Unless required by applicable law or agreed to in writing, software 66 | distributed under the License is distributed on an "AS IS" BASIS, 67 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 68 | See the License for the specific language governing permissions and 69 | limitations under the License. 70 | -------------------------------------------------------------------------------- /cookbooks/mercurial/metadata.rb: -------------------------------------------------------------------------------- 1 | maintainer "Opscode, Inc." 2 | maintainer_email "cookbooks@opscode.com" 3 | license "Apache 2.0" 4 | description "Installs mercurial" 5 | version "0.8.0" 6 | 7 | recipe "mercurial", "Installs mercurial" 8 | 9 | %w{ debian ubuntu }.each do |os| 10 | supports os 11 | end 12 | -------------------------------------------------------------------------------- /cookbooks/mercurial/providers/default.rb: -------------------------------------------------------------------------------- 1 | action :sync do 2 | execute "sync repository #{new_resource.path}" do 3 | not_if "hg identify #{new_resource.path}" 4 | command "hg clone -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository} #{new_resource.path}" 5 | end 6 | execute "pull changes #{new_resource.path}" do 7 | command "cd #{new_resource.path} && hg pull -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository}" 8 | end 9 | execute "update #{new_resource.path}" do 10 | command "cd #{new_resource.path} && hg update -r #{new_resource.reference}" 11 | end 12 | execute "sync update owner #{new_resource.path}" do 13 | command "chown -R #{new_resource.owner}:#{new_resource.group} #{new_resource.path}" 14 | end 15 | execute "sync update permissions #{new_resource.path}" do 16 | command "chmod -R #{new_resource.mode} #{new_resource.path}" 17 | end 18 | end 19 | 20 | action :clone do 21 | execute "clone repository #{new_resource.path}" do 22 | not_if "hg identify #{new_resource.path}" 23 | command "hg clone -e 'ssh -i #{new_resource.key} -o StrictHostKeyChecking=no' #{new_resource.repository} #{new_resource.path}" 24 | end 25 | if new_resource.reference 26 | command "cd #{new_resource.path} && hg update -r #{new_resource.reference}" 27 | end 28 | execute "update owner #{new_resource.path}" do 29 | command "chown -R #{new_resource.owner}:#{new_resource.group} #{new_resource.path}" 30 | end 31 | execute "update permissions #{new_resource.path}" do 32 | command "chmod -R #{new_resource.mode} #{new_resource.path}" 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /cookbooks/mercurial/recipes/default.rb: -------------------------------------------------------------------------------- 1 | # 2 | # Cookbook Name:: mercurial 3 | # Recipe:: default 4 | # 5 | # Copyright 2009, Opscode, Inc. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | package "mercurial" do 20 | action :upgrade 21 | end 22 | -------------------------------------------------------------------------------- /cookbooks/mercurial/resources/default.rb: -------------------------------------------------------------------------------- 1 | actions :sync, :clone 2 | 3 | attribute :path, :kind_of => String, :name_attribute => true 4 | attribute :repository, :kind_of => String 5 | attribute :reference, :kind_of => [Integer, String] 6 | attribute :key, :kind_of => String 7 | attribute :owner, :kind_of => String 8 | attribute :group, :kind_of => String 9 | attribute :mode, :kind_of => String, :default => '0775' 10 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2012 Matthew Baird 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | -------------------------------------------------------------------------------- /lib/baserequest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "bytes" 16 | "encoding/json" 17 | "fmt" 18 | "io" 19 | "io/ioutil" 20 | "log" 21 | "time" 22 | ) 23 | 24 | func (c *Conn) DoCommand(method string, url string, args map[string]interface{}, data interface{}) ([]byte, error) { 25 | var response map[string]interface{} 26 | var body []byte 27 | var httpStatusCode int 28 | 29 | query, err := Escape(args) 30 | if err != nil { 31 | return nil, err 32 | } 33 | req, err := c.NewRequest(method, url, query) 34 | if err != nil { 35 | return body, err 36 | } 37 | 38 | if data != nil { 39 | if c.Gzip { 40 | req.SetBodyGzip(data) 41 | } else { 42 | switch v := data.(type) { 43 | case string: 44 | req.SetBodyString(v) 45 | case io.Reader: 46 | req.SetBody(v) 47 | case []byte: 48 | req.SetBodyBytes(v) 49 | default: 50 | err = req.SetBodyJson(v) 51 | if err != nil { 52 | return body, err 53 | } 54 | } 55 | } 56 | } 57 | 58 | // uncomment this to print out the request that hits the wire 59 | // (requires net/http/httputil) 60 | //reqbuf, err := httputil.DumpRequest(req.Request, true) 61 | //log.Println(fmt.Sprintf("\n========= req:\nURL: %s\n%s", req.URL, bytes.NewBuffer(reqbuf).String())) 62 | 63 | // Copy request body for tracer 64 | if c.RequestTracer != nil { 65 | rbody := "" 66 | if req.Body != nil { 67 | requestBody, err := ioutil.ReadAll(req.Body) 68 | if err != nil { 69 | return body, err 70 | } 71 | 72 | req.SetBody(bytes.NewReader(requestBody)) 73 | rbody = string(requestBody) 74 | } 75 | c.RequestTracer(req.Method, req.URL.String(), rbody) 76 | } 77 | 78 | httpStatusCode, body, err = req.Do(&response) 79 | if err != nil { 80 | return body, err 81 | } 82 | if httpStatusCode > 304 { 83 | 84 | jsonErr := json.Unmarshal(body, &response) 85 | if jsonErr == nil { 86 | if res_err, ok := response["error"]; ok { 87 | status, _ := response["status"] 88 | return body, ESError{time.Now(), fmt.Sprintf("Error [%s] Status [%v]", res_err, status), httpStatusCode} 89 | } 90 | } 91 | return body, jsonErr 92 | } 93 | return body, nil 94 | } 95 | 96 | // ESError is an error implementation that includes a time, message, and code. 97 | type ESError struct { 98 | When time.Time 99 | What string 100 | Code int 101 | } 102 | 103 | func (e ESError) Error() string { 104 | return fmt.Sprintf("%v: %v [%v]", e.When, e.What, e.Code) 105 | } 106 | 107 | // Exists allows the caller to check for the existence of a document using HEAD 108 | // This appears to be broken in the current version of elasticsearch 0.19.10, currently 109 | // returning nothing 110 | func (c *Conn) Exists(index string, _type string, id string, args map[string]interface{}) (BaseResponse, error) { 111 | var response map[string]interface{} 112 | var body []byte 113 | var url string 114 | var retval BaseResponse 115 | var httpStatusCode int 116 | 117 | query, err := Escape(args) 118 | if err != nil { 119 | return retval, err 120 | } 121 | 122 | if len(_type) > 0 { 123 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 124 | } else { 125 | url = fmt.Sprintf("/%s/%s", index, id) 126 | } 127 | req, err := c.NewRequest("HEAD", url, query) 128 | if err != nil { 129 | // some sort of generic error handler 130 | } 131 | httpStatusCode, body, err = req.Do(&response) 132 | if httpStatusCode > 304 { 133 | if error, ok := response["error"]; ok { 134 | status, _ := response["status"] 135 | log.Printf("Error: %v (%v)\n", error, status) 136 | } 137 | } else { 138 | // marshall into json 139 | jsonErr := json.Unmarshal(body, &retval) 140 | if jsonErr != nil { 141 | log.Println(jsonErr) 142 | } 143 | } 144 | return retval, err 145 | } 146 | -------------------------------------------------------------------------------- /lib/baseresponse.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strconv" 18 | "strings" 19 | ) 20 | 21 | type BaseResponse struct { 22 | Ok bool `json:"ok"` 23 | Index string `json:"_index,omitempty"` 24 | Type string `json:"_type,omitempty"` 25 | Id string `json:"_id,omitempty"` 26 | Source *json.RawMessage `json:"_source,omitempty"` // depends on the schema you've defined 27 | Version int `json:"_version,omitempty"` 28 | Found bool `json:"found,omitempty"` 29 | Exists bool `json:"exists,omitempty"` 30 | Created bool `json:"created,omitempty"` 31 | Matches []string `json:"matches,omitempty"` // percolate matches 32 | } 33 | 34 | // StatusInt is required because /_optimize, at least, returns its status as 35 | // strings instead of integers. 36 | type StatusInt int 37 | 38 | func (self *StatusInt) UnmarshalJSON(b []byte) error { 39 | s := "" 40 | if json.Unmarshal(b, &s) == nil { 41 | if i, err := strconv.Atoi(s); err == nil { 42 | *self = StatusInt(i) 43 | return nil 44 | } 45 | } 46 | i := 0 47 | err := json.Unmarshal(b, &i) 48 | if err == nil { 49 | *self = StatusInt(i) 50 | } 51 | return err 52 | } 53 | 54 | func (self *StatusInt) MarshalJSON() ([]byte, error) { 55 | return json.Marshal(*self) 56 | } 57 | 58 | // StatusBool is required because /_optimize, at least, returns its status as 59 | // strings instead of booleans. 60 | type StatusBool bool 61 | 62 | func (self *StatusBool) UnmarshalJSON(b []byte) error { 63 | s := "" 64 | if json.Unmarshal(b, &s) == nil { 65 | switch s { 66 | case "true": 67 | *self = StatusBool(true) 68 | return nil 69 | case "false": 70 | *self = StatusBool(false) 71 | return nil 72 | default: 73 | } 74 | } 75 | b2 := false 76 | err := json.Unmarshal(b, &b2) 77 | if err == nil { 78 | *self = StatusBool(b2) 79 | } 80 | return err 81 | } 82 | 83 | func (self *StatusBool) MarshalJSON() ([]byte, error) { 84 | return json.Marshal(*self) 85 | } 86 | 87 | type Status struct { 88 | Total StatusInt `json:"total"` 89 | Successful StatusInt `json:"successful"` 90 | Failed StatusInt `json:"failed"` 91 | Failures []Failure `json:"failures,omitempty"` 92 | } 93 | 94 | type Failure struct { 95 | Index string `json:"index"` 96 | Shard StatusInt `json:"shard"` 97 | Reason string `json:"reason"` 98 | } 99 | 100 | func (f Failure) String() string { 101 | return fmt.Sprintf("Failed on shard %d on index %s:\n%s", f.Shard, f.Index, f.Reason) 102 | } 103 | 104 | // failures is a convenience type to allow []Failure formated easily in the 105 | // library 106 | type failures []Failure 107 | 108 | func (f failures) String() string { 109 | message := make([]string, len(f)) 110 | for i, failure := range f { 111 | message[i] = failure.String() 112 | } 113 | return strings.Join(message, "\n") 114 | } 115 | 116 | type ExtendedStatus struct { 117 | Ok StatusBool `json:"ok"` 118 | ShardsStatus Status `json:"_shards"` 119 | } 120 | 121 | type MatchRes struct { 122 | Index string `json:"_index"` 123 | Id string `json:"_id"` 124 | } 125 | 126 | type Match struct { 127 | OK bool `json:"ok"` 128 | Matches []MatchRes `json:"matches"` 129 | Explanation *Explanation `json:"explanation,omitempty"` 130 | } 131 | 132 | type Explanation struct { 133 | Value float32 `json:"value"` 134 | Description string `json:"description"` 135 | Details []*Explanation `json:"details,omitempty"` 136 | } 137 | 138 | func ScrollDuration(duration string) string { 139 | scrollString := "" 140 | if duration != "" { 141 | scrollString = "&scroll=" + duration 142 | } 143 | return scrollString 144 | } 145 | 146 | // http://www.elasticsearch.org/guide/reference/api/search/search-type/ 147 | -------------------------------------------------------------------------------- /lib/cataliasinfo.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | var ErrInvalidAliasLine = errors.New("Cannot parse aliasline") 9 | 10 | //Create an AliasInfo from the string _cat/alias would produce 11 | //EX: alias alias 12 | //i production production_20160405075824 13 | func NewCatAliasInfo(aliasLine string) (catAlias *CatAliasInfo, err error) { 14 | split := strings.Fields(aliasLine) 15 | if len(split) < 2 { 16 | return nil, ErrInvalidAliasLine 17 | } 18 | catAlias = &CatAliasInfo{} 19 | catAlias.Name = split[0] 20 | catAlias.Index = split[1] 21 | return catAlias, nil 22 | } 23 | 24 | // Pull all the alias info from the connection 25 | func (c *Conn) GetCatAliasInfo(pattern string) (catAliases []CatAliasInfo) { 26 | catAliases = make([]CatAliasInfo, 0) 27 | //force it to only show the fields we know about 28 | aliases, err := c.DoCommand("GET", "/_cat/aliases/"+pattern, nil, nil) 29 | if err == nil { 30 | aliasLines := strings.Split(string(aliases[:]), "\n") 31 | for _, alias := range aliasLines { 32 | ci, _ := NewCatAliasInfo(alias) 33 | if nil != ci { 34 | catAliases = append(catAliases, *ci) 35 | } 36 | } 37 | } 38 | return catAliases 39 | } 40 | -------------------------------------------------------------------------------- /lib/cataliasinfo_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestCatAliasInfo(t *testing.T) { 10 | Convey("catAlias Create alias line from a broken alias listing", t, func() { 11 | _, err := NewCatAliasInfo("production ") 12 | So(err, ShouldNotBeNil) 13 | }) 14 | Convey("catAlias Create alias line from a complete alias listing", t, func() { 15 | i, err := NewCatAliasInfo("production production-2016") 16 | So(err, ShouldBeNil) 17 | So(i.Name, ShouldEqual, "production") 18 | So(i.Index, ShouldEqual, "production-2016") 19 | }) 20 | Convey("catAlias Create alias line from an over-complete alias listing", t, func() { 21 | i, err := NewCatAliasInfo("production production-2016 - - -") 22 | So(err, ShouldBeNil) 23 | So(i.Name, ShouldEqual, "production") 24 | So(i.Index, ShouldEqual, "production-2016") 25 | }) 26 | } 27 | -------------------------------------------------------------------------------- /lib/catindexinfo.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | var ErrInvalidIndexLine = errors.New("Cannot parse indexline") 10 | 11 | //Create an IndexInfo from the string _cat/indices would produce 12 | //EX: health status index pri rep docs.count docs.deleted store.size pri.store.size 13 | //green open logs-2015-06-19 2 0 135389346 0 53048922233 53048922233 14 | func NewCatIndexInfo(indexLine string) (catIndex *CatIndexInfo, err error) { 15 | split := strings.Fields(indexLine) 16 | if len(split) < 5 { 17 | return nil, ErrInvalidIndexLine 18 | } 19 | catIndex = &CatIndexInfo{} 20 | catIndex.Store = CatIndexStore{} 21 | catIndex.Docs = CatIndexDocs{} 22 | catIndex.Health = split[0] 23 | catIndex.Status = split[1] 24 | catIndex.Name = split[2] 25 | catIndex.Shards, err = strconv.Atoi(split[3]) 26 | if err != nil { 27 | catIndex.Shards = 0 28 | } 29 | catIndex.Replicas, err = strconv.Atoi(split[4]) 30 | if err != nil { 31 | catIndex.Replicas = 0 32 | } 33 | if len(split) == 5 { 34 | return catIndex, nil 35 | } 36 | catIndex.Docs.Count, err = strconv.ParseInt(split[5], 10, 64) 37 | if err != nil { 38 | catIndex.Docs.Count = 0 39 | } 40 | if len(split) == 6 { 41 | return catIndex, nil 42 | } 43 | catIndex.Docs.Deleted, err = strconv.ParseInt(split[6], 10, 64) 44 | if err != nil { 45 | catIndex.Docs.Deleted = 0 46 | } 47 | if len(split) == 7 { 48 | return catIndex, nil 49 | } 50 | catIndex.Store.Size, err = strconv.ParseInt(split[7], 10, 64) 51 | if err != nil { 52 | catIndex.Store.Size = 0 53 | } 54 | if len(split) == 8 { 55 | return catIndex, nil 56 | } 57 | catIndex.Store.PriSize, err = strconv.ParseInt(split[8], 10, 64) 58 | if err != nil { 59 | catIndex.Store.PriSize = 0 60 | } 61 | return catIndex, nil 62 | } 63 | 64 | // Pull all the index info from the connection 65 | func (c *Conn) GetCatIndexInfo(pattern string) (catIndices []CatIndexInfo) { 66 | catIndices = make([]CatIndexInfo, 0) 67 | //force it to only show the fileds we know about 68 | args := map[string]interface{}{"bytes": "b", "h": "health,status,index,pri,rep,docs.count,docs.deleted,store.size,pri.store.size"} 69 | indices, err := c.DoCommand("GET", "/_cat/indices/"+pattern, args, nil) 70 | if err == nil { 71 | indexLines := strings.Split(string(indices[:]), "\n") 72 | for _, index := range indexLines { 73 | ci, _ := NewCatIndexInfo(index) 74 | if nil != ci { 75 | catIndices = append(catIndices, *ci) 76 | } 77 | } 78 | } 79 | return catIndices 80 | } 81 | -------------------------------------------------------------------------------- /lib/catindexinfo_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | . "github.com/smartystreets/goconvey/convey" 5 | "testing" 6 | ) 7 | 8 | func TestCatIndexInfo(t *testing.T) { 9 | Convey("Create index line from a broken index listing", t, func() { 10 | _, err := NewCatIndexInfo("red ") 11 | So(err, ShouldNotBeNil) 12 | }) 13 | Convey("catIndex Create index line from a bad shards index listing", t, func() { 14 | i, err := NewCatIndexInfo("green open logs-2015-06-19 2 1 135389346 20 53048922233 53048922233") 15 | So(err, ShouldBeNil) 16 | So(i.Health, ShouldEqual, "green") 17 | So(i.Status, ShouldEqual, "open") 18 | So(i.Name, ShouldEqual, "logs-2015-06-19") 19 | So(i.Shards, ShouldEqual, 2) 20 | So(i.Replicas, ShouldEqual, 1) 21 | So(i.Docs.Count, ShouldEqual, 135389346) 22 | So(i.Docs.Deleted, ShouldEqual, 20) 23 | So(i.Store.Size, ShouldEqual, 53048922233) 24 | So(i.Store.PriSize, ShouldEqual, 53048922233) 25 | }) 26 | Convey("catIndex Create index line from a bad replicas index listing", t, func() { 27 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 0 1234 3 11000 13000") 28 | So(err, ShouldBeNil) 29 | So(i.Health, ShouldEqual, "red") 30 | So(i.Status, ShouldEqual, "open") 31 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 32 | So(i.Shards, ShouldEqual, 2) 33 | So(i.Replicas, ShouldEqual, 0) 34 | So(i.Docs.Count, ShouldEqual, 1234) 35 | So(i.Docs.Deleted, ShouldEqual, 3) 36 | So(i.Store.Size, ShouldEqual, 11000) 37 | So(i.Store.PriSize, ShouldEqual, 13000) 38 | }) 39 | Convey("catIndex Create index line from a complete index listing", t, func() { 40 | i, err := NewCatIndexInfo("red closed foo-2000-01-01-bar 2 1 1234 3 11000 13000") 41 | So(err, ShouldBeNil) 42 | So(i.Status, ShouldEqual, "closed") 43 | So(i.Health, ShouldEqual, "red") 44 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 45 | So(i.Shards, ShouldEqual, 2) 46 | So(i.Replicas, ShouldEqual, 1) 47 | So(i.Docs.Count, ShouldEqual, 1234) 48 | So(i.Docs.Deleted, ShouldEqual, 3) 49 | So(i.Store.Size, ShouldEqual, 11000) 50 | So(i.Store.PriSize, ShouldEqual, 13000) 51 | }) 52 | Convey("catIndex Create index line from a bad docs index listing", t, func() { 53 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 a 3 11000 13000") 54 | So(err, ShouldBeNil) 55 | So(i.Health, ShouldEqual, "red") 56 | So(i.Status, ShouldEqual, "open") 57 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 58 | So(i.Shards, ShouldEqual, 2) 59 | So(i.Replicas, ShouldEqual, 1) 60 | So(i.Docs.Count, ShouldEqual, 0) 61 | So(i.Docs.Deleted, ShouldEqual, 3) 62 | So(i.Store.Size, ShouldEqual, 11000) 63 | So(i.Store.PriSize, ShouldEqual, 13000) 64 | }) 65 | Convey("catIndex Create index line from a bad deletes index listing", t, func() { 66 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234 a 11000 13000") 67 | So(err, ShouldBeNil) 68 | So(i.Health, ShouldEqual, "red") 69 | So(i.Status, ShouldEqual, "open") 70 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 71 | So(i.Shards, ShouldEqual, 2) 72 | So(i.Replicas, ShouldEqual, 1) 73 | So(i.Docs.Count, ShouldEqual, 1234) 74 | So(i.Docs.Deleted, ShouldEqual, 0) 75 | So(i.Store.Size, ShouldEqual, 11000) 76 | So(i.Store.PriSize, ShouldEqual, 13000) 77 | }) 78 | Convey("catIndex Create index line from a kinda short index listing", t, func() { 79 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234") 80 | So(err, ShouldBeNil) 81 | So(i.Health, ShouldEqual, "red") 82 | So(i.Status, ShouldEqual, "open") 83 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 84 | So(i.Shards, ShouldEqual, 2) 85 | So(i.Replicas, ShouldEqual, 1) 86 | So(i.Docs.Count, ShouldEqual, 1234) 87 | So(i.Docs.Deleted, ShouldEqual, 0) 88 | So(i.Store.Size, ShouldEqual, 0) 89 | So(i.Store.PriSize, ShouldEqual, 0) 90 | }) 91 | Convey("catIndex Create index line from a kinda sorta short index listing", t, func() { 92 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1 1234 3") 93 | So(err, ShouldBeNil) 94 | So(i.Health, ShouldEqual, "red") 95 | So(i.Status, ShouldEqual, "open") 96 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 97 | So(i.Shards, ShouldEqual, 2) 98 | So(i.Replicas, ShouldEqual, 1) 99 | So(i.Docs.Count, ShouldEqual, 1234) 100 | So(i.Docs.Deleted, ShouldEqual, 3) 101 | So(i.Store.Size, ShouldEqual, 0) 102 | So(i.Store.PriSize, ShouldEqual, 0) 103 | }) 104 | Convey("catIndex Create index line from a short index listing", t, func() { 105 | i, err := NewCatIndexInfo("red open foo-2000-01-01-bar 2 1") 106 | So(err, ShouldBeNil) 107 | So(i.Health, ShouldEqual, "red") 108 | So(i.Status, ShouldEqual, "open") 109 | So(i.Name, ShouldEqual, "foo-2000-01-01-bar") 110 | So(i.Shards, ShouldEqual, 2) 111 | So(i.Replicas, ShouldEqual, 1) 112 | So(i.Docs.Count, ShouldEqual, 0) 113 | So(i.Docs.Deleted, ShouldEqual, 0) 114 | So(i.Store.Size, ShouldEqual, 0) 115 | So(i.Store.PriSize, ShouldEqual, 0) 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /lib/catnodeinfo_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | . "github.com/smartystreets/goconvey/convey" 5 | "testing" 6 | ) 7 | 8 | func TestCatNode(t *testing.T) { 9 | 10 | c := NewTestConn() 11 | 12 | Convey("Basic cat nodes", t, func() { 13 | 14 | fields := []string{"fm", "fe", "fcm", "fce", "ft", "ftt", "im", "rp", "n"} 15 | catNodes, err := c.GetCatNodeInfo(fields) 16 | 17 | So(err, ShouldBeNil) 18 | So(catNodes, ShouldNotBeNil) 19 | So(len(catNodes), ShouldBeGreaterThan, 0) 20 | 21 | for _, catNode := range catNodes { 22 | So(catNode.FieldMem, ShouldNotBeEmpty) 23 | So(catNode.FiltMem, ShouldNotBeEmpty) 24 | So(catNode.IDCacheMemory, ShouldNotBeEmpty) 25 | So(catNode.RamPerc, ShouldNotBeEmpty) 26 | So(catNode.Name, ShouldNotBeEmpty) 27 | } 28 | }) 29 | 30 | Convey("Cat nodes with default arguments", t, func() { 31 | 32 | fields := []string{} 33 | catNodes, err := c.GetCatNodeInfo(fields) 34 | 35 | So(err, ShouldBeNil) 36 | So(catNodes, ShouldNotBeNil) 37 | So(len(catNodes), ShouldBeGreaterThan, 0) 38 | 39 | for _, catNode := range catNodes { 40 | So(catNode.Host, ShouldNotBeEmpty) 41 | So(catNode.IP, ShouldNotBeEmpty) 42 | So(catNode.NodeRole, ShouldNotBeEmpty) 43 | So(catNode.Name, ShouldNotBeEmpty) 44 | } 45 | }) 46 | 47 | Convey("Invalid field error behavior", t, func() { 48 | 49 | fields := []string{"fm", "bogus"} 50 | catNodes, err := c.GetCatNodeInfo(fields) 51 | 52 | So(err, ShouldNotBeNil) 53 | 54 | for _, catNode := range catNodes { 55 | So(catNode.FieldMem, ShouldNotBeEmpty) 56 | } 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /lib/catresponses.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | type CatIndexInfo struct { 4 | Health string 5 | Status string 6 | Name string 7 | Shards int 8 | Replicas int 9 | Docs CatIndexDocs 10 | Store CatIndexStore 11 | } 12 | 13 | type CatIndexDocs struct { 14 | Count int64 15 | Deleted int64 16 | } 17 | 18 | type CatIndexStore struct { 19 | Size int64 20 | PriSize int64 21 | } 22 | 23 | type CatAliasInfo struct { 24 | Name string 25 | Index string 26 | } 27 | 28 | type CatShardInfo struct { 29 | IndexName string 30 | Shard int 31 | Primary string 32 | State string 33 | Docs int64 34 | Store int64 35 | NodeIP string 36 | NodeName string 37 | } 38 | 39 | type CatNodeInfo struct { 40 | Id string 41 | PID string 42 | Host string 43 | IP string 44 | Port string 45 | Version string 46 | Build string 47 | JDK string 48 | DiskAvail string 49 | HeapCur string 50 | HeapPerc string 51 | HeapMax string 52 | RamCur string 53 | RamPerc int16 54 | RamMax string 55 | FileDescCur string 56 | FileDescPerc string 57 | FileDescMax string 58 | Load string 59 | UpTime string 60 | NodeRole string 61 | Master string 62 | Name string 63 | CmpltSize string 64 | FieldMem int 65 | FieldEvict int 66 | FiltMem int 67 | FiltEvict int 68 | FlushTotal int 69 | FlushTotalTime string 70 | GetCur string 71 | GetTime string 72 | GetTotal string 73 | GetExistsTime string 74 | GetExistsTotal string 75 | GetMissingTime string 76 | GetMissingTotal string 77 | IDCacheMemory int 78 | IdxDelCur string 79 | IdxDelTime string 80 | IdxDelTotal string 81 | IdxIdxCur string 82 | IdxIdxTime string 83 | IdxIdxTotal string 84 | MergCur string 85 | MergCurDocs string 86 | MergCurSize string 87 | MergTotal string 88 | MergTotalDocs string 89 | MergTotalSize string 90 | MergTotalTime string 91 | PercCur string 92 | PercMem string 93 | PercQueries string 94 | PercTime string 95 | PercTotal string 96 | RefreshTotal string 97 | RefreshTime string 98 | SearchFetchCur string 99 | SearchFetchTime string 100 | SearchFetchTotal string 101 | SearchOpenContexts string 102 | SearchQueryCur string 103 | SearchQueryTime string 104 | SearchQueryTotal string 105 | SegCount string 106 | SegMem string 107 | SegIdxWriterMem string 108 | SegIdxWriterMax string 109 | SegVerMapMem string 110 | } 111 | -------------------------------------------------------------------------------- /lib/catshardinfo.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | type CatShards []CatShardInfo 12 | 13 | // Stringify the shards 14 | func (s *CatShards) String() string { 15 | var buffer bytes.Buffer 16 | 17 | if s != nil { 18 | for _, cs := range *s { 19 | buffer.WriteString(fmt.Sprintf("%v\n", cs)) 20 | } 21 | } 22 | return buffer.String() 23 | } 24 | 25 | var ErrInvalidShardLine = errors.New("Cannot parse shardline") 26 | 27 | // Create a CatShard from a line of the raw output of a _cat/shards 28 | func NewCatShardInfo(rawCat string) (catshard *CatShardInfo, err error) { 29 | 30 | split := strings.Fields(rawCat) 31 | if len(split) < 4 { 32 | return nil, ErrInvalidShardLine 33 | } 34 | catshard = &CatShardInfo{} 35 | catshard.IndexName = split[0] 36 | catshard.Shard, err = strconv.Atoi(split[1]) 37 | if err != nil { 38 | catshard.Shard = -1 39 | } 40 | catshard.Primary = split[2] 41 | catshard.State = split[3] 42 | if len(split) == 4 { 43 | return catshard, nil 44 | } 45 | 46 | catshard.Docs, err = strconv.ParseInt(split[4], 10, 64) 47 | if err != nil { 48 | catshard.Docs = 0 49 | } 50 | if len(split) == 5 { 51 | return catshard, nil 52 | } 53 | catshard.Store, err = strconv.ParseInt(split[5], 10, 64) 54 | if err != nil { 55 | catshard.Store = 0 56 | } 57 | if len(split) == 6 { 58 | return catshard, nil 59 | } 60 | catshard.NodeIP = split[6] 61 | if len(split) == 7 { 62 | return catshard, nil 63 | } 64 | catshard.NodeName = split[7] 65 | if len(split) > 8 { 66 | loop: 67 | for i, moreName := range split { 68 | if i > 7 { 69 | if moreName == "->" { 70 | break loop 71 | } 72 | catshard.NodeName += " " 73 | catshard.NodeName += moreName 74 | } 75 | } 76 | } 77 | 78 | return catshard, nil 79 | } 80 | 81 | // Print shard info 82 | func (s *CatShardInfo) String() string { 83 | if s == nil { 84 | return ":::::::" 85 | } 86 | return fmt.Sprintf("%v:%v:%v:%v:%v:%v:%v:%v", s.IndexName, s.Shard, s.Primary, 87 | s.State, s.Docs, s.Store, s.NodeIP, s.NodeName) 88 | } 89 | 90 | // Get all the shards, even the bad ones 91 | func (c *Conn) GetCatShards() (shards CatShards) { 92 | shards = make(CatShards, 0) 93 | //force it to only respond with the columns we know about and in a forced order 94 | args := map[string]interface{}{"bytes": "b", "h": "index,shard,prirep,state,docs,store,ip,node"} 95 | s, err := c.DoCommand("GET", "/_cat/shards", args, nil) 96 | if err == nil { 97 | catShardLines := strings.Split(string(s[:]), "\n") 98 | for _, shardLine := range catShardLines { 99 | shard, _ := NewCatShardInfo(shardLine) 100 | if nil != shard { 101 | shards = append(shards, *shard) 102 | } 103 | } 104 | } 105 | return shards 106 | } 107 | -------------------------------------------------------------------------------- /lib/catshardinfo_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | . "github.com/smartystreets/goconvey/convey" 5 | "testing" 6 | ) 7 | 8 | func TestCatShardInfo(t *testing.T) { 9 | Convey("Create cat shard from started shard", t, func() { 10 | c, err := NewCatShardInfo("foo-2000-01-01-bar 0 p STARTED 1234 121 127.0.0.1 Ultra Man") 11 | So(err, ShouldBeNil) 12 | So(c, ShouldNotBeNil) 13 | So(c.IndexName, ShouldEqual, "foo-2000-01-01-bar") 14 | So(c.Primary, ShouldEqual, "p") 15 | So(c.State, ShouldEqual, "STARTED") 16 | So(c.Docs, ShouldEqual, 1234) 17 | So(c.Store, ShouldEqual, 121) 18 | So(c.NodeIP, ShouldEqual, "127.0.0.1") 19 | So(c.NodeName, ShouldEqual, "Ultra Man") 20 | 21 | }) 22 | Convey("Create cat shard from realocating shard", t, func() { 23 | c, err := NewCatShardInfo("foo-2000-01-01-bar 0 p RELOCATING 1234 121 127.0.0.1 Ultra Man -> 10.0.0.1 Super Man") 24 | So(err, ShouldBeNil) 25 | So(c, ShouldNotBeNil) 26 | So(c.IndexName, ShouldEqual, "foo-2000-01-01-bar") 27 | So(c.Primary, ShouldEqual, "p") 28 | So(c.State, ShouldEqual, "RELOCATING") 29 | So(c.Docs, ShouldEqual, 1234) 30 | So(c.Store, ShouldEqual, 121) 31 | So(c.NodeIP, ShouldEqual, "127.0.0.1") 32 | So(c.NodeName, ShouldEqual, "Ultra Man") 33 | }) 34 | Convey("Create cat shard from unallocated shard", t, func() { 35 | c, err := NewCatShardInfo("foo-2000-01-01-bar 0 p UNASSIGNED") 36 | So(err, ShouldBeNil) 37 | So(c, ShouldNotBeNil) 38 | So(c.IndexName, ShouldEqual, "foo-2000-01-01-bar") 39 | So(c.Primary, ShouldEqual, "p") 40 | So(c.State, ShouldEqual, "UNASSIGNED") 41 | So(c.Docs, ShouldEqual, 0) 42 | So(c.Store, ShouldEqual, 0) 43 | So(c.NodeIP, ShouldEqual, "") 44 | So(c.NodeName, ShouldEqual, "") 45 | }) 46 | Convey("Create cat shard from invalid shard", t, func() { 47 | c, err := NewCatShardInfo("foo-2000-01-01-bar 0 p") 48 | So(err, ShouldEqual, ErrInvalidShardLine) 49 | So(c, ShouldBeNil) 50 | }) 51 | Convey("Create cat shard from garbled shard", t, func() { 52 | c, err := NewCatShardInfo("foo-2000-01-01-bar a p STARTED abc 121 127.0.0.1 Ultra Man") 53 | So(err, ShouldBeNil) 54 | So(c, ShouldNotBeNil) 55 | So(c.Shard, ShouldEqual, -1) 56 | So(c.IndexName, ShouldEqual, "foo-2000-01-01-bar") 57 | So(c.Primary, ShouldEqual, "p") 58 | So(c.State, ShouldEqual, "STARTED") 59 | So(c.Docs, ShouldEqual, 0) 60 | So(c.Store, ShouldEqual, 121) 61 | So(c.NodeIP, ShouldEqual, "127.0.0.1") 62 | So(c.NodeName, ShouldEqual, "Ultra Man") 63 | }) 64 | Convey("Print cat shard from started shard", t, func() { 65 | c, _ := NewCatShardInfo("foo-2000-01-01-bar 0 p STARTED 1234 121 127.0.0.1 Ultra Man") 66 | s := c.String() 67 | So(s, ShouldContainSubstring, "foo-2000-01-01-bar:") 68 | So(s, ShouldContainSubstring, ":Ultra Man") 69 | c = nil 70 | s = c.String() 71 | So(s, ShouldEqual, ":::::::") 72 | }) 73 | Convey("Print cat shard from short shard", t, func() { 74 | c, _ := NewCatShardInfo("foo-2000-01-01-bar 0 p STARTED 1234") 75 | s := c.String() 76 | So(s, ShouldContainSubstring, "foo-2000-01-01-bar:0:p:STARTED:1234") 77 | c, _ = NewCatShardInfo("foo-2000-01-01-bar 0 p STARTED 1234 121") 78 | s = c.String() 79 | So(s, ShouldContainSubstring, "oo-2000-01-01-bar:0:p:STARTED:1234:121") 80 | c, _ = NewCatShardInfo("foo-2000-01-01-bar 0 p STARTED 1234 121 127.0.0.1") 81 | s = c.String() 82 | So(s, ShouldContainSubstring, "oo-2000-01-01-bar:0:p:STARTED:1234:121:127.0.0.1") 83 | }) 84 | 85 | } 86 | -------------------------------------------------------------------------------- /lib/clusterhealth.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | package elastigo 12 | 13 | import ( 14 | "encoding/json" 15 | "fmt" 16 | "strings" 17 | ) 18 | 19 | // The cluster health API allows to get a very simple status on the health of the cluster. 20 | // see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/cluster-health.html 21 | // TODO: implement wait_for_status, timeout, wait_for_relocating_shards, wait_for_nodes 22 | // TODO: implement level (Can be one of cluster, indices or shards. Controls the details level of the health 23 | // information returned. Defaults to cluster.) 24 | func (c *Conn) Health(indices ...string) (ClusterHealthResponse, error) { 25 | var url string 26 | var retval ClusterHealthResponse 27 | if len(indices) > 0 { 28 | url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ",")) 29 | } else { 30 | url = "/_cluster/health" 31 | } 32 | body, err := c.DoCommand("GET", url, nil, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | 46 | func (c *Conn) WaitForStatus(status string, timeout int, indices ...string) (ClusterHealthResponse, error) { 47 | var url string 48 | var retval ClusterHealthResponse 49 | if len(indices) > 0 { 50 | url = fmt.Sprintf("/_cluster/health/%s", strings.Join(indices, ",")) 51 | } else { 52 | url = "/_cluster/health" 53 | } 54 | 55 | body, err := c.DoCommand("GET", url, map[string]interface{}{ 56 | "wait_for_status": status, 57 | "timeout": timeout, 58 | }, nil) 59 | 60 | if err != nil { 61 | return retval, err 62 | } 63 | 64 | if err == nil { 65 | jsonErr := json.Unmarshal(body, &retval) 66 | if jsonErr != nil { 67 | return retval, jsonErr 68 | } 69 | } 70 | return retval, err 71 | } 72 | 73 | type ClusterStateFilter struct { 74 | FilterNodes bool 75 | FilterRoutingTable bool 76 | FilterMetadata bool 77 | FilterBlocks bool 78 | FilterIndices []string 79 | } 80 | 81 | func (f ClusterStateFilter) Parameterize() []string { 82 | var parts []string 83 | 84 | if f.FilterNodes { 85 | parts = append(parts, "filter_nodes=true") 86 | } 87 | 88 | if f.FilterRoutingTable { 89 | parts = append(parts, "filter_routing_table=true") 90 | } 91 | 92 | if f.FilterMetadata { 93 | parts = append(parts, "filter_metadata=true") 94 | } 95 | 96 | if f.FilterBlocks { 97 | parts = append(parts, "filter_blocks=true") 98 | } 99 | 100 | if f.FilterIndices != nil && len(f.FilterIndices) > 0 { 101 | parts = append(parts, strings.Join([]string{"filter_indices=", strings.Join(f.FilterIndices, ",")}, "")) 102 | } 103 | 104 | return parts 105 | } 106 | 107 | func (c *Conn) ClusterState(filter ClusterStateFilter) (ClusterStateResponse, error) { 108 | var parameters []string 109 | var url string 110 | var retval ClusterStateResponse 111 | 112 | parameters = filter.Parameterize() 113 | 114 | url = fmt.Sprintf("/_cluster/state?%s", strings.Join(parameters, "&")) 115 | 116 | body, err := c.DoCommand("GET", url, nil, nil) 117 | if err != nil { 118 | return retval, err 119 | } 120 | if err == nil { 121 | // marshall into json 122 | jsonErr := json.Unmarshal(body, &retval) 123 | if jsonErr != nil { 124 | return retval, jsonErr 125 | } 126 | } 127 | return retval, err 128 | } 129 | -------------------------------------------------------------------------------- /lib/clusterhealthresponses.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | type ClusterHealthResponse struct { 4 | ClusterName string `json:"cluster_name"` 5 | Status string `json:"status"` 6 | TimedOut bool `json:"timed_out"` 7 | NumberOfNodes int `json:"number_of_nodes"` 8 | NumberOfDataNodes int `json:"number_of_data_nodes"` 9 | ActivePrimaryShards int `json:"active_primary_shards"` 10 | ActiveShards int `json:"active_shards"` 11 | RelocatingShards int `json:"relocating_shards"` 12 | InitializingShards int `json:"initializing_shards"` 13 | UnassignedShards int `json:"unassigned_shards"` 14 | } 15 | 16 | type ClusterStateResponse struct { 17 | ClusterName string `json:"cluster_name"` 18 | MasterNode string `json:"master_node"` 19 | Nodes map[string]ClusterStateNodeResponse `json:"nodes"` 20 | Metadata ClusterStateMetadataResponse `json:"metadata"` 21 | // TODO: Routing Table 22 | // TODO: Routing Nodes 23 | // TODO: Allocations 24 | 25 | } 26 | 27 | type ClusterStateNodeResponse struct { 28 | Name string `json:"name"` 29 | TransportAddress string `json:"transport_address"` 30 | // TODO: Attributes 31 | } 32 | 33 | type ClusterStateMetadataResponse struct { 34 | // TODO: templates 35 | Indices map[string]ClusterStateIndiceResponse `json:"indices"` 36 | } 37 | 38 | type ClusterStateIndiceResponse struct { 39 | State string `json:"state"` 40 | } 41 | 42 | type ClusterStateRoutingTableResponse struct { 43 | // TODO: unassigned 44 | // 45 | } 46 | -------------------------------------------------------------------------------- /lib/clusternodeshotthreads.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/clusternodesinfo_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "fmt" 16 | "github.com/bmizerany/assert" 17 | "testing" 18 | ) 19 | 20 | func TestGetAll(t *testing.T) { 21 | InitTests(true) 22 | c := NewTestConn() 23 | nodesInfo, err := c.AllNodesInfo() 24 | assert.T(t, err == nil, fmt.Sprintf("should not have gotten error, received: %v", err)) 25 | assert.T(t, nodesInfo.ClusterName != "", fmt.Sprintf("clustername should have been not empty. received: %q", nodesInfo.ClusterName)) 26 | for _, node := range nodesInfo.Nodes { 27 | assert.T(t, node.Settings != nil, fmt.Sprintf("Settings should not have been null")) 28 | assert.T(t, node.OS != nil, fmt.Sprintf("OS should not have been null")) 29 | assert.T(t, node.Process != nil, fmt.Sprintf("Process should not have been null")) 30 | assert.T(t, node.JVM != nil, fmt.Sprintf("JVM should not have been null")) 31 | assert.T(t, node.ThreadPool != nil, fmt.Sprintf("ThreadPool should not have been null")) 32 | assert.T(t, node.Network != nil, fmt.Sprintf("Network should not have been null")) 33 | assert.T(t, node.Transport != nil, fmt.Sprintf("Transport should not have been null")) 34 | assert.T(t, node.Http != nil, fmt.Sprintf("Http should not have been null")) 35 | assert.T(t, node.Plugins != nil, fmt.Sprintf("Plugins should not have been null")) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /lib/clusternodesshutdown.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "fmt" 16 | "net/url" 17 | "strconv" 18 | "strings" 19 | ) 20 | 21 | // NodesShutdown allows the caller to shutdown between one and all nodes in the cluster 22 | // delay is a integer representing number of seconds 23 | // passing "" or "_all" for the nodes parameter will shut down all nodes 24 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-nodes-shutdown/ 25 | func (c *Conn) NodesShutdown(delay int, nodes ...string) error { 26 | shutdownUrl := fmt.Sprintf("/_cluster/nodes/%s/_shutdown", strings.Join(nodes, ",")) 27 | if delay > 0 { 28 | var values url.Values = url.Values{} 29 | values.Add("delay", strconv.Itoa(delay)) 30 | shutdownUrl += "?" + values.Encode() 31 | } 32 | _, err := c.DoCommand("POST", shutdownUrl, nil, nil) 33 | if err != nil { 34 | return err 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /lib/clusternodesstats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Niels Freier 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | ) 17 | 18 | func (c *Conn) NodesStats() (NodeStatsResponse, error) { 19 | var retval NodeStatsResponse 20 | 21 | body, err := c.DoCommand("GET", "/_nodes/stats", nil, nil) 22 | if err != nil { 23 | return retval, err 24 | } 25 | // marshall into json 26 | jsonErr := json.Unmarshal(body, &retval) 27 | if jsonErr != nil { 28 | return retval, jsonErr 29 | } 30 | return retval, err 31 | } 32 | -------------------------------------------------------------------------------- /lib/clusterreroute.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | ) 19 | 20 | // The cluster health API allows to get a very simple status on the health of the cluster. 21 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-health.html 22 | // information returned. Defaults to cluster.) 23 | func (c *Conn) Reroute(dryRun bool, commands Commands) (ClusterHealthResponse, error) { 24 | var url string 25 | var retval ClusterHealthResponse 26 | 27 | if len(commands.Commands) > 0 { 28 | url = fmt.Sprintf("/_cluster/reroute%s&%s", dryRunOption(dryRun)) 29 | } else { 30 | return retval, errors.New("Must pass at least one command") 31 | } 32 | m := map[string]interface{}{"commands": commands.Commands} 33 | body, err := c.DoCommand("POST", url, m, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | 47 | func dryRunOption(isDryRun bool) string { 48 | if isDryRun { 49 | return "dry_run" 50 | } 51 | return "" 52 | } 53 | 54 | // supported commands are 55 | // move (index, shard, from_node, to_node) 56 | // cancel (index, shard, node, allow_primary) 57 | // allocate (index, shard, node, allow_primary) 58 | 59 | type Commands struct { 60 | Commands []interface{} `json:"commands"` 61 | } 62 | 63 | type MoveCommand struct { 64 | Index string `json:"index"` 65 | Shard string `json:"shard"` 66 | FromNode string `json:"from_node"` 67 | ToNode string `json:"to_node"` 68 | } 69 | 70 | type CancelCommand struct { 71 | Index string `json:"index"` 72 | Shard string `json:"shard"` 73 | Node string `json:"node"` 74 | AllowPrimary bool `json:"allow_primary,omitempty"` 75 | } 76 | type AllocateCommand struct { 77 | Index string `json:"index"` 78 | Shard string `json:"shard"` 79 | Node string `json:"node"` 80 | AllowPrimary bool `json:"allow_primary,omitempty"` 81 | } 82 | -------------------------------------------------------------------------------- /lib/clusterstate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | ) 17 | 18 | // State gets the comprehensive state information for the whole cluster 19 | // see http://www.elasticsearch.org/guide/reference/api/admin-cluster-state/ 20 | func (c *Conn) UpdateSetting(args map[string]interface{}, filter_indices ...string) (ClusterStateResponse, error) { 21 | var url string 22 | var retval ClusterStateResponse 23 | 24 | url = "/_cluster/state" 25 | 26 | body, err := c.DoCommand("GET", url, args, nil) 27 | if err != nil { 28 | return retval, err 29 | } 30 | if err == nil { 31 | // marshall into json 32 | jsonErr := json.Unmarshal(body, &retval) 33 | if jsonErr != nil { 34 | return retval, jsonErr 35 | } 36 | } 37 | return retval, err 38 | } 39 | -------------------------------------------------------------------------------- /lib/clusterupdatesettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // UpdateSettings allows to update cluster wide specific settings. Defaults to Transient setting 20 | // Settings updated can either be persistent (applied cross restarts) or transient (will not survive a full cluster restart). 21 | // http://www.elasticsearch.org/guide/reference/api/admin-cluster-update-settings.html 22 | func (c *Conn) UpdateSettings(settingType string, key string, value int) (ClusterSettingsResponse, error) { 23 | var retval ClusterSettingsResponse 24 | if settingType != "transient" && settingType != "persistent" { 25 | return retval, fmt.Errorf("settingType must be one of transient or persistent, you passed %s", settingType) 26 | } 27 | var url string = "/_cluster/state" 28 | m := map[string]map[string]int{settingType: {key: value}} 29 | body, err := c.DoCommand("PUT", url, nil, m) 30 | if err != nil { 31 | return retval, err 32 | } 33 | if err == nil { 34 | // marshall into json 35 | jsonErr := json.Unmarshal(body, &retval) 36 | if jsonErr != nil { 37 | return retval, jsonErr 38 | } 39 | } 40 | return retval, err 41 | } 42 | 43 | type ClusterSettingsResponse struct { 44 | Transient map[string]int `json:"transient"` 45 | Persistent map[string]int `json:"persistent"` 46 | } 47 | -------------------------------------------------------------------------------- /lib/connection.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "errors" 16 | "fmt" 17 | "net/http" 18 | "net/url" 19 | "runtime" 20 | "strings" 21 | "sync" 22 | "time" 23 | 24 | hostpool "github.com/bitly/go-hostpool" 25 | ) 26 | 27 | const ( 28 | Version = "0.0.2" 29 | DefaultProtocol = "http" 30 | DefaultDomain = "localhost" 31 | DefaultPort = "9200" 32 | // A decay duration of zero results in the default behaviour 33 | DefaultDecayDuration = 0 34 | ) 35 | 36 | type Conn struct { 37 | // Maintain these for backwards compatibility 38 | Protocol string 39 | Domain string 40 | ClusterDomains []string 41 | Port string 42 | Username string 43 | Password string 44 | Hosts []string 45 | Gzip bool 46 | RequestTracer func(method, url, body string) 47 | hp hostpool.HostPool 48 | once sync.Once 49 | 50 | // To compute the weighting scores, we perform a weighted average of recent response times, 51 | // over the course of `DecayDuration`. DecayDuration may be set to 0 to use the default 52 | // value of 5 minutes. The EpsilonValueCalculator uses this to calculate a score 53 | // from the weighted average response time. 54 | DecayDuration time.Duration 55 | } 56 | 57 | func NewConn() *Conn { 58 | return &Conn{ 59 | // Maintain these for backwards compatibility 60 | Protocol: DefaultProtocol, 61 | Domain: DefaultDomain, 62 | ClusterDomains: []string{DefaultDomain}, 63 | Port: DefaultPort, 64 | DecayDuration: time.Duration(DefaultDecayDuration * time.Second), 65 | } 66 | } 67 | 68 | func (c *Conn) SetFromUrl(u string) error { 69 | if u == "" { 70 | return errors.New("Url is empty") 71 | } 72 | 73 | parsedUrl, err := url.Parse(u) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | c.Protocol = parsedUrl.Scheme 79 | host, portNum := splitHostnamePartsFromHost(parsedUrl.Host, c.Port) 80 | c.Port = portNum 81 | c.Domain = host 82 | 83 | if parsedUrl.User != nil { 84 | c.Username = parsedUrl.User.Username() 85 | password, passwordIsSet := parsedUrl.User.Password() 86 | if passwordIsSet { 87 | c.Password = password 88 | } 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func (c *Conn) SetPort(port string) { 95 | c.Port = port 96 | } 97 | 98 | func (c *Conn) SetHosts(newhosts []string) { 99 | 100 | // Store the new host list 101 | c.Hosts = newhosts 102 | 103 | // Reinitialise the host pool Pretty naive as this will nuke the current 104 | // hostpool, and therefore reset any scoring 105 | c.initializeHostPool() 106 | } 107 | 108 | // Set up the host pool to be used 109 | func (c *Conn) initializeHostPool() { 110 | 111 | // If no hosts are set, fallback to defaults 112 | if len(c.Hosts) == 0 { 113 | c.Hosts = append(c.Hosts, fmt.Sprintf("%s:%s", c.Domain, c.Port)) 114 | } 115 | 116 | // Epsilon Greedy is an algorithm that allows HostPool not only to 117 | // track failure state, but also to learn about "better" options in 118 | // terms of speed, and to pick from available hosts based on how well 119 | // they perform. This gives a weighted request rate to better 120 | // performing hosts, while still distributing requests to all hosts 121 | // (proportionate to their performance). The interface is the same as 122 | // the standard HostPool, but be sure to mark the HostResponse 123 | // immediately after executing the request to the host, as that will 124 | // stop the implicitly running request timer. 125 | // 126 | // A good overview of Epsilon Greedy is here http://stevehanov.ca/blog/index.php?id=132 127 | if c.hp != nil { 128 | c.hp.Close() 129 | } 130 | c.hp = hostpool.NewEpsilonGreedy( 131 | c.Hosts, c.DecayDuration, &hostpool.LinearEpsilonValueCalculator{}) 132 | } 133 | 134 | func (c *Conn) Close() { 135 | c.hp.Close() 136 | } 137 | 138 | func (c *Conn) NewRequest(method, path, query string) (*Request, error) { 139 | // Setup the hostpool on our first run 140 | c.once.Do(c.initializeHostPool) 141 | 142 | // Get a host from the host pool 143 | hr := c.hp.Get() 144 | 145 | // Get the final host and port 146 | host, portNum := splitHostnamePartsFromHost(hr.Host(), c.Port) 147 | 148 | // Build request 149 | var uri string 150 | // If query parameters are provided, the add them to the URL, 151 | // otherwise, leave them out 152 | if len(query) > 0 { 153 | uri = fmt.Sprintf("%s://%s:%s%s?%s", c.Protocol, host, portNum, path, query) 154 | } else { 155 | uri = fmt.Sprintf("%s://%s:%s%s", c.Protocol, host, portNum, path) 156 | } 157 | req, err := http.NewRequest(method, uri, nil) 158 | if err != nil { 159 | return nil, err 160 | } 161 | req.Header.Add("Accept", "application/json") 162 | req.Header.Add("User-Agent", "elasticSearch/"+Version+" ("+runtime.GOOS+"-"+runtime.GOARCH+")") 163 | 164 | if c.Username != "" || c.Password != "" { 165 | req.SetBasicAuth(c.Username, c.Password) 166 | } 167 | 168 | newRequest := &Request{ 169 | Request: req, 170 | hostResponse: hr, 171 | } 172 | return newRequest, nil 173 | } 174 | 175 | // Split apart the hostname on colon 176 | // Return the host and a default port if there is no separator 177 | func splitHostnamePartsFromHost(fullHost string, defaultPortNum string) (string, string) { 178 | 179 | h := strings.Split(fullHost, ":") 180 | 181 | if len(h) == 2 { 182 | return h[0], h[1] 183 | } 184 | 185 | return h[0], defaultPortNum 186 | } 187 | -------------------------------------------------------------------------------- /lib/connection_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. 3 | // You may obtain a copy of the License at 4 | // http://www.apache.org/licenses/LICENSE-2.0 5 | // Unless required by applicable law or agreed to in writing, software 6 | // distributed under the License is distributed on an "AS IS" BASIS, 7 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 8 | // See the License for the specific language governing permissions and 9 | // limitations under the License. 10 | 11 | package elastigo 12 | 13 | import ( 14 | "fmt" 15 | "testing" 16 | 17 | "github.com/bmizerany/assert" 18 | ) 19 | 20 | func TestSetFromUrl(t *testing.T) { 21 | c := NewConn() 22 | 23 | err := c.SetFromUrl("http://localhost") 24 | exp := "localhost" 25 | assert.T(t, c.Domain == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Domain)) 26 | 27 | c = NewConn() 28 | 29 | err = c.SetFromUrl("http://localhost:9200") 30 | exp = "9200" 31 | assert.T(t, c.Port == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Port)) 32 | 33 | c = NewConn() 34 | 35 | err = c.SetFromUrl("http://localhost:9200") 36 | exp = "localhost" 37 | assert.T(t, c.Domain == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Domain)) 38 | 39 | c = NewConn() 40 | 41 | err = c.SetFromUrl("http://someuser@localhost:9200") 42 | exp = "someuser" 43 | assert.T(t, c.Username == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Username)) 44 | 45 | c = NewConn() 46 | 47 | err = c.SetFromUrl("http://someuser:password@localhost:9200") 48 | exp = "password" 49 | assert.T(t, c.Password == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Password)) 50 | 51 | c = NewConn() 52 | 53 | err = c.SetFromUrl("http://someuser:password@localhost:9200") 54 | exp = "someuser" 55 | assert.T(t, c.Username == exp && err == nil, fmt.Sprintf("Expected %s, got: %s", exp, c.Username)) 56 | 57 | c = NewConn() 58 | 59 | err = c.SetFromUrl("") 60 | exp = "Url is empty" 61 | assert.T(t, err != nil && err.Error() == exp, fmt.Sprintf("Expected %s, got: %s", exp, err.Error())) 62 | } 63 | -------------------------------------------------------------------------------- /lib/corebulkudp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/corecount.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | type CountResponse struct { 20 | Count int `json:"count"` 21 | Shard Status `json:"_shards"` 22 | } 23 | 24 | // Count allows the caller to easily execute a query and get the number of matches for that query. 25 | // It can be executed across one or more indices and across one or more types. 26 | // The query can either be provided using a simple query string as a parameter, 27 | // or using the Query DSL defined within the request body. 28 | // http://www.elasticsearch.org/guide/reference/api/count.html 29 | func (c *Conn) Count(index string, _type string, args map[string]interface{}, query interface{}) (CountResponse, error) { 30 | var url string 31 | var retval CountResponse 32 | url = fmt.Sprintf("/%s/%s/_count", index, _type) 33 | body, err := c.DoCommand("GET", url, args, query) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | -------------------------------------------------------------------------------- /lib/coredelete.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // Delete API allows to delete a typed JSON document from a specific index based on its id. 20 | // http://www.elasticsearch.org/guide/reference/api/delete.html 21 | func (c *Conn) Delete(index string, _type string, id string, args map[string]interface{}) (BaseResponse, error) { 22 | var url string 23 | var retval BaseResponse 24 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 25 | body, err := c.DoCommand("DELETE", url, args, nil) 26 | if err != nil { 27 | return retval, err 28 | } 29 | if err == nil { 30 | // marshall into json 31 | jsonErr := json.Unmarshal(body, &retval) 32 | if jsonErr != nil { 33 | return retval, jsonErr 34 | } 35 | } 36 | return retval, err 37 | } 38 | -------------------------------------------------------------------------------- /lib/coredeletebyquery.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // DeleteByQuery allows the caller to delete documents from one or more indices and one or more types based on a query. 21 | // The query can either be provided using a simple query string as a parameter, or using the Query DSL defined within 22 | // the request body. 23 | // see: http://www.elasticsearch.org/guide/reference/api/delete-by-query.html 24 | func (c *Conn) DeleteByQuery(indices []string, types []string, args map[string]interface{}, query interface{}) (BaseResponse, error) { 25 | var url string 26 | var retval BaseResponse 27 | if len(indices) > 0 && len(types) > 0 { 28 | url = fmt.Sprintf("/%s/%s/_query", strings.Join(indices, ","), strings.Join(types, ",")) 29 | } else if len(indices) > 0 { 30 | url = fmt.Sprintf("/%s/_query", strings.Join(indices, ",")) 31 | } 32 | body, err := c.DoCommand("DELETE", url, args, query) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | 46 | func buildQuery() string { 47 | return "" 48 | } 49 | 50 | type DeleteByQueryResponse struct { 51 | Status bool `json:"ok"` 52 | Indicies map[string]IndexStatus `json:"_indices"` 53 | } 54 | 55 | type IndexStatus struct { 56 | Shards Status `json:"_shards"` 57 | } 58 | -------------------------------------------------------------------------------- /lib/coreexample_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo_test 13 | 14 | import ( 15 | "bytes" 16 | "fmt" 17 | "strconv" 18 | 19 | elastigo "github.com/mattbaird/elastigo/lib" 20 | ) 21 | 22 | // The simplest usage of background bulk indexing 23 | func ExampleBulkIndexer_simple() { 24 | c := elastigo.NewConn() 25 | 26 | indexer := c.NewBulkIndexerErrors(10, 60) 27 | indexer.Start() 28 | indexer.Index("twitter", "user", "1", "", "", nil, `{"name":"bob"}`) 29 | indexer.Stop() 30 | } 31 | 32 | // The inspecting the response 33 | func ExampleBulkIndexer_responses() { 34 | c := elastigo.NewConn() 35 | 36 | indexer := c.NewBulkIndexer(10) 37 | // Create a custom Sender Func, to allow inspection of response/error 38 | indexer.Sender = func(buf *bytes.Buffer) error { 39 | // @buf is the buffer of docs about to be written 40 | respJson, err := c.DoCommand("POST", "/_bulk", nil, buf) 41 | if err != nil { 42 | // handle it better than this 43 | fmt.Println(string(respJson)) 44 | } 45 | return err 46 | } 47 | indexer.Start() 48 | for i := 0; i < 20; i++ { 49 | indexer.Index("twitter", "user", strconv.Itoa(i), "", "", nil, `{"name":"bob"}`) 50 | } 51 | indexer.Stop() 52 | } 53 | -------------------------------------------------------------------------------- /lib/coreexplain.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // Explain computes a score explanation for a query and a specific document. 20 | // This can give useful feedback whether a document matches or didn’t match a specific query. 21 | // This feature is available from version 0.19.9 and up. 22 | // see http://www.elasticsearch.org/guide/reference/api/explain.html 23 | func (c *Conn) Explain(index string, _type string, id string, args map[string]interface{}, query string) (Match, error) { 24 | var url string 25 | var retval Match 26 | if len(_type) > 0 { 27 | url = fmt.Sprintf("/%s/%s/_explain", index, _type) 28 | } else { 29 | url = fmt.Sprintf("/%s/_explain", index) 30 | } 31 | body, err := c.DoCommand("GET", url, args, query) 32 | if err != nil { 33 | return retval, err 34 | } 35 | if err == nil { 36 | // marshall into json 37 | jsonErr := json.Unmarshal(body, &retval) 38 | if jsonErr != nil { 39 | return retval, jsonErr 40 | } 41 | } 42 | return retval, err 43 | } 44 | -------------------------------------------------------------------------------- /lib/coreget.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "net/http" 18 | ) 19 | 20 | // Get allows caller to get a typed JSON document from the index based on its id. 21 | // GET - retrieves the doc 22 | // HEAD - checks for existence of the doc 23 | // http://www.elasticsearch.org/guide/reference/api/get.html 24 | // TODO: make this implement an interface 25 | func (c *Conn) get(index string, _type string, id string, args map[string]interface{}, source *json.RawMessage) (BaseResponse, error) { 26 | var url string 27 | retval := BaseResponse{Source: source} 28 | if len(_type) > 0 { 29 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 30 | } else { 31 | url = fmt.Sprintf("/%s/%s", index, id) 32 | } 33 | body, err := c.DoCommand("GET", url, args, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | 47 | // The get API allows to get a typed JSON document from the index based on its id. 48 | // GET - retrieves the doc 49 | // HEAD - checks for existence of the doc 50 | // http://www.elasticsearch.org/guide/reference/api/get.html 51 | // TODO: make this implement an interface 52 | func (c *Conn) Get(index string, _type string, id string, args map[string]interface{}) (BaseResponse, error) { 53 | return c.get(index, _type, id, args, nil) 54 | } 55 | 56 | // Same as Get but with custom source type. 57 | func (c *Conn) GetCustom(index string, _type string, id string, args map[string]interface{}, source *json.RawMessage) (BaseResponse, error) { 58 | return c.get(index, _type, id, args, source) 59 | } 60 | 61 | // GetSource retrieves the document by id and converts it to provided interface 62 | func (c *Conn) GetSource(index string, _type string, id string, args map[string]interface{}, source interface{}) error { 63 | url := fmt.Sprintf("/%s/%s/%s/_source", index, _type, id) 64 | body, err := c.DoCommand("GET", url, args, nil) 65 | if err == nil { 66 | err = json.Unmarshal(body, &source) 67 | } 68 | return err 69 | } 70 | 71 | // ExistsBool allows caller to check for the existence of a document using HEAD 72 | // TODO(shutej): This looks redundant with the Exists function in 73 | // baserequest.go, check with mattbaird@. 74 | func (c *Conn) ExistsBool(index string, _type string, id string, args map[string]interface{}) (bool, error) { 75 | 76 | var url string 77 | 78 | query, err := Escape(args) 79 | if err != nil { 80 | return false, err 81 | } 82 | 83 | if len(_type) > 0 { 84 | url = fmt.Sprintf("/%s/%s/%s", index, _type, id) 85 | } else { 86 | url = fmt.Sprintf("/%s/%s", index, id) 87 | } 88 | 89 | req, err := c.NewRequest("HEAD", url, query) 90 | if err != nil { 91 | return false, err 92 | } 93 | 94 | httpStatusCode, _, err := req.Do(nil) 95 | 96 | // RecordNotFound is the expected response for a non-existent document, 97 | // so we don't return an error to our caller 98 | if err == RecordNotFound { 99 | return false, nil 100 | } 101 | 102 | return httpStatusCode == http.StatusOK, err 103 | } 104 | 105 | // ExistsIndex allows caller to check for the existence of an index or a type using HEAD 106 | func (c *Conn) ExistsIndex(index string, _type string, args map[string]interface{}) (bool, error) { 107 | var url string 108 | 109 | query, err := Escape(args) 110 | if err != nil { 111 | return false, err 112 | } 113 | 114 | if len(_type) > 0 { 115 | url = fmt.Sprintf("/%s/%s", index, _type) 116 | } else { 117 | url = fmt.Sprintf("/%s", index) 118 | } 119 | req, err := c.NewRequest("HEAD", url, query) 120 | httpStatusCode, _, err := req.Do(nil) 121 | 122 | if err != nil { 123 | return false, err 124 | } 125 | if httpStatusCode == http.StatusOK { 126 | return true, err 127 | } 128 | return false, err 129 | } 130 | -------------------------------------------------------------------------------- /lib/coreindex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | "net/url" 19 | "strconv" 20 | ) 21 | 22 | // Index adds or updates a typed JSON document in a specific index, making it searchable, creating an index 23 | // if it did not exist. 24 | // if id is omited, op_type 'create' will be passed and http method will default to "POST" 25 | // _type is optional 26 | // id is optional 27 | // parentId is optional 28 | // version is optional 29 | // op_type is optional 30 | // routing is optional 31 | // timestamp is optional 32 | // ttl is optional 33 | // percolate is optional 34 | // timeout is optional 35 | // http://www.elasticsearch.org/guide/reference/api/index_.html 36 | func (c *Conn) Index(index string, _type string, id string, args map[string]interface{}, data interface{}) (BaseResponse, error) { 37 | return c.IndexWithParameters(index, _type, id, "", 0, "", "", "", 0, "", "", false, args, data) 38 | } 39 | 40 | // IndexWithParameters takes all the potential parameters available 41 | func (c *Conn) IndexWithParameters(index string, _type string, id string, parentId string, version int, op_type string, 42 | routing string, timestamp string, ttl int, percolate string, timeout string, refresh bool, 43 | args map[string]interface{}, data interface{}) (BaseResponse, error) { 44 | var url string 45 | var retval BaseResponse 46 | url, err := GetIndexUrl(index, _type, id, parentId, version, op_type, routing, timestamp, ttl, percolate, timeout, refresh) 47 | if err != nil { 48 | return retval, err 49 | } 50 | var method string 51 | if len(id) == 0 { 52 | method = "POST" 53 | } else { 54 | method = "PUT" 55 | } 56 | body, err := c.DoCommand(method, url, args, data) 57 | if err != nil { 58 | return retval, err 59 | } 60 | if err == nil { 61 | // marshall into json 62 | jsonErr := json.Unmarshal(body, &retval) 63 | if jsonErr != nil { 64 | return retval, jsonErr 65 | } 66 | } 67 | return retval, err 68 | } 69 | 70 | func GetIndexUrl(index string, _type string, id string, parentId string, version int, op_type string, 71 | routing string, timestamp string, ttl int, percolate string, timeout string, refresh bool) (retval string, e error) { 72 | 73 | if len(index) == 0 { 74 | return "", errors.New("index can not be blank") 75 | } 76 | var partialURL string 77 | var values url.Values = url.Values{} 78 | if len(_type) == 0 && len(id) > 0 { 79 | e = errors.New("Can't specify id when _type is blank") 80 | return 81 | } 82 | if len(_type) > 0 && len(id) > 0 { 83 | partialURL = fmt.Sprintf("/%s/%s/%s", index, _type, id) 84 | } else if len(_type) > 0 { 85 | partialURL = fmt.Sprintf("/%s/%s", index, _type) 86 | } else { 87 | partialURL = fmt.Sprintf("/%s", index) 88 | } 89 | // A child document can be indexed by specifying it’s parent when indexing. 90 | if len(parentId) > 0 { 91 | values.Add("parent", parentId) 92 | } 93 | // versions start at 1, so if greater than 0 94 | if version > 0 { 95 | values.Add("version", strconv.Itoa(version)) 96 | } 97 | if len(op_type) > 0 { 98 | if len(id) == 0 { 99 | //if id is omited, op_type defaults to 'create' 100 | values.Add("op_type", "create") 101 | } else { 102 | values.Add("op_type", op_type) 103 | } 104 | } 105 | if len(routing) > 0 { 106 | values.Add("routing", routing) 107 | } 108 | // A document can be indexed with a timestamp associated with it. 109 | // The timestamp value of a document can be set using the timestamp parameter. 110 | if len(timestamp) > 0 { 111 | values.Add("timestamp", timestamp) 112 | } 113 | // A document can be indexed with a ttl (time to live) associated with it. Expired documents 114 | // will be expunged automatically. 115 | if ttl > 0 { 116 | values.Add("ttl", strconv.Itoa(ttl)) 117 | } 118 | if len(percolate) > 0 { 119 | values.Add("percolate", percolate) 120 | } 121 | // example 5m 122 | if len(timeout) > 0 { 123 | values.Add("timeout", timeout) 124 | } 125 | 126 | if refresh { 127 | values.Add("refresh", "true") 128 | } 129 | 130 | partialURL += "?" + values.Encode() 131 | return partialURL, nil 132 | } 133 | -------------------------------------------------------------------------------- /lib/coremget.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // MGet allows the caller to get multiple documents based on an index, type (optional) and id (and possibly routing). 20 | // The response includes a docs array with all the fetched documents, each element similar in structure to a document 21 | // provided by the get API. 22 | // see http://www.elasticsearch.org/guide/reference/api/multi-get.html 23 | func (c *Conn) MGet(index string, _type string, mgetRequest MGetRequestContainer, args map[string]interface{}) (MGetResponseContainer, error) { 24 | var url string 25 | var retval MGetResponseContainer 26 | if len(index) <= 0 { 27 | url = fmt.Sprintf("/_mget") 28 | } 29 | if len(_type) > 0 && len(index) > 0 { 30 | url = fmt.Sprintf("/%s/%s/_mget", index, _type) 31 | } else if len(index) > 0 { 32 | url = fmt.Sprintf("/%s/_mget", index) 33 | } 34 | body, err := c.DoCommand("GET", url, args, mgetRequest) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | 48 | type MGetRequestContainer struct { 49 | Docs []MGetRequest `json:"docs"` 50 | } 51 | 52 | type MGetRequest struct { 53 | Index string `json:"_index"` 54 | Type string `json:"_type"` 55 | ID string `json:"_id"` 56 | IDS []string `json:"_ids,omitempty"` 57 | Fields []string `json:"fields,omitempty"` 58 | } 59 | 60 | type MGetResponseContainer struct { 61 | Docs []BaseResponse `json:"docs"` 62 | } 63 | -------------------------------------------------------------------------------- /lib/coremorelikethis.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // MoreLikeThis allows the caller to get documents that are “like” a specified document. 20 | // http://www.elasticsearch.org/guide/reference/api/more-like-this.html 21 | func (c *Conn) MoreLikeThis(index string, _type string, id string, args map[string]interface{}, query MoreLikeThisQuery) (BaseResponse, error) { 22 | var url string 23 | var retval BaseResponse 24 | url = fmt.Sprintf("/%s/%s/%s/_mlt", index, _type, id) 25 | body, err := c.DoCommand("GET", url, args, query) 26 | if err != nil { 27 | return retval, err 28 | } 29 | if err == nil { 30 | // marshall into json 31 | jsonErr := json.Unmarshal(body, &retval) 32 | if jsonErr != nil { 33 | return retval, jsonErr 34 | } 35 | } 36 | return retval, err 37 | } 38 | 39 | type MoreLikeThisQuery struct { 40 | MoreLikeThis MLT `json:"more_like_this"` 41 | } 42 | 43 | type MLT struct { 44 | Fields []string `json:"fields"` 45 | LikeText string `json:"like_text"` 46 | PercentTermsToMatch float32 `json:"percent_terms_to_match"` 47 | MinTermFrequency int `json:"min_term_freq"` 48 | MaxQueryTerms int `json:"max_query_terms"` 49 | StopWords []string `json:"stop_words"` 50 | MinDocFrequency int `json:"min_doc_freq"` 51 | MaxDocFrequency int `json:"max_doc_freq"` 52 | MinWordLength int `json:"min_word_len"` 53 | MaxWordLength int `json:"max_word_len"` 54 | BoostTerms int `json:"boost_terms"` 55 | Boost float32 `json:"boost"` 56 | Analyzer string `json:"analyzer"` 57 | } 58 | -------------------------------------------------------------------------------- /lib/coremsearch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/corepercolate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | type PercolatorResult struct { 20 | SearchResult 21 | Matches []PercolatorMatch `json:"matches"` 22 | } 23 | 24 | type PercolatorMatch struct { 25 | Index string `json:"_index"` 26 | Id string `json:"_id"` 27 | } 28 | 29 | // See http://www.elasticsearch.org/guide/reference/api/percolate.html 30 | func (c *Conn) RegisterPercolate(index string, id string, data interface{}) (BaseResponse, error) { 31 | var url string 32 | var retval BaseResponse 33 | url = fmt.Sprintf("/%s/.percolator/%s", index, id) 34 | body, err := c.DoCommand("PUT", url, nil, data) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | 48 | func (c *Conn) Percolate(index string, _type string, name string, args map[string]interface{}, doc string) (PercolatorResult, error) { 49 | var url string 50 | var retval PercolatorResult 51 | url = fmt.Sprintf("/%s/%s/_percolate", index, _type) 52 | body, err := c.DoCommand("GET", url, args, doc) 53 | if err != nil { 54 | return retval, err 55 | } 56 | if err == nil { 57 | // marshall into json 58 | jsonErr := json.Unmarshal(body, &retval) 59 | if jsonErr != nil { 60 | return retval, jsonErr 61 | } 62 | } 63 | return retval, err 64 | } 65 | -------------------------------------------------------------------------------- /lib/corepercolate_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | . "github.com/smartystreets/goconvey/convey" 5 | "testing" 6 | ) 7 | 8 | const ( 9 | percIndexName = "test-perc-index" 10 | ) 11 | 12 | func TestPercolate(t *testing.T) { 13 | Convey("With a registered percolator", t, func() { 14 | c := NewTestConn() 15 | _, createErr := c.CreateIndex(percIndexName) 16 | So(createErr, ShouldBeNil) 17 | defer c.DeleteIndex(percIndexName) 18 | 19 | options := `{ 20 | "percType": { 21 | "properties": { 22 | "message": { 23 | "type": "string" 24 | } 25 | } 26 | } 27 | }` 28 | 29 | err := c.PutMappingFromJSON(percIndexName, "percType", []byte(options)) 30 | So(err, ShouldBeNil) 31 | 32 | data := `{ 33 | "query": { 34 | "match": { 35 | "message": "bonsai tree" 36 | } 37 | } 38 | }` 39 | 40 | _, err = c.RegisterPercolate(percIndexName, "PERCID", data) 41 | So(err, ShouldBeNil) 42 | 43 | Convey("That matches the document", func() { 44 | // Should return the percolator id (registered query) 45 | doc := `{"doc": { "message": "A new bonsai tree in the office" }}` 46 | 47 | result, err := c.Percolate(percIndexName, "percType", "", nil, doc) 48 | So(err, ShouldBeNil) 49 | So(len(result.Matches), ShouldEqual, 1) 50 | match := result.Matches[0] 51 | So(match.Id, ShouldEqual, "PERCID") 52 | So(match.Index, ShouldEqual, percIndexName) 53 | }) 54 | 55 | Convey("That does not match the document", func() { 56 | // Should NOT return the percolator id (registered query) 57 | doc := `{"doc": { "message": "Barren wasteland with no matches" }}` 58 | 59 | result, err := c.Percolate(percIndexName, "percType", "", nil, doc) 60 | So(err, ShouldBeNil) 61 | So(len(result.Matches), ShouldEqual, 0) 62 | }) 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /lib/coresearch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | . "github.com/smartystreets/goconvey/convey" 17 | "testing" 18 | ) 19 | 20 | type SuggestTest struct { 21 | Completion string `json:"completion"` 22 | } 23 | 24 | type hash map[string]interface{} 25 | 26 | func TestCoreSearch(t *testing.T) { 27 | 28 | c := NewTestConn() 29 | c.CreateIndex("github") 30 | waitFor(func() bool { return false }, 5) 31 | 32 | defer func() { 33 | c.DeleteIndex("github") 34 | }() 35 | 36 | Convey("Convert a search result to JSON", t, func() { 37 | 38 | qry := map[string]interface{}{ 39 | "query": map[string]interface{}{ 40 | "wildcard": map[string]string{"actor": "a*"}, 41 | }, 42 | } 43 | var args map[string]interface{} 44 | out, err := c.Search("github", "", args, qry) 45 | So(err, ShouldBeNil) 46 | 47 | _, err = json.Marshal(out.Hits.Hits) 48 | So(err, ShouldBeNil) 49 | }) 50 | 51 | Convey("Update a document and verify that it is reflected", t, func() { 52 | mappingOpts := MappingOptions{Properties: hash{ 53 | "completion": hash{ 54 | "type": "completion", 55 | }, 56 | }} 57 | err := c.PutMapping("github", "SuggestTest", SuggestTest{}, mappingOpts) 58 | So(err, ShouldBeNil) 59 | 60 | _, err = c.UpdateWithPartialDoc("github", "SuggestTest", "1", nil, SuggestTest{"foobar"}, true) 61 | So(err, ShouldBeNil) 62 | 63 | query := hash{"completion_completion": hash{ 64 | "text": "foo", 65 | "completion": hash{ 66 | "size": 10, 67 | "field": "completion", 68 | }, 69 | }} 70 | 71 | _, err = c.Refresh("github") 72 | So(err, ShouldBeNil) 73 | 74 | res, err := c.Suggest("github", nil, query) 75 | So(err, ShouldBeNil) 76 | 77 | opts, err := res.Result("completion_completion") 78 | So(err, ShouldBeNil) 79 | 80 | So(len(opts[0].Options), ShouldBeGreaterThan, 0) 81 | So(opts[0].Options[0].Text, ShouldEqual, "foobar") 82 | }) 83 | } 84 | -------------------------------------------------------------------------------- /lib/coretest_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "bufio" 16 | "bytes" 17 | "compress/gzip" 18 | "crypto/md5" 19 | "encoding/json" 20 | "flag" 21 | "fmt" 22 | "io" 23 | "log" 24 | "net/http" 25 | "time" 26 | ) 27 | 28 | /* 29 | 30 | usage: 31 | 32 | test -v -host eshost -loaddata 33 | 34 | */ 35 | 36 | const ( 37 | testIndex = "github" 38 | ) 39 | 40 | var ( 41 | bulkStarted bool 42 | hasStartedTesting bool 43 | hasLoadedData bool 44 | sleepAfterLoad bool 45 | loadData *bool = flag.Bool("loaddata", false, "This loads a bunch of test data into elasticsearch for testing") 46 | sleep *int = flag.Int("sleep", 0, "Post bulk loading sleep test to make drone.io work") 47 | ) 48 | 49 | func InitTests(startIndexer bool) *Conn { 50 | c := NewConn() 51 | 52 | if !hasStartedTesting { 53 | flag.Parse() 54 | hasStartedTesting = true 55 | log.SetFlags(log.Ltime | log.Lshortfile) 56 | c.Domain = *eshost 57 | } 58 | if startIndexer && !bulkStarted { 59 | bulkStarted = true 60 | b := c.NewBulkIndexer(100) 61 | b.Start() 62 | if *loadData && !hasLoadedData { 63 | log.Println("loading test data ") 64 | hasLoadedData = true 65 | LoadTestData() 66 | } 67 | b.Stop() 68 | } 69 | c.Flush("_all") 70 | c.Refresh("_all") 71 | if !sleepAfterLoad { 72 | time.Sleep(time.Duration(*sleep) * time.Second) 73 | } 74 | sleepAfterLoad = true 75 | return c 76 | } 77 | 78 | func NewTestConn() *Conn { 79 | c := NewConn() 80 | c.Domain = *eshost 81 | return c 82 | } 83 | 84 | // Wait for condition (defined by func) to be true, a utility to create a ticker 85 | // checking every 100 ms to see if something (the supplied check func) is done 86 | // 87 | // waitFor(func() bool { 88 | // return ctr.Ct == 0 89 | // }, 10) 90 | // 91 | // @timeout (in seconds) is the last arg 92 | func waitFor(check func() bool, timeoutSecs int) { 93 | timer := time.NewTicker(100 * time.Millisecond) 94 | tryct := 0 95 | for range timer.C { 96 | if check() { 97 | timer.Stop() 98 | break 99 | } 100 | if tryct >= timeoutSecs*10 { 101 | timer.Stop() 102 | break 103 | } 104 | tryct++ 105 | } 106 | } 107 | 108 | type GithubEvent struct { 109 | Url string 110 | Created time.Time `json:"created_at"` 111 | Type string 112 | } 113 | 114 | // This loads test data from github archives (~6700 docs) 115 | func LoadTestData() { 116 | c := NewConn() 117 | c.Domain = *eshost 118 | 119 | c.DeleteIndex(testIndex) 120 | 121 | docCt := 0 122 | errCt := 0 123 | indexer := c.NewBulkIndexer(1) 124 | indexer.Sender = func(buf *bytes.Buffer) error { 125 | // log.Printf("Sent %d bytes total %d docs sent", buf.Len(), docCt) 126 | req, err := c.NewRequest("POST", "/_bulk", "") 127 | if err != nil { 128 | errCt += 1 129 | log.Fatalf("ERROR: %v", err) 130 | return err 131 | } 132 | req.SetBody(buf) 133 | // res, err := http.DefaultClient.Do(*(api.Request(req))) 134 | var response map[string]interface{} 135 | httpStatusCode, _, err := req.Do(&response) 136 | if err != nil { 137 | errCt += 1 138 | log.Fatalf("ERROR: %v", err) 139 | return err 140 | } 141 | if httpStatusCode != 200 { 142 | log.Fatalf("Not 200! %d %q\n", httpStatusCode, buf.String()) 143 | } 144 | return nil 145 | } 146 | indexer.Start() 147 | resp, err := http.Get("http://data.githubarchive.org/2012-12-10-15.json.gz") 148 | if err != nil || resp == nil { 149 | panic("Could not download data") 150 | } 151 | defer resp.Body.Close() 152 | if err != nil { 153 | log.Println(err) 154 | return 155 | } 156 | gzReader, err := gzip.NewReader(resp.Body) 157 | defer gzReader.Close() 158 | if err != nil { 159 | panic(err) 160 | } 161 | r := bufio.NewReader(gzReader) 162 | var ge GithubEvent 163 | docsm := make(map[string]bool) 164 | h := md5.New() 165 | for { 166 | line, err := r.ReadBytes('\n') 167 | if err != nil { 168 | if err == io.EOF { 169 | indexer.Flush() 170 | break 171 | } 172 | log.Fatalf("could not read line: %v", err) 173 | } 174 | if err := json.Unmarshal(line, &ge); err == nil { 175 | // create an "ID" 176 | h.Write(line) 177 | id := fmt.Sprintf("%x", h.Sum(nil)) 178 | if _, ok := docsm[id]; ok { 179 | log.Println("HM, already exists? ", ge.Url) 180 | } 181 | docsm[id] = true 182 | indexer.Index(testIndex, ge.Type, id, "", "", &ge.Created, line) 183 | docCt++ 184 | } else { 185 | log.Println("ERROR? ", string(line)) 186 | } 187 | } 188 | if errCt != 0 { 189 | log.Println("FATAL, could not load ", errCt) 190 | } 191 | // lets wait a bit to ensure that elasticsearch finishes? 192 | indexer.Stop() 193 | if len(docsm) != docCt { 194 | panic(fmt.Sprintf("Docs didn't match? %d:%d", len(docsm), docCt)) 195 | } 196 | c.Flush(testIndex) 197 | c.Refresh(testIndex) 198 | } 199 | -------------------------------------------------------------------------------- /lib/coreupdate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // Update updates a document based on a script provided. The operation gets the document 20 | // (collocated with the shard) from the index, runs the script (with optional script language and parameters), 21 | // and index back the result (also allows to delete, or ignore the operation). It uses versioning to make sure 22 | // no updates have happened during the “get” and “reindex”. (available from 0.19 onwards). 23 | // Note, this operation still means full reindex of the document, it just removes some network roundtrips 24 | // and reduces chances of version conflicts between the get and the index. The _source field need to be enabled 25 | // for this feature to work. 26 | // 27 | // http://www.elasticsearch.org/guide/reference/api/update.html 28 | // TODO: finish this, it's fairly complex 29 | func (c *Conn) Update(index string, _type string, id string, args map[string]interface{}, data interface{}) (BaseResponse, error) { 30 | var url string 31 | var retval BaseResponse 32 | 33 | url = fmt.Sprintf("/%s/%s/%s/_update", index, _type, id) 34 | body, err := c.DoCommand("POST", url, args, data) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | 48 | // UpdateWithPartialDoc updates a document based on partial document provided. The update API also 49 | // support passing a partial document (since 0.20), which will be merged into the existing 50 | // document (simple recursive merge, inner merging of objects, replacing core "keys/values" and arrays). 51 | // If both doc and script is specified, then doc is ignored. Best is to put your field pairs of the partial 52 | // document in the script itself. 53 | // 54 | // http://www.elasticsearch.org/guide/reference/api/update.html 55 | func (c *Conn) UpdateWithPartialDoc(index string, _type string, id string, args map[string]interface{}, doc interface{}, upsert bool) (BaseResponse, error) { 56 | switch v := doc.(type) { 57 | case string: 58 | upsertStr := "" 59 | if upsert { 60 | upsertStr = ", \"doc_as_upsert\":true" 61 | } 62 | content := fmt.Sprintf("{\"doc\":%s %s}", v, upsertStr) 63 | return c.Update(index, _type, id, args, content) 64 | } 65 | var data map[string]interface{} = make(map[string]interface{}) 66 | data["doc"] = doc 67 | if upsert { 68 | data["doc_as_upsert"] = true 69 | } 70 | return c.Update(index, _type, id, args, data) 71 | } 72 | 73 | // UpdateWithScript updates a document based on a script provided. 74 | // The operation gets the document (collocated with the shard) from the index, runs the script 75 | // (with optional script language and parameters), and index back the result (also allows to 76 | // delete, or ignore the operation). It uses versioning to make sure no updates have happened 77 | // during the "get" and "reindex". (available from 0.19 onwards). 78 | // 79 | // Note, this operation still means full reindex of the document, it just removes some network 80 | // roundtrips and reduces chances of version conflicts between the get and the index. The _source 81 | // field need to be enabled for this feature to work. 82 | // http://www.elasticsearch.org/guide/reference/api/update.html 83 | func (c *Conn) UpdateWithScript(index string, _type string, id string, args map[string]interface{}, script string, params interface{}) (BaseResponse, error) { 84 | switch v := params.(type) { 85 | case string: 86 | paramsPart := fmt.Sprintf("{\"params\":%s}", v) 87 | data := fmt.Sprintf("{\"script\":\"%s\", \"params\":%s}", script, paramsPart) 88 | return c.Update(index, _type, id, args, data) 89 | } 90 | var data map[string]interface{} = make(map[string]interface{}) 91 | data["params"] = params 92 | data["script"] = script 93 | return c.Update(index, _type, id, args, data) 94 | } 95 | -------------------------------------------------------------------------------- /lib/corevalidate.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // Validate allows a user to validate a potentially expensive query without executing it. 20 | // see http://www.elasticsearch.org/guide/reference/api/validate.html 21 | func (c *Conn) Validate(index string, _type string, args map[string]interface{}) (BaseResponse, error) { 22 | var url string 23 | var retval BaseResponse 24 | if len(_type) > 0 { 25 | url = fmt.Sprintf("/%s/%s/_validate/", index, _type) 26 | } else { 27 | url = fmt.Sprintf("/%s/_validate/", index) 28 | } 29 | body, err := c.DoCommand("GET", url, args, nil) 30 | if err != nil { 31 | return retval, err 32 | } 33 | if err == nil { 34 | // marshall into json 35 | jsonErr := json.Unmarshal(body, &retval) 36 | if jsonErr != nil { 37 | return retval, jsonErr 38 | } 39 | } 40 | return retval, err 41 | } 42 | 43 | type Validation struct { 44 | Valid bool `json:"valid"` 45 | Shards Status `json:"_shards"` 46 | Explainations []Explaination `json:"explanations,omitempty"` 47 | } 48 | 49 | type Explaination struct { 50 | Index string `json:"index"` 51 | Valid bool `json:"valid"` 52 | Error string `json:"error"` 53 | } 54 | -------------------------------------------------------------------------------- /lib/error.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // 404 Response. 8 | var RecordNotFound = errors.New("record not found") 9 | -------------------------------------------------------------------------------- /lib/indicesaliases.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http:www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | type JsonAliases struct { 20 | Actions []JsonAliasAdd `json:"actions"` 21 | } 22 | 23 | type JsonAliasAdd struct { 24 | Add JsonAlias `json:"add"` 25 | } 26 | 27 | type JsonAlias struct { 28 | Index string `json:"index"` 29 | Alias string `json:"alias"` 30 | } 31 | 32 | // The API allows you to create an index alias through an API. 33 | func (c *Conn) AddAlias(index string, alias string) (BaseResponse, error) { 34 | var url string 35 | var retval BaseResponse 36 | 37 | if len(index) > 0 { 38 | url = "/_aliases" 39 | } else { 40 | return retval, fmt.Errorf("You must specify an index to create the alias on") 41 | } 42 | 43 | jsonAliases := JsonAliases{} 44 | jsonAliasAdd := JsonAliasAdd{} 45 | jsonAliasAdd.Add.Alias = alias 46 | jsonAliasAdd.Add.Index = index 47 | jsonAliases.Actions = append(jsonAliases.Actions, jsonAliasAdd) 48 | requestBody, err := json.Marshal(jsonAliases) 49 | 50 | if err != nil { 51 | return retval, err 52 | } 53 | 54 | body, err := c.DoCommand("POST", url, nil, requestBody) 55 | if err != nil { 56 | return retval, err 57 | } 58 | 59 | jsonErr := json.Unmarshal(body, &retval) 60 | if jsonErr != nil { 61 | return retval, jsonErr 62 | } 63 | 64 | return retval, err 65 | } 66 | -------------------------------------------------------------------------------- /lib/indicesanalyze.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "errors" 17 | "fmt" 18 | ) 19 | 20 | // AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text. 21 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/ 22 | func (c *Conn) AnalyzeIndices(index string, args map[string]interface{}) (AnalyzeResponse, error) { 23 | var retval AnalyzeResponse 24 | if len(args["text"].(string)) == 0 { 25 | return retval, errors.New("text to analyze must not be blank") 26 | } 27 | var analyzeUrl string = "/_analyze" 28 | if len(index) > 0 { 29 | analyzeUrl = fmt.Sprintf("/%s/%s", index, analyzeUrl) 30 | } 31 | 32 | body, err := c.DoCommand("GET", analyzeUrl, args, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | 46 | type AnalyzeResponse struct { 47 | Tokens []Token `json:"tokens"` 48 | } 49 | type Token struct { 50 | Name string `json:"token"` 51 | StartOffset int `json:"start_offset"` 52 | EndOffset int `json:"end_offset"` 53 | Type string `json:"type"` 54 | Position int `json:"position"` 55 | } 56 | -------------------------------------------------------------------------------- /lib/indicesclearcache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // ClearCache allows to clear either all caches or specific cached associated with one ore more indices. 21 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-clearcache/ 22 | func (c *Conn) ClearCache(clearId bool, clearBloom bool, args map[string]interface{}, indices ...string) (ExtendedStatus, error) { 23 | var retval ExtendedStatus 24 | var clearCacheUrl string 25 | if len(indices) > 0 { 26 | clearCacheUrl = fmt.Sprintf("/%s/_cache/clear", strings.Join(indices, ",")) 27 | 28 | } else { 29 | clearCacheUrl = fmt.Sprintf("/_cache/clear") 30 | } 31 | 32 | body, err := c.DoCommand("POST", clearCacheUrl, args, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | -------------------------------------------------------------------------------- /lib/indicescreateindex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "reflect" 18 | ) 19 | 20 | // The create API allows you to create an indices through an API. 21 | func (c *Conn) CreateIndex(index string) (BaseResponse, error) { 22 | var url string 23 | var retval BaseResponse 24 | 25 | if len(index) > 0 { 26 | url = fmt.Sprintf("/%s", index) 27 | } else { 28 | return retval, fmt.Errorf("You must specify an index to create") 29 | } 30 | 31 | body, err := c.DoCommand("PUT", url, nil, nil) 32 | if err != nil { 33 | return retval, err 34 | } 35 | 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | 41 | return retval, err 42 | } 43 | 44 | // The create API allows you to create an indices through an API. 45 | func (c *Conn) CreateIndexWithSettings(index string, settings interface{}) (BaseResponse, error) { 46 | var url string 47 | var retval BaseResponse 48 | 49 | settingsType := reflect.TypeOf(settings).Kind() 50 | if settingsType != reflect.Struct && settingsType != reflect.Map { 51 | return retval, fmt.Errorf("Settings kind was not struct or map") 52 | } 53 | 54 | requestBody, err := json.Marshal(settings) 55 | 56 | if err != nil { 57 | return retval, err 58 | } 59 | 60 | if len(index) > 0 { 61 | url = fmt.Sprintf("/%s", index) 62 | } else { 63 | return retval, fmt.Errorf("You must specify an index to create") 64 | } 65 | 66 | body, err := c.DoCommand("PUT", url, nil, requestBody) 67 | if err != nil { 68 | return retval, err 69 | } 70 | 71 | jsonErr := json.Unmarshal(body, &retval) 72 | if jsonErr != nil { 73 | return retval, jsonErr 74 | } 75 | 76 | return retval, err 77 | } 78 | -------------------------------------------------------------------------------- /lib/indicesdeleteindex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // The delete API allows you to delete one or more indices through an API. This operation may fail 20 | // if the elasticsearch configuration has been set to forbid deleting indexes. 21 | func (c *Conn) DeleteIndex(index string) (BaseResponse, error) { 22 | var url string 23 | var retval BaseResponse 24 | 25 | if len(index) > 0 { 26 | url = fmt.Sprintf("/%s", index) 27 | } else { 28 | return retval, fmt.Errorf("You must specify at least one index to delete") 29 | } 30 | 31 | body, err := c.DoCommand("DELETE", url, nil, nil) 32 | if err != nil { 33 | return retval, err 34 | } 35 | 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | 41 | return retval, err 42 | } 43 | -------------------------------------------------------------------------------- /lib/indicesdeletemapping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // The delete API allows you to delete a mapping through an API. 20 | func (c *Conn) DeleteMapping(index string, typeName string) (BaseResponse, error) { 21 | var retval BaseResponse 22 | 23 | if len(index) == 0 { 24 | return retval, fmt.Errorf("You must specify at least one index to delete a mapping from") 25 | } 26 | 27 | if len(typeName) == 0 { 28 | return retval, fmt.Errorf("You must specify at least one mapping to delete") 29 | } 30 | 31 | // As documented at http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-delete-mapping.html 32 | url := fmt.Sprintf("/%s/%s", index, typeName) 33 | 34 | body, err := c.DoCommand("DELETE", url, nil, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | 44 | return retval, err 45 | } 46 | -------------------------------------------------------------------------------- /lib/indicesdeletemapping_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "net/url" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func TestDeleteMapping(t *testing.T) { 12 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | if r.Method != "DELETE" { 14 | t.Errorf("Expected HTTP Verb, DELETE") 15 | } 16 | 17 | if r.URL.Path == "/this/exists" { 18 | w.Write([]byte(`{"acknowledged": true}`)) 19 | } else if r.URL.Path == "/this/not_exists" { 20 | w.WriteHeader(http.StatusNotFound) 21 | w.Write([]byte(`{"error": "TypeMissingException[[_all] type[[not_exists]] missing: No index has the type.]","status": 404}`)) 22 | } else { 23 | t.Errorf("Unexpected request path, %s", r.URL.Path) 24 | } 25 | })) 26 | defer ts.Close() 27 | 28 | serverURL, _ := url.Parse(ts.URL) 29 | 30 | c := NewTestConn() 31 | 32 | c.Domain = strings.Split(serverURL.Host, ":")[0] 33 | c.Port = strings.Split(serverURL.Host, ":")[1] 34 | 35 | _, err := c.DeleteMapping("this", "exists") 36 | if err != nil { 37 | t.Errorf("Expected no error and got, %s", err) 38 | } 39 | 40 | _, err = c.DeleteMapping("this", "not_exists") 41 | if err == nil { 42 | t.Errorf("Expected error and got none deleting /this/not_exists") 43 | } 44 | 45 | _, err = c.DeleteMapping("", "two") 46 | if err == nil { 47 | t.Errorf("Expected error for no index and got none") 48 | } 49 | 50 | _, err = c.DeleteMapping("one", "") 51 | if err == nil { 52 | t.Errorf("Expected error for no mapping and got none") 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lib/indicesdoc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/indicesflush.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // Flush flushes one or more indices through an API. The flush process of an index basically 21 | // frees memory from the index by flushing data to the index storage and clearing the internal transaction 22 | // log. By default, ElasticSearch uses memory heuristics in order to automatically trigger flush operations 23 | // as required in order to clear memory. 24 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-flush.html 25 | // TODO: add Shards to response 26 | func (c *Conn) Flush(indices ...string) (BaseResponse, error) { 27 | var url string 28 | var retval BaseResponse 29 | if len(indices) > 0 { 30 | url = fmt.Sprintf("/%s/_flush", strings.Join(indices, ",")) 31 | } else { 32 | url = "/_flush" 33 | } 34 | body, err := c.DoCommand("POST", url, nil, nil) 35 | if err != nil { 36 | return retval, err 37 | } 38 | if err == nil { 39 | // marshall into json 40 | jsonErr := json.Unmarshal(body, &retval) 41 | if jsonErr != nil { 42 | return retval, jsonErr 43 | } 44 | } 45 | return retval, err 46 | } 47 | -------------------------------------------------------------------------------- /lib/indicesgetsettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/indicesindicesexists.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "fmt" 16 | "strings" 17 | ) 18 | 19 | // IndicesExists checks for the existence of indices. uses RecordNotFound message if it doesn't exist and 20 | // "no error" situation if it exists. If there is some other error, gives the error and says it exists 21 | // just in case 22 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-indices-exists/ 23 | func (c *Conn) IndicesExists(indices ...string) (bool, error) { 24 | var url string 25 | if len(indices) > 0 { 26 | url = fmt.Sprintf("/%s", strings.Join(indices, ",")) 27 | } 28 | _, err := c.DoCommand("HEAD", url, nil, nil) 29 | if err != nil { 30 | if err == RecordNotFound { 31 | return false, nil 32 | } else { 33 | return true, err 34 | } 35 | } 36 | return true, nil 37 | } 38 | -------------------------------------------------------------------------------- /lib/indicesopencloseindex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | func (c *Conn) OpenIndices() (BaseResponse, error) { 20 | return c.openCloseOperation("_all", "_open") 21 | } 22 | 23 | func (c *Conn) CloseIndices() (BaseResponse, error) { 24 | return c.openCloseOperation("_all", "_close") 25 | } 26 | 27 | func (c *Conn) OpenIndex(index string) (BaseResponse, error) { 28 | return c.openCloseOperation(index, "_open") 29 | } 30 | 31 | func (c *Conn) CloseIndex(index string) (BaseResponse, error) { 32 | return c.openCloseOperation(index, "_close") 33 | } 34 | 35 | func (c *Conn) openCloseOperation(index, mode string) (BaseResponse, error) { 36 | var url string 37 | var retval BaseResponse 38 | 39 | if len(index) > 0 { 40 | url = fmt.Sprintf("/%s/%s", index, mode) 41 | } else { 42 | url = fmt.Sprintf("/%s", mode) 43 | } 44 | 45 | body, errDo := c.DoCommand("POST", url, nil, nil) 46 | if errDo != nil { 47 | return retval, errDo 48 | } 49 | jsonErr := json.Unmarshal(body, &retval) 50 | if jsonErr != nil { 51 | return retval, jsonErr 52 | } 53 | return retval, errDo 54 | } 55 | -------------------------------------------------------------------------------- /lib/indicesoptimize.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // AnalyzeIndices performs the analysis process on a text and return the tokens breakdown of the text. 21 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-analyze/ 22 | func (c *Conn) OptimizeIndices(args map[string]interface{}, indices ...string) (ExtendedStatus, error) { 23 | var retval ExtendedStatus 24 | var optimizeUrl string = "/_optimize" 25 | if len(indices) > 0 { 26 | optimizeUrl = fmt.Sprintf("/%s%s", strings.Join(indices, ","), optimizeUrl) 27 | } 28 | 29 | body, err := c.DoCommand("POST", optimizeUrl, args, nil) 30 | if err != nil { 31 | return retval, err 32 | } 33 | if err == nil { 34 | // marshall into json 35 | jsonErr := json.Unmarshal(body, &retval) 36 | if jsonErr != nil { 37 | return retval, jsonErr 38 | } 39 | } 40 | return retval, err 41 | } 42 | -------------------------------------------------------------------------------- /lib/indicesputmapping.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "reflect" 18 | "strings" 19 | ) 20 | 21 | type Mapping map[string]MappingOptions 22 | 23 | type MappingOptions struct { 24 | Id IdOptions `json:"_id"` 25 | Timestamp TimestampOptions `json:"_timestamp"` 26 | Analyzer *AnalyzerOptions `json:"_analyzer,omitempty"` 27 | Parent *ParentOptions `json:"_parent,omitempty"` 28 | Routing *RoutingOptions `json:"_routing,omitempty"` 29 | Size *SizeOptions `json:"_size,omitempty"` 30 | Source *SourceOptions `json:"_source,omitempty"` 31 | TTL *TTLOptions `json:"_ttl,omitempty"` 32 | Type *TypeOptions `json:"_type,omitempty"` 33 | Properties map[string]interface{} `json:"properties"` 34 | DynamicTemplates []map[string]interface{} `json:"dynamic_templates,omitempty"` 35 | } 36 | 37 | type TimestampOptions struct { 38 | Enabled bool `json:"enabled"` 39 | } 40 | 41 | type AnalyzerOptions struct { 42 | Path string `json:"path,omitempty"` 43 | Index string `json:"index,omitempty"` 44 | } 45 | 46 | type ParentOptions struct { 47 | Type string `json:"type"` 48 | } 49 | 50 | type RoutingOptions struct { 51 | Required bool `json:"required,omitempty"` 52 | Path string `json:"path,omitempty"` 53 | } 54 | 55 | type SizeOptions struct { 56 | Enabled bool `json:"enabled,omitempty"` 57 | Store bool `json:"store,omitempty"` 58 | } 59 | 60 | type SourceOptions struct { 61 | Enabled bool `json:"enabled,omitempty"` 62 | Includes []string `json:"includes,omitempty"` 63 | Excludes []string `json:"excludes,omitempty"` 64 | } 65 | 66 | type TypeOptions struct { 67 | Store bool `json:"store,omitempty"` 68 | Index string `json:"index,omitempty"` 69 | } 70 | 71 | type TTLOptions struct { 72 | Enabled bool `json:"enabled"` 73 | Default string `json:"default,omitempty"` 74 | } 75 | 76 | type IdOptions struct { 77 | Index string `json:"index,omitempty"` 78 | Path string `json:"path,omitempty"` 79 | } 80 | 81 | func (m_ Mapping) Options() MappingOptions { 82 | m := map[string]MappingOptions(m_) 83 | for _, v := range m { 84 | return v 85 | } 86 | panic(fmt.Errorf("Malformed input: %v", m_)) 87 | } 88 | 89 | func MappingForType(typeName string, opts MappingOptions) Mapping { 90 | return map[string]MappingOptions{typeName: opts} 91 | } 92 | 93 | func (c *Conn) PutMapping(index string, typeName string, instance interface{}, opt MappingOptions) error { 94 | instanceType := reflect.TypeOf(instance) 95 | if instanceType.Kind() != reflect.Struct { 96 | return fmt.Errorf("instance kind was not struct") 97 | } 98 | 99 | if opt.Properties == nil { 100 | opt.Properties = make(map[string]interface{}) 101 | } 102 | getProperties(instanceType, opt.Properties) 103 | body, err := json.Marshal(MappingForType(typeName, opt)) 104 | if err != nil { 105 | return err 106 | } 107 | _, err = c.DoCommand("PUT", fmt.Sprintf("/%s/%s/_mapping", index, typeName), nil, string(body)) 108 | if err != nil { 109 | return err 110 | } 111 | 112 | return nil 113 | } 114 | 115 | //Same as PutMapping, but takes a []byte for mapping and provides no check of structure 116 | func (c *Conn) PutMappingFromJSON(index string, typeName string, mapping []byte) error { 117 | _, err := c.DoCommand("PUT", fmt.Sprintf("/%s/%s/_mapping", index, typeName), nil, string(mapping)) 118 | return err 119 | } 120 | 121 | func getProperties(t reflect.Type, prop map[string]interface{}) { 122 | n := t.NumField() 123 | for i := 0; i < n; i++ { 124 | field := t.Field(i) 125 | 126 | name := strings.Split(field.Tag.Get("json"), ",")[0] 127 | if name == "-" { 128 | continue 129 | } else if name == "" { 130 | name = field.Name 131 | } 132 | 133 | attrMap := make(map[string]interface{}) 134 | attrs := splitTag(field.Tag.Get("elastic")) 135 | for _, attr := range attrs { 136 | keyvalue := strings.Split(attr, ":") 137 | attrMap[keyvalue[0]] = keyvalue[1] 138 | } 139 | 140 | if len(attrMap) == 0 || attrMap["type"] == "nested" { 141 | 142 | // We are looking for tags on any inner struct, independently of 143 | // whether the field is a struct, a pointer to struct, or a slice of structs 144 | targetType := field.Type 145 | if targetType.Kind() == reflect.Ptr || 146 | targetType.Kind() == reflect.Slice { 147 | targetType = field.Type.Elem() 148 | } 149 | if targetType.Kind() == reflect.Struct { 150 | if field.Anonymous { 151 | getProperties(targetType, prop) 152 | } else { 153 | innerStructProp := make(map[string]interface{}) 154 | getProperties(targetType, innerStructProp) 155 | attrMap["properties"] = innerStructProp 156 | } 157 | } 158 | } 159 | if len(attrMap) != 0 { 160 | prop[name] = attrMap 161 | } 162 | } 163 | } 164 | 165 | func splitTag(tag string) []string { 166 | tag = strings.Trim(tag, " ") 167 | if tag == "" { 168 | return []string{} 169 | } else { 170 | return strings.Split(tag, ",") 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /lib/indicesputsettings.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "reflect" 7 | ) 8 | 9 | func (c *Conn) PutSettings(index string, settings interface{}) (BaseResponse, error) { 10 | 11 | var url string 12 | var retval BaseResponse 13 | 14 | settingsType := reflect.TypeOf(settings) 15 | if settingsType.Kind() != reflect.Struct { 16 | return retval, fmt.Errorf("Settings kind was not struct") 17 | } 18 | 19 | if len(index) > 0 { 20 | url = fmt.Sprintf("/%s/_settings", index) 21 | } else { 22 | url = fmt.Sprintf("/_settings") 23 | } 24 | 25 | requestBody, err := json.Marshal(settings) 26 | 27 | if err != nil { 28 | return retval, err 29 | } 30 | 31 | body, errDo := c.DoCommand("PUT", url, nil, requestBody) 32 | if errDo != nil { 33 | return retval, errDo 34 | } 35 | 36 | jsonErr := json.Unmarshal(body, &retval) 37 | if jsonErr != nil { 38 | return retval, jsonErr 39 | } 40 | 41 | return retval, err 42 | } 43 | -------------------------------------------------------------------------------- /lib/indicesrefresh.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // Refresh explicitly refreshes one or more index, making all operations performed since 21 | // the last refresh available for search. The (near) real-time capabilities depend on the index engine used. 22 | // For example, the internal one requires refresh to be called, but by default a refresh is scheduled periodically. 23 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-refresh.html 24 | // TODO: add Shards to response 25 | func (c *Conn) Refresh(indices ...string) (BaseResponse, error) { 26 | var url string 27 | var retval BaseResponse 28 | if len(indices) > 0 { 29 | url = fmt.Sprintf("/%s/_refresh", strings.Join(indices, ",")) 30 | } else { 31 | url = "/_refresh" 32 | } 33 | body, err := c.DoCommand("POST", url, nil, nil) 34 | if err != nil { 35 | return retval, err 36 | } 37 | if err == nil { 38 | // marshall into json 39 | jsonErr := json.Unmarshal(body, &retval) 40 | if jsonErr != nil { 41 | return retval, jsonErr 42 | } 43 | } 44 | return retval, err 45 | } 46 | -------------------------------------------------------------------------------- /lib/indicessegments.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/indicessnapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // Snapshot allows to explicitly perform a snapshot through the gateway of one or more indices (backup them). 21 | // By default, each index gateway periodically snapshot changes, though it can be disabled and be controlled completely through this API. 22 | // see http://www.elasticsearch.org/guide/reference/api/admin-indices-gateway-snapshot/ 23 | func (c *Conn) Snapshot(indices ...string) (ExtendedStatus, error) { 24 | var retval ExtendedStatus 25 | var url string 26 | if len(indices) > 0 { 27 | url = fmt.Sprintf("/%s/_gateway/snapshot", strings.Join(indices, ",")) 28 | 29 | } else { 30 | url = fmt.Sprintf("/_gateway/snapshot") 31 | } 32 | body, err := c.DoCommand("POST", url, nil, nil) 33 | if err != nil { 34 | return retval, err 35 | } 36 | if err == nil { 37 | // marshall into json 38 | jsonErr := json.Unmarshal(body, &retval) 39 | if jsonErr != nil { 40 | return retval, jsonErr 41 | } 42 | } 43 | return retval, err 44 | } 45 | -------------------------------------------------------------------------------- /lib/indicesstats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/indicesstatus.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "strings" 18 | ) 19 | 20 | // Status lists status details of all indices or the specified index. 21 | // http://www.elasticsearch.org/guide/reference/api/admin-indices-status.html 22 | func (c *Conn) Status(args map[string]interface{}, indices ...string) (BaseResponse, error) { 23 | var retval BaseResponse 24 | var url string 25 | if len(indices) > 0 { 26 | url = fmt.Sprintf("/%s/_status", strings.Join(indices, ",")) 27 | 28 | } else { 29 | url = "/_status" 30 | } 31 | body, err := c.DoCommand("GET", url, args, nil) 32 | if err != nil { 33 | return retval, err 34 | } 35 | if err == nil { 36 | // marshall into json 37 | jsonErr := json.Unmarshal(body, &retval) 38 | if jsonErr != nil { 39 | return retval, jsonErr 40 | } 41 | } 42 | return retval, err 43 | } 44 | -------------------------------------------------------------------------------- /lib/indicestemplates.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/indicesupdatesettings.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | -------------------------------------------------------------------------------- /lib/request.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "bytes" 16 | "compress/gzip" 17 | "encoding/json" 18 | "fmt" 19 | "io" 20 | "io/ioutil" 21 | "mime" 22 | "net/http" 23 | "net/url" 24 | "reflect" 25 | "strconv" 26 | "strings" 27 | 28 | hostpool "github.com/bitly/go-hostpool" 29 | ) 30 | 31 | type Request struct { 32 | *http.Client 33 | *http.Request 34 | hostResponse hostpool.HostPoolResponse 35 | } 36 | 37 | func (r *Request) SetBodyGzip(data interface{}) error { 38 | buf := new(bytes.Buffer) 39 | gw := gzip.NewWriter(buf) 40 | 41 | switch v := data.(type) { 42 | case string: 43 | if _, err := gw.Write([]byte(v)); err != nil { 44 | return err 45 | } 46 | case []byte: 47 | if _, err := gw.Write([]byte(v)); err != nil { 48 | return err 49 | } 50 | case io.Reader: 51 | if _, err := io.Copy(gw, v); err != nil { 52 | return err 53 | } 54 | default: 55 | b, err := json.Marshal(data) 56 | if err != nil { 57 | return err 58 | } 59 | if _, err := gw.Write(b); err != nil { 60 | return err 61 | } 62 | } 63 | 64 | if err := gw.Close(); err != nil { 65 | return err 66 | } 67 | r.SetBody(bytes.NewReader(buf.Bytes())) 68 | r.ContentLength = int64(len(buf.Bytes())) 69 | r.Header.Add("Accept-Charset", "utf-8") 70 | r.Header.Set("Content-Encoding", "gzip") 71 | return nil 72 | } 73 | 74 | func (r *Request) SetBodyJson(data interface{}) error { 75 | body, err := json.Marshal(data) 76 | if err != nil { 77 | return err 78 | } 79 | r.SetBodyBytes(body) 80 | r.Header.Set("Content-Type", "application/json") 81 | return nil 82 | } 83 | 84 | func (r *Request) SetBodyString(body string) { 85 | r.SetBody(strings.NewReader(body)) 86 | } 87 | 88 | func (r *Request) SetBodyBytes(body []byte) { 89 | r.SetBody(bytes.NewReader(body)) 90 | } 91 | 92 | func (r *Request) SetBody(body io.Reader) { 93 | rc, ok := body.(io.ReadCloser) 94 | if !ok && body != nil { 95 | rc = ioutil.NopCloser(body) 96 | } 97 | r.Body = rc 98 | r.ContentLength = -1 99 | } 100 | 101 | func (r *Request) Do(v interface{}) (int, []byte, error) { 102 | response, bodyBytes, err := r.DoResponse(v) 103 | if err != nil { 104 | return -1, nil, err 105 | } 106 | return response.StatusCode, bodyBytes, err 107 | } 108 | 109 | func (r *Request) DoResponse(v interface{}) (*http.Response, []byte, error) { 110 | var client = r.Client 111 | if client == nil { 112 | client = http.DefaultClient 113 | } 114 | 115 | res, err := client.Do(r.Request) 116 | // Inform the HostPool of what happened to the request and allow it to update 117 | r.hostResponse.Mark(err) 118 | if err != nil { 119 | return nil, nil, err 120 | } 121 | 122 | defer res.Body.Close() 123 | bodyBytes, err := ioutil.ReadAll(res.Body) 124 | 125 | if err != nil { 126 | return nil, nil, err 127 | } 128 | 129 | if res.StatusCode == 404 { 130 | return nil, bodyBytes, RecordNotFound 131 | } 132 | 133 | if res.StatusCode > 304 && v != nil { 134 | // Make sure the response is JSON and not some other type. 135 | // i.e. 502 or 504 errors from a proxy. 136 | mediaType, _, err := mime.ParseMediaType(res.Header.Get("Content-Type")) 137 | if err != nil { 138 | return nil, bodyBytes, err 139 | } 140 | 141 | if mediaType != "application/json" { 142 | return nil, bodyBytes, fmt.Errorf(http.StatusText(res.StatusCode)) 143 | } 144 | 145 | jsonErr := json.Unmarshal(bodyBytes, v) 146 | if jsonErr != nil { 147 | return nil, nil, fmt.Errorf("Json response unmarshal error: [%s], response content: [%s]", jsonErr.Error(), string(bodyBytes)) 148 | } 149 | } 150 | return res, bodyBytes, err 151 | } 152 | 153 | func Escape(args map[string]interface{}) (s string, err error) { 154 | vals := url.Values{} 155 | for key, val := range args { 156 | switch v := val.(type) { 157 | case string: 158 | vals.Add(key, v) 159 | case bool: 160 | vals.Add(key, strconv.FormatBool(v)) 161 | case int, int32, int64: 162 | vInt := reflect.ValueOf(v).Int() 163 | vals.Add(key, strconv.FormatInt(vInt, 10)) 164 | case float32, float64: 165 | vFloat := reflect.ValueOf(v).Float() 166 | vals.Add(key, strconv.FormatFloat(vFloat, 'f', -1, 32)) 167 | case []string: 168 | vals.Add(key, strings.Join(v, ",")) 169 | default: 170 | err = fmt.Errorf("Could not format URL argument: %s", key) 171 | return 172 | } 173 | } 174 | s = vals.Encode() 175 | return 176 | } 177 | -------------------------------------------------------------------------------- /lib/searchaggregate.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import "encoding/json" 4 | 5 | func Aggregate(name string) *AggregateDsl { 6 | return &AggregateDsl{Name: name} 7 | } 8 | 9 | type AggregateDsl struct { 10 | Name string 11 | TypeName string 12 | Type interface{} 13 | Filters *FilterWrap `json:"filters,omitempty"` 14 | AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"` 15 | } 16 | 17 | type FieldAggregate struct { 18 | Field string `json:"field"` 19 | Size *int `json:"size,omitempty"` 20 | } 21 | 22 | /** 23 | * Aggregates accepts n "sub-aggregates" to be applied to this aggregate 24 | * 25 | * agg := Aggregate("user").Term("user_id") 26 | * agg.Aggregates( 27 | * Aggregate("total_spent").Sum("price"), 28 | * Aggregate("total_saved").Sum("discount"), 29 | * ) 30 | */ 31 | func (d *AggregateDsl) Aggregates(aggs ...*AggregateDsl) *AggregateDsl { 32 | if len(aggs) < 1 { 33 | return d 34 | } 35 | if len(d.AggregatesVal) == 0 { 36 | d.AggregatesVal = make(map[string]*AggregateDsl) 37 | } 38 | 39 | for _, agg := range aggs { 40 | d.AggregatesVal[agg.Name] = agg 41 | } 42 | return d 43 | } 44 | 45 | func (d *AggregateDsl) Min(field string) *AggregateDsl { 46 | d.Type = FieldAggregate{Field: field} 47 | d.TypeName = "min" 48 | return d 49 | } 50 | 51 | func (d *AggregateDsl) Max(field string) *AggregateDsl { 52 | d.Type = FieldAggregate{Field: field} 53 | d.TypeName = "max" 54 | return d 55 | } 56 | 57 | func (d *AggregateDsl) Sum(field string) *AggregateDsl { 58 | d.Type = FieldAggregate{Field: field} 59 | d.TypeName = "sum" 60 | return d 61 | } 62 | 63 | func (d *AggregateDsl) Avg(field string) *AggregateDsl { 64 | d.Type = FieldAggregate{Field: field} 65 | d.TypeName = "avg" 66 | return d 67 | } 68 | 69 | func (d *AggregateDsl) Stats(field string) *AggregateDsl { 70 | d.Type = FieldAggregate{Field: field} 71 | d.TypeName = "stats" 72 | return d 73 | } 74 | 75 | func (d *AggregateDsl) ExtendedStats(field string) *AggregateDsl { 76 | d.Type = FieldAggregate{Field: field} 77 | d.TypeName = "extended_stats" 78 | return d 79 | } 80 | 81 | func (d *AggregateDsl) ValueCount(field string) *AggregateDsl { 82 | d.Type = FieldAggregate{Field: field} 83 | d.TypeName = "value_count" 84 | return d 85 | } 86 | 87 | func (d *AggregateDsl) Percentiles(field string) *AggregateDsl { 88 | d.Type = FieldAggregate{Field: field} 89 | d.TypeName = "percentiles" 90 | return d 91 | } 92 | 93 | type Cardinality struct { 94 | Field string `json:"field"` 95 | PrecisionThreshold float64 `json:"precision_threshold,omitempty"` 96 | Rehash bool `json:"rehash,omitempty"` 97 | } 98 | 99 | /** 100 | * Cardinality( 101 | * "field_name", 102 | * true, 103 | * 0, 104 | * ) 105 | */ 106 | func (d *AggregateDsl) Cardinality(field string, rehash bool, threshold int) *AggregateDsl { 107 | c := Cardinality{Field: field} 108 | 109 | // Only set if it's false, since the default is true 110 | if !rehash { 111 | c.Rehash = false 112 | } 113 | 114 | if threshold > 0 { 115 | c.PrecisionThreshold = float64(threshold) 116 | } 117 | d.Type = c 118 | d.TypeName = "cardinality" 119 | return d 120 | } 121 | 122 | func (d *AggregateDsl) Global() *AggregateDsl { 123 | d.Type = struct{}{} 124 | d.TypeName = "global" 125 | return d 126 | } 127 | 128 | func (d *AggregateDsl) Filter(filters ...interface{}) *AggregateDsl { 129 | 130 | if len(filters) == 0 { 131 | return d 132 | } 133 | 134 | if d.Filters == nil { 135 | d.Filters = NewFilterWrap() 136 | } 137 | 138 | d.Filters.addFilters(filters) 139 | return d 140 | } 141 | 142 | func (d *AggregateDsl) Missing(field string) *AggregateDsl { 143 | d.Type = FieldAggregate{Field: field} 144 | d.TypeName = "missing" 145 | return d 146 | } 147 | 148 | func (d *AggregateDsl) Terms(field string) *AggregateDsl { 149 | d.Type = FieldAggregate{Field: field} 150 | d.TypeName = "terms" 151 | return d 152 | } 153 | 154 | func (d *AggregateDsl) TermsWithSize(field string, size int) *AggregateDsl { 155 | d.Type = FieldAggregate{Field: field, Size: &size} 156 | d.TypeName = "terms" 157 | return d 158 | } 159 | 160 | func (d *AggregateDsl) SignificantTerms(field string) *AggregateDsl { 161 | d.Type = FieldAggregate{Field: field} 162 | d.TypeName = "significant_terms" 163 | return d 164 | } 165 | 166 | type Histogram struct { 167 | Field string `json:"field"` 168 | Interval float64 `json:"interval"` 169 | } 170 | 171 | func (d *AggregateDsl) Histogram(field string, interval int) *AggregateDsl { 172 | d.Type = Histogram{ 173 | Field: field, 174 | Interval: float64(interval), 175 | } 176 | d.TypeName = "histogram" 177 | return d 178 | } 179 | 180 | type DateHistogram struct { 181 | Field string `json:"field"` 182 | Interval string `json:"interval"` 183 | } 184 | 185 | func (d *AggregateDsl) DateHistogram(field, interval string) *AggregateDsl { 186 | d.Type = DateHistogram{ 187 | Field: field, 188 | Interval: interval, 189 | } 190 | d.TypeName = "date_histogram" 191 | return d 192 | } 193 | 194 | func (d *AggregateDsl) MarshalJSON() ([]byte, error) { 195 | return json.Marshal(d.toMap()) 196 | } 197 | 198 | func (d *AggregateDsl) toMap() map[string]interface{} { 199 | root := map[string]interface{}{} 200 | 201 | if d.Type != nil { 202 | root[d.TypeName] = d.Type 203 | } 204 | aggregates := d.aggregatesMap() 205 | 206 | if d.Filters != nil { 207 | root["filter"] = d.Filters 208 | } 209 | 210 | if len(aggregates) > 0 { 211 | root["aggregations"] = aggregates 212 | } 213 | return root 214 | 215 | } 216 | 217 | func (d *AggregateDsl) aggregatesMap() map[string]interface{} { 218 | root := map[string]interface{}{} 219 | 220 | if len(d.AggregatesVal) > 0 { 221 | for _, agg := range d.AggregatesVal { 222 | root[agg.Name] = agg.toMap() 223 | } 224 | } 225 | return root 226 | } 227 | -------------------------------------------------------------------------------- /lib/searchaggregate_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | // Test all aggregate types and nested aggregations 10 | func TestAggregateDsl(t *testing.T) { 11 | 12 | min := Aggregate("min_price").Min("price") 13 | max := Aggregate("max_price").Max("price") 14 | sum := Aggregate("sum_price").Sum("price") 15 | avg := Aggregate("avg_price").Avg("price") 16 | stats := Aggregate("stats_price").Stats("price") 17 | extendedStats := Aggregate("extended_stats_price").ExtendedStats("price") 18 | valueCount := Aggregate("value_count_price").ValueCount("price") 19 | percentiles := Aggregate("percentiles_price").Percentiles("price") 20 | cardinality := Aggregate("cardinality_price").Cardinality("price", true, 50) 21 | global := Aggregate("global").Global() 22 | missing := Aggregate("missing_price").Missing("price") 23 | terms := Aggregate("terms_price").Terms("price") 24 | termsSize := Aggregate("terms_price_size").TermsWithSize("price", 0) 25 | significantTerms := Aggregate("significant_terms_price").SignificantTerms("price") 26 | histogram := Aggregate("histogram_price").Histogram("price", 50) 27 | 28 | dateAgg := Aggregate("articles_over_time").DateHistogram("date", "month") 29 | dateAgg.Aggregates( 30 | min, 31 | max, 32 | sum, 33 | avg, 34 | stats, 35 | extendedStats, 36 | valueCount, 37 | percentiles, 38 | cardinality, 39 | global, 40 | missing, 41 | terms, 42 | termsSize, 43 | significantTerms, 44 | histogram, 45 | ) 46 | 47 | qry := Search("github").Aggregates(dateAgg) 48 | 49 | marshaled, err := json.MarshalIndent(qry.AggregatesVal, "", " ") 50 | if err != nil { 51 | t.Errorf("Failed to marshal AggregatesVal: %s", err.Error()) 52 | return 53 | } 54 | 55 | assertJsonMatch( 56 | t, 57 | marshaled, 58 | []byte(` 59 | { 60 | "articles_over_time": { 61 | "date_histogram" : { 62 | "field" : "date", 63 | "interval" : "month" 64 | }, 65 | "aggregations": { 66 | "min_price":{ 67 | "min": { "field": "price" } 68 | }, 69 | "max_price":{ 70 | "max": { "field": "price" } 71 | }, 72 | "sum_price":{ 73 | "sum": { "field": "price" } 74 | }, 75 | "avg_price": { 76 | "avg": { "field": "price" } 77 | }, 78 | "stats_price":{ 79 | "stats": { "field": "price" } 80 | }, 81 | "extended_stats_price":{ 82 | "extended_stats": { "field": "price" } 83 | }, 84 | "value_count_price":{ 85 | "value_count": { "field": "price" } 86 | }, 87 | "percentiles_price":{ 88 | "percentiles": { "field": "price" } 89 | }, 90 | "cardinality_price":{ 91 | "cardinality": { "field": "price", "precision_threshold": 50 } 92 | }, 93 | "global":{ 94 | "global": {} 95 | }, 96 | "missing_price":{ 97 | "missing": { "field": "price" } 98 | }, 99 | "terms_price":{ 100 | "terms": { "field": "price" } 101 | }, 102 | "terms_price_size":{ 103 | "terms": { "field": "price", "size": 0 } 104 | }, 105 | "significant_terms_price":{ 106 | "significant_terms": { "field": "price" } 107 | }, 108 | "histogram_price":{ 109 | "histogram": { "field": "price", "interval": 50 } 110 | } 111 | } 112 | } 113 | } 114 | `), 115 | ) 116 | 117 | } 118 | 119 | func TestAggregateFilter(t *testing.T) { 120 | 121 | avg := Aggregate("avg_price").Avg("price") 122 | 123 | dateAgg := Aggregate("in_stock_products").Filter( 124 | Filter().Range("stock", nil, 0, nil, nil, ""), 125 | ) 126 | 127 | dateAgg.Aggregates( 128 | avg, 129 | ) 130 | 131 | qry := Search("github").Aggregates(dateAgg) 132 | 133 | marshaled, err := json.MarshalIndent(qry.AggregatesVal, "", " ") 134 | if err != nil { 135 | t.Errorf("Failed to marshal AggregatesVal: %s", err.Error()) 136 | return 137 | } 138 | 139 | assertJsonMatch( 140 | t, 141 | marshaled, 142 | []byte(` 143 | { 144 | "in_stock_products" : { 145 | "filter" : { 146 | "range" : { "stock" : { "gt" : 0 } } 147 | }, 148 | "aggregations" : { 149 | "avg_price" : { "avg" : { "field" : "price" } } 150 | } 151 | } 152 | } 153 | `), 154 | ) 155 | } 156 | 157 | func assertJsonMatch(t *testing.T, match, expected []byte) { 158 | var m interface{} 159 | var e interface{} 160 | 161 | err := json.Unmarshal(expected, &e) 162 | if err != nil { 163 | t.Errorf("Failed to unmarshal expectation: %s", err.Error()) 164 | return 165 | } 166 | err = json.Unmarshal(match, &m) 167 | if err != nil { 168 | t.Errorf("Failed to unmarshal match: %s", err.Error()) 169 | return 170 | } 171 | 172 | if !reflect.DeepEqual(m, e) { 173 | t.Errorf("Expected %s but got %s", string(expected), string(match)) 174 | return 175 | } 176 | 177 | } 178 | -------------------------------------------------------------------------------- /lib/searchdsl.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | type SearchRequest struct { 15 | From int `json:"from,omitempty"` 16 | Size int `json:"size,omitempty"` 17 | Query OneTermQuery `json:"query,omitempty"` 18 | 19 | Filter struct { 20 | Term Term `json:"term"` 21 | } `json:"filter,omitempty"` 22 | } 23 | 24 | type Facets struct { 25 | Tag struct { 26 | Terms string `json:"terms"` 27 | } `json:"tag"` 28 | } 29 | -------------------------------------------------------------------------------- /lib/searchfacet.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | 17 | u "github.com/araddon/gou" 18 | ) 19 | 20 | var ( 21 | _ = u.DEBUG 22 | ) 23 | 24 | /* 25 | "facets": { 26 | "terms": { 27 | "terms": { 28 | "field": [ 29 | "@fields.category" 30 | ], 31 | "size": 25 32 | } 33 | } 34 | } 35 | 36 | 37 | "facets": { 38 | "actors": { "terms": {"field": ["actor"],"size": "10" }} 39 | , "langauge": { "terms": {"field": ["repository.language"],"size": "10" }} 40 | } 41 | 42 | */ 43 | func Facet() *FacetDsl { 44 | return &FacetDsl{} 45 | } 46 | 47 | func FacetRange(field string) *RangeDsl { 48 | out := &RangeDsl{&RangeDef{}, nil} 49 | out.RangeDef.Field = field 50 | return out 51 | } 52 | 53 | type FacetDsl struct { 54 | size string 55 | Terms map[string]*Term `json:"terms,omitempty"` 56 | Ranges map[string]*RangeDsl `json:"terms,omitempty"` 57 | } 58 | 59 | type RangeDsl struct { 60 | RangeDef *RangeDef `json:"range,omitempty"` 61 | FilterVal *FilterWrap `json:"facet_filter,omitempty"` 62 | } 63 | 64 | type RangeDef struct { 65 | Field string `json:"field,omitempty"` 66 | Values []*RangeVal `json:"ranges,omitempty"` 67 | } 68 | 69 | type RangeVal struct { 70 | From string `json:"from,omitempty"` 71 | To string `json:"to,omitempty"` 72 | } 73 | 74 | func (m *RangeDsl) Range(from, to string) *RangeDsl { 75 | if len(m.RangeDef.Values) == 0 { 76 | m.RangeDef.Values = make([]*RangeVal, 0) 77 | } 78 | 79 | m.RangeDef.Values = append(m.RangeDef.Values, &RangeVal{From: from, To: to}) 80 | return m 81 | } 82 | 83 | func (s *RangeDsl) Filter(fl ...interface{}) *RangeDsl { 84 | if s.FilterVal == nil { 85 | s.FilterVal = NewFilterWrap() 86 | } 87 | 88 | s.FilterVal.addFilters(fl) 89 | return s 90 | } 91 | 92 | func (m *FacetDsl) Size(size string) *FacetDsl { 93 | m.size = size 94 | return m 95 | } 96 | 97 | func (m *FacetDsl) Fields(fields ...string) *FacetDsl { 98 | if len(fields) < 1 { 99 | return m 100 | } 101 | if len(m.Terms) == 0 { 102 | m.Terms = make(map[string]*Term) 103 | } 104 | m.Terms[fields[0]] = &Term{Terms{Fields: fields}, nil} 105 | return m 106 | } 107 | 108 | func (m *FacetDsl) Regex(field, match string) *FacetDsl { 109 | if len(m.Terms) == 0 { 110 | m.Terms = make(map[string]*Term) 111 | } 112 | m.Terms[field] = &Term{Terms{Fields: []string{field}, Regex: match}, nil} 113 | return m 114 | } 115 | 116 | func (m *FacetDsl) Term(t *Term) *FacetDsl { 117 | if len(m.Terms) == 0 { 118 | m.Terms = make(map[string]*Term) 119 | } 120 | m.Terms[t.Terms.Fields[0]] = t 121 | return m 122 | } 123 | 124 | func (m *FacetDsl) Range(r *RangeDsl) *FacetDsl { 125 | if len(m.Ranges) == 0 { 126 | m.Ranges = make(map[string]*RangeDsl) 127 | } 128 | m.Ranges[r.RangeDef.Field] = r 129 | return m 130 | } 131 | 132 | func (m *FacetDsl) MarshalJSON() ([]byte, error) { 133 | data := map[string]interface{}{} 134 | for key, t := range m.Terms { 135 | t.Terms.Size = m.size 136 | data[key] = t 137 | } 138 | for key, r := range m.Ranges { 139 | data[key] = r 140 | } 141 | return json.Marshal(&data) 142 | } 143 | -------------------------------------------------------------------------------- /lib/searchfacet_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "github.com/araddon/gou" 16 | . "github.com/smartystreets/goconvey/convey" 17 | "testing" 18 | ) 19 | 20 | func TestFacetRegex(t *testing.T) { 21 | 22 | c := NewTestConn() 23 | PopulateTestDB(t, c) 24 | defer TearDownTestDB(c) 25 | 26 | Convey("Facted regex query", t, func() { 27 | 28 | // This is a possible solution for auto-complete 29 | out, err := Search("oilers").Size("0").Facet( 30 | Facet().Regex("name", "[jk].*").Size("8"), 31 | ).Result(c) 32 | So(err, ShouldBeNil) 33 | So(out, ShouldNotBeNil) 34 | 35 | // Debug(string(out.Facets)) 36 | fh := gou.NewJsonHelper([]byte(out.Facets)) 37 | facets := fh.Helpers("/name/terms") 38 | So(err, ShouldBeNil) 39 | So(facets, ShouldNotBeNil) 40 | So(len(facets), ShouldEqual, 4) 41 | }) 42 | } 43 | -------------------------------------------------------------------------------- /lib/searchhighlight.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import "encoding/json" 4 | 5 | func NewHighlight() *HighlightDsl { 6 | return &HighlightDsl{} 7 | } 8 | 9 | type HighlightDsl struct { 10 | Settings *HighlightEmbed `-` 11 | TagSchema string `json:"tag_schema,omitempty"` 12 | Fields map[string]HighlightEmbed `json:"fields,omitempty"` 13 | } 14 | 15 | func NewHighlightOpts() *HighlightEmbed { 16 | return &HighlightEmbed{} 17 | } 18 | 19 | type HighlightEmbed struct { 20 | BoundaryCharsVal string `json:"boundary_chars,omitempty"` 21 | BoundaryMaxScanVal int `json:"boundary_max_scan,omitempty"` 22 | PreTags []string `json:"pre_tags,omitempty"` 23 | PostTags []string `json:"post_tags,omitempty"` 24 | FragmentSizeVal int `json:"fragment_size,omitempty"` 25 | NumOfFragmentsVal int `json:"number_of_fragments,omitempty"` 26 | HighlightQuery *QueryDsl `json:"highlight_query,omitempty"` 27 | MatchedFieldsVal []string `json:"matched_fields,omitempty"` 28 | OrderVal string `json:"order,omitempty"` 29 | TypeVal string `json:"type,omitempty"` 30 | } 31 | 32 | // Custom marshalling 33 | func (t *HighlightDsl) MarshalJSON() ([]byte, error) { 34 | m := make(map[string]interface{}) 35 | 36 | if t.Fields != nil { 37 | m["fields"] = t.Fields 38 | } 39 | 40 | if t.TagSchema != "" { 41 | m["tag_schema"] = t.TagSchema 42 | } 43 | 44 | if t.Settings == nil { 45 | return json.Marshal(m) 46 | } 47 | 48 | //This is terrible :(, could use structs package to avoid extra serialization. 49 | embed, err := json.Marshal(t.Settings) 50 | if err == nil { 51 | err = json.Unmarshal(embed, &m) 52 | } 53 | 54 | if err == nil { 55 | return json.Marshal(m) 56 | } 57 | 58 | return nil, err 59 | } 60 | 61 | func (h *HighlightDsl) AddField(name string, settings *HighlightEmbed) *HighlightDsl { 62 | if h.Fields == nil { 63 | h.Fields = make(map[string]HighlightEmbed) 64 | } 65 | 66 | if settings != nil { 67 | h.Fields[name] = *settings 68 | } else { 69 | h.Fields[name] = HighlightEmbed{} 70 | } 71 | 72 | return h 73 | } 74 | 75 | func (h *HighlightDsl) Schema(schema string) *HighlightDsl { 76 | h.TagSchema = schema 77 | return h 78 | } 79 | 80 | func (h *HighlightDsl) SetOptions(options *HighlightEmbed) *HighlightDsl { 81 | h.Settings = options 82 | return h 83 | } 84 | 85 | func (o *HighlightEmbed) BoundaryChars(chars string) *HighlightEmbed { 86 | o.BoundaryCharsVal = chars 87 | return o 88 | } 89 | 90 | func (o *HighlightEmbed) BoundaryMaxScan(max int) *HighlightEmbed { 91 | o.BoundaryMaxScanVal = max 92 | return o 93 | } 94 | 95 | func (he *HighlightEmbed) FragSize(size int) *HighlightEmbed { 96 | he.FragmentSizeVal = size 97 | return he 98 | } 99 | 100 | func (he *HighlightEmbed) NumFrags(numFrags int) *HighlightEmbed { 101 | he.NumOfFragmentsVal = numFrags 102 | return he 103 | } 104 | 105 | func (he *HighlightEmbed) MatchedFields(fields ...string) *HighlightEmbed { 106 | he.MatchedFieldsVal = fields 107 | return he 108 | } 109 | 110 | func (he *HighlightEmbed) Order(order string) *HighlightEmbed { 111 | he.OrderVal = order 112 | return he 113 | } 114 | 115 | func (he *HighlightEmbed) Tags(pre string, post string) *HighlightEmbed { 116 | if he == nil { 117 | he = &HighlightEmbed{} 118 | } 119 | 120 | if he.PreTags == nil { 121 | he.PreTags = []string{pre} 122 | } else { 123 | he.PreTags = append(he.PreTags, pre) 124 | } 125 | 126 | if he.PostTags == nil { 127 | he.PostTags = []string{post} 128 | } else { 129 | he.PostTags = append(he.PostTags, post) 130 | } 131 | 132 | return he 133 | } 134 | 135 | func (he *HighlightEmbed) Type(highlightType string) *HighlightEmbed { 136 | he.TypeVal = highlightType 137 | return he 138 | } 139 | -------------------------------------------------------------------------------- /lib/searchhighlight_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "github.com/bmizerany/assert" 5 | "testing" 6 | ) 7 | 8 | func TestEmbedDsl(t *testing.T) { 9 | highlight := NewHighlight().SetOptions(NewHighlightOpts(). 10 | Tags("
", "
"). 11 | BoundaryChars("asdf").BoundaryMaxScan(100). 12 | FragSize(10).NumFrags(50). 13 | Order("order").Type("fdsa"). 14 | MatchedFields("1", "2")) 15 | 16 | actual, err := GetJson(highlight) 17 | 18 | assert.Equal(t, nil, err) 19 | assert.Equal(t, "
", actual["pre_tags"].([]interface{})[0]) 20 | assert.Equal(t, "
", actual["post_tags"].([]interface{})[0]) 21 | assert.Equal(t, "asdf", actual["boundary_chars"]) 22 | assert.Equal(t, float64(100), actual["boundary_max_scan"]) 23 | assert.Equal(t, float64(10), actual["fragment_size"]) 24 | assert.Equal(t, float64(50), actual["number_of_fragments"]) 25 | assert.Equal(t, "1", actual["matched_fields"].([]interface{})[0]) 26 | assert.Equal(t, "2", actual["matched_fields"].([]interface{})[1]) 27 | assert.Equal(t, "order", actual["order"]) 28 | assert.Equal(t, "fdsa", actual["type"]) 29 | } 30 | 31 | func TestFieldDsl(t *testing.T) { 32 | highlight := NewHighlight().AddField("whatever", NewHighlightOpts(). 33 | Tags("
", "
"). 34 | BoundaryChars("asdf").BoundaryMaxScan(100). 35 | FragSize(10).NumFrags(50). 36 | Order("order").Type("fdsa"). 37 | MatchedFields("1", "2")) 38 | 39 | result, err := GetJson(highlight) 40 | actual := result["fields"].(map[string]interface{})["whatever"].(map[string]interface{}) 41 | 42 | assert.Equal(t, nil, err) 43 | assert.Equal(t, "
", actual["pre_tags"].([]interface{})[0]) 44 | assert.Equal(t, "
", actual["post_tags"].([]interface{})[0]) 45 | assert.Equal(t, "asdf", actual["boundary_chars"]) 46 | assert.Equal(t, float64(100), actual["boundary_max_scan"]) 47 | assert.Equal(t, float64(10), actual["fragment_size"]) 48 | assert.Equal(t, float64(50), actual["number_of_fragments"]) 49 | assert.Equal(t, "1", actual["matched_fields"].([]interface{})[0]) 50 | assert.Equal(t, "2", actual["matched_fields"].([]interface{})[1]) 51 | assert.Equal(t, "order", actual["order"]) 52 | assert.Equal(t, "fdsa", actual["type"]) 53 | } 54 | 55 | func TestEmbedAndFieldDsl(t *testing.T) { 56 | highlight := NewHighlight(). 57 | SetOptions(NewHighlightOpts().Tags("
", "
")). 58 | AddField("afield", NewHighlightOpts().Type("something")) 59 | 60 | actual, err := GetJson(highlight) 61 | actualField := actual["fields"].(map[string]interface{})["afield"].(map[string]interface{}) 62 | 63 | assert.Equal(t, nil, err) 64 | assert.Equal(t, "
", actual["pre_tags"].([]interface{})[0]) 65 | assert.Equal(t, "
", actual["post_tags"].([]interface{})[0]) 66 | assert.Equal(t, "something", actualField["type"]) 67 | } 68 | -------------------------------------------------------------------------------- /lib/searchreadme: -------------------------------------------------------------------------------- 1 | 2 | 3 | To run tests on this, you must first have run/imported data inside of *core* 4 | 5 | -------------------------------------------------------------------------------- /lib/searchsearch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | u "github.com/araddon/gou" 18 | "strconv" 19 | "strings" 20 | ) 21 | 22 | var ( 23 | _ = u.DEBUG 24 | ) 25 | 26 | // Search is the entry point to the SearchDsl, it is a chainable set of utilities 27 | // to create searches. 28 | // 29 | // params 30 | // @index = elasticsearch index to search 31 | // 32 | // out, err := Search("github").Type("Issues").Pretty().Query( 33 | // Query().Range( 34 | // Range().Field("created_at").From("2012-12-10T15:00:00-08:00").To("2012-12-10T15:10:00-08:00"), 35 | // ).Search("add"), 36 | // ).Result() 37 | func Search(index string) *SearchDsl { 38 | return &SearchDsl{Index: index, args: map[string]interface{}{}} 39 | } 40 | 41 | type SearchDsl struct { 42 | args map[string]interface{} 43 | types []string 44 | FromVal int `json:"from,omitempty"` 45 | SizeVal int `json:"size,omitempty"` 46 | Index string `json:"-"` 47 | FacetVal *FacetDsl `json:"facets,omitempty"` 48 | QueryVal *QueryDsl `json:"query,omitempty"` 49 | SortBody []*SortDsl `json:"sort,omitempty"` 50 | FilterVal *FilterOp `json:"filter,omitempty"` 51 | AggregatesVal map[string]*AggregateDsl `json:"aggregations,omitempty"` 52 | HighlightVal *HighlightDsl `json:"highlight,omitempty"` 53 | } 54 | 55 | func (s *SearchDsl) Bytes(conn *Conn) ([]byte, error) { 56 | return conn.DoCommand("POST", s.url(), s.args, s) 57 | } 58 | 59 | func (s *SearchDsl) Result(conn *Conn) (*SearchResult, error) { 60 | var retval SearchResult 61 | body, err := s.Bytes(conn) 62 | retval.RawJSON = body 63 | if err != nil { 64 | u.Errorf("%v", err) 65 | return nil, err 66 | } 67 | jsonErr := json.Unmarshal(body, &retval) 68 | if jsonErr != nil { 69 | u.Errorf("%v \n\t%s", jsonErr, string(body)) 70 | } 71 | return &retval, jsonErr 72 | } 73 | 74 | func (s *SearchDsl) url() string { 75 | url := fmt.Sprintf("/%s%s/_search", s.Index, s.getType()) 76 | return url 77 | } 78 | 79 | func (s *SearchDsl) Pretty() *SearchDsl { 80 | s.args["pretty"] = "1" 81 | return s 82 | } 83 | 84 | // Type is the elasticsearch *Type* within a specific index 85 | func (s *SearchDsl) Type(indexType string) *SearchDsl { 86 | if len(s.types) == 0 { 87 | s.types = make([]string, 0) 88 | } 89 | s.types = append(s.types, indexType) 90 | return s 91 | } 92 | 93 | func (s *SearchDsl) getType() string { 94 | if len(s.types) > 0 { 95 | return "/" + strings.Join(s.types, ",") 96 | } 97 | return "" 98 | } 99 | 100 | func (s *SearchDsl) From(from string) *SearchDsl { 101 | s.args["from"] = from 102 | return s 103 | } 104 | 105 | // Search is a simple interface to search, doesn't have the power of query 106 | // but uses a simple query_string search 107 | func (s *SearchDsl) Search(srch string) *SearchDsl { 108 | s.QueryVal = Query().Search(srch) 109 | return s 110 | } 111 | 112 | func (s *SearchDsl) Size(size string) *SearchDsl { 113 | s.args["size"] = size 114 | return s 115 | } 116 | 117 | func (s *SearchDsl) Fields(fields ...string) *SearchDsl { 118 | s.args["fields"] = strings.Join(fields, ",") 119 | return s 120 | } 121 | 122 | func (s *SearchDsl) Source(returnSource bool) *SearchDsl { 123 | s.args["_source"] = strconv.FormatBool(returnSource) 124 | return s 125 | } 126 | 127 | func (s *SearchDsl) SourceFields(fields ...string) *SearchDsl { 128 | s.args["_source"] = fields 129 | return s 130 | } 131 | 132 | // Facet passes a Query expression to this search 133 | // 134 | // qry := Search("github").Size("0").Facet( 135 | // Facet().Regex("repository.name", "no.*").Size("8"), 136 | // ) 137 | // 138 | // qry := Search("github").Pretty().Facet( 139 | // Facet().Fields("type").Size("25"), 140 | // ) 141 | func (s *SearchDsl) Facet(f *FacetDsl) *SearchDsl { 142 | s.FacetVal = f 143 | return s 144 | } 145 | 146 | func (s *SearchDsl) Aggregates(aggs ...*AggregateDsl) *SearchDsl { 147 | if len(aggs) < 1 { 148 | return s 149 | } 150 | if len(s.AggregatesVal) == 0 { 151 | s.AggregatesVal = make(map[string]*AggregateDsl) 152 | } 153 | 154 | for _, agg := range aggs { 155 | s.AggregatesVal[agg.Name] = agg 156 | } 157 | return s 158 | } 159 | 160 | func (s *SearchDsl) Query(q *QueryDsl) *SearchDsl { 161 | s.QueryVal = q 162 | return s 163 | } 164 | 165 | // Filter adds a Filter Clause with optional Boolean Clause. This accepts n number of 166 | // filter clauses. If more than one, and missing Boolean Clause it assumes "and" 167 | // 168 | // qry := Search("github").Filter( 169 | // Filter().Exists("repository.name"), 170 | // ) 171 | // 172 | // qry := Search("github").Filter( 173 | // "or", 174 | // Filter().Exists("repository.name"), 175 | // Filter().Terms("actor_attributes.location", "portland"), 176 | // ) 177 | // 178 | // qry := Search("github").Filter( 179 | // Filter().Exists("repository.name"), 180 | // Filter().Terms("repository.has_wiki", true) 181 | // ) 182 | 183 | func (s *SearchDsl) Filter(fl *FilterOp) *SearchDsl { 184 | s.FilterVal = fl 185 | return s 186 | } 187 | 188 | func (s *SearchDsl) Sort(sort ...*SortDsl) *SearchDsl { 189 | if s.SortBody == nil { 190 | s.SortBody = make([]*SortDsl, 0) 191 | } 192 | s.SortBody = append(s.SortBody, sort...) 193 | return s 194 | } 195 | 196 | func (s *SearchDsl) Scroll(duration string) *SearchDsl { 197 | s.args["scroll"] = duration 198 | return s 199 | } 200 | 201 | func (s *SearchDsl) SearchType(searchType string) *SearchDsl { 202 | s.args["search_type"] = searchType 203 | return s 204 | } 205 | 206 | func (s *SearchDsl) Highlight(highlight *HighlightDsl) *SearchDsl { 207 | s.HighlightVal = highlight 208 | return s 209 | } 210 | -------------------------------------------------------------------------------- /lib/searchsort.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | ) 18 | 19 | // SortDsl accepts any number of Sort commands 20 | // 21 | // Query().Sort( 22 | // Sort("last_name").Desc(), 23 | // Sort("age"), 24 | // ) 25 | func Sort(field string) *SortDsl { 26 | return &SortDsl{Name: field} 27 | } 28 | 29 | func GeoDistanceSort(field interface{}) *SortDsl { 30 | return &SortDsl{GeoDistance: field} 31 | } 32 | 33 | type SortBody []interface{} 34 | type SortDsl struct { 35 | Name string 36 | IsDesc bool 37 | GeoDistance interface{} 38 | } 39 | 40 | func (s *SortDsl) Desc() *SortDsl { 41 | s.IsDesc = true 42 | return s 43 | } 44 | func (s *SortDsl) Asc() *SortDsl { 45 | s.IsDesc = false 46 | return s 47 | } 48 | 49 | func (s *SortDsl) MarshalJSON() ([]byte, error) { 50 | if s.GeoDistance != nil { 51 | return json.Marshal(map[string]interface{}{"_geo_distance": s.GeoDistance}) 52 | } 53 | if s.IsDesc { 54 | return json.Marshal(map[string]string{s.Name: "desc"}) 55 | } 56 | if s.Name == "_score" { 57 | return []byte(`"_score"`), nil 58 | } 59 | return []byte(fmt.Sprintf(`"%s"`, s.Name)), nil // "user" assuming default = asc? 60 | } 61 | -------------------------------------------------------------------------------- /lib/setup_test.go: -------------------------------------------------------------------------------- 1 | package elastigo 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | /* 9 | // elastigo Conn adapter to avoid a circular dependency 10 | type conn interface { 11 | CreateIndex(name string) (interface{}, error) 12 | DeleteIndex(name string) (interface{}, error) 13 | 14 | Index(index string, _type string, id string, args map[string]interface{}, data interface{}) (interface{}, error) 15 | } 16 | */ 17 | 18 | func newIndexWorker(c *Conn, t *testing.T) func(interface{}) { 19 | 20 | return func(d interface{}) { 21 | _, err := c.Index("oilers", "heyday", "", nil, d) 22 | if err != nil { 23 | t.Fatalf("Index failed: %s", err) 24 | } 25 | } 26 | } 27 | 28 | func PopulateTestDB(t *testing.T, c *Conn) { 29 | 30 | // it is not technically necessary to create an index here 31 | _, err := c.CreateIndex("oilers") 32 | if err != nil { 33 | t.Fatal("Error in CreateIndex", err) 34 | } 35 | 36 | // set the mapping for dob to be a date so it can be used for range searches 37 | _, err = c.DoCommand("PUT", "/oilers/heyday/_mapping?ignore_conflicts", nil, 38 | string(`{"heyday": {"properties": { 39 | "dob": {"type": "date", "format": "basic_date"}, 40 | "pos": {"type": "string", "index": "not_analyzed"}, 41 | "teams": {"type": "string", "index": "not_analyzed"} 42 | }}}`)) 43 | if err != nil { 44 | t.Fatal("Error setting dob mapping", err) 45 | } 46 | 47 | idx := newIndexWorker(c, t) 48 | 49 | idx(`{"name": "Mark Messier", "jersey": 11, "pos": "LW", "goals": 37, "PIM": 165, 50 | "dob": "19610118", "teams": ["EDM", "NYR", "VAN"]}`) 51 | idx(`{"name": "Wayne Gretzky", "jersey": 99, "pos": "C", "goals": 87, 52 | "dob": "19610126", "teams": ["EDM", "NYR", "STL"]}`) 53 | idx(`{"name": "Paul Coffey", "jersey": 7, "pos": "D", "goals": 40, 54 | "dob": "19610601", "teams": ["EDM", "DET"]}`) 55 | idx(`{"name": "Jari Kurri", "jersey": 17, "pos": "RW", "goals": 52, 56 | "dob": "19600518", "teams": ["EDM", "VAN"]}`) 57 | idx(`{"name": "Glenn Anderson", "jersey": 9, "pos": "RW", "goals": 54, 58 | "dob": "19601002", "teams": ["EDM", "NYR", "TOR", "STL"]}`) 59 | idx(`{"name": "Ken Linseman", "jersey": 13, "pos": "C", "goals": 18, 60 | "dob": "19580811", "teams": ["EDM", "TOR"]}`) 61 | idx(`{"name": "Pat Hughes", "jersey": 16, "pos": "RW", "goals": 27, 62 | "dob": "19550325", "teams": ["EDM", "MTL", "PIT"]}`) 63 | idx(`{"name": "Dave Hunter", "jersey": 12, "pos": "LW", "goals": 22, 64 | "dob": "19580101", "teams": ["EDM", "PIT"]}`) 65 | idx(`{"name": "Kevin Lowe", "jersey": 4, "pos": "D", "goals": 4, 66 | "dob": "19590415", "teams": ["EDM", "NYR"]}`) 67 | idx(`{"name": "Charlie Huddy", "jersey": 22, "pos": "D", "goals": 8, 68 | "dob": "19590602", "teams": ["EDM", "BUF", "STL"]}`) 69 | idx(`{"name": "Randy Gregg", "jersey": 21, "pos": "D", "goals": 13, 70 | "dob": "19560219", "teams": ["EDM", "VAN"]}`) 71 | idx(`{"name": "Dave Semenko", "jersey": 27, "pos": "LW", "goals": 4, "PIM": 118, 72 | "dob": "19570712", "teams": ["EDM"]}`) 73 | idx(`{"name": "Grant Fuhr", "jersey": 31, "pos": "G", "GAA": 3.91, 74 | "dob": "19620928", "teams": ["EDM", "TOR", "BUF", "STL"]}`) 75 | idx(`{"name": "Andy Moog", "jersey": 35, "pos": "G", "GAA": 3.77, 76 | "dob": "19600218", "teams": ["EDM", "BOS", "DAL", "MTL"]}`) 77 | 78 | // HACK to let the ES magic happen 79 | time.Sleep(time.Second) 80 | } 81 | 82 | func TearDownTestDB(c *Conn) { 83 | c.DeleteIndex("oilers") 84 | } 85 | -------------------------------------------------------------------------------- /lib/shared.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | type OneTermQuery struct { 15 | Query struct { 16 | Term string `json:"term"` 17 | } `json:"query"` 18 | } 19 | -------------------------------------------------------------------------------- /lib/shared_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "flag" 17 | "log" 18 | ) 19 | 20 | var ( 21 | _ = log.Ldate 22 | eshost *string = flag.String("host", "localhost", "Elasticsearch Server Host Address") 23 | logLevel *string = flag.String("logging", "info", "Which log level: [debug,info,warn,error,fatal]") 24 | ) 25 | 26 | func GetJson(input interface{}) (map[string]interface{}, error) { 27 | var result map[string]interface{} 28 | bytes, err := json.Marshal(input) 29 | 30 | if err == nil { 31 | err = json.Unmarshal(bytes, &result) 32 | } 33 | 34 | return result, err 35 | } 36 | 37 | func HasKey(input map[string]interface{}, key string) bool { 38 | if _, ok := input[key]; ok { 39 | return true 40 | } 41 | 42 | return false 43 | } 44 | -------------------------------------------------------------------------------- /lib/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Niels Freier 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package elastigo 13 | 14 | import ( 15 | "encoding/json" 16 | "fmt" 17 | "time" 18 | ) 19 | 20 | type GetSnapshotsResponse struct { 21 | Snapshots []struct { 22 | Snapshot string `json:"snapshot"` 23 | Indices []string `json:"indices"` 24 | State string `json:"state"` 25 | StartTime time.Time `json:"start_time"` 26 | EndTime time.Time `json:"end_time"` 27 | } `json:"snapshots"` 28 | } 29 | 30 | // CreateSnapshotRepository creates a new snapshot repository on the cluster 31 | // http://www.elastic.co/guide/en/elasticsearch/reference/1.3/modules-snapshots.html 32 | func (c *Conn) CreateSnapshotRepository(name string, args map[string]interface{}, settings interface{}) (BaseResponse, error) { 33 | var url string 34 | var retval BaseResponse 35 | url = fmt.Sprintf("/_snapshot/%s", name) 36 | body, err := c.DoCommand("POST", url, args, settings) 37 | if err != nil { 38 | return retval, err 39 | } 40 | if err == nil { 41 | jsonErr := json.Unmarshal(body, &retval) 42 | if jsonErr != nil { 43 | return retval, jsonErr 44 | } 45 | } 46 | 47 | return retval, nil 48 | 49 | } 50 | 51 | // TakeSnapshot takes a snapshot of the current state of the cluster with a specific name and for a existing repositoriy 52 | // http://www.elastic.co/guide/en/elasticsearch/reference/1.3/modules-snapshots.html 53 | func (c *Conn) TakeSnapshot(repository, name string, args map[string]interface{}, query interface{}) (BaseResponse, error) { 54 | var url string 55 | var retval BaseResponse 56 | url = fmt.Sprintf("/_snapshot/%s/%s", repository, name) 57 | body, err := c.DoCommand("PUT", url, args, query) 58 | if err != nil { 59 | return retval, err 60 | } 61 | if err == nil { 62 | jsonErr := json.Unmarshal(body, &retval) 63 | if jsonErr != nil { 64 | return retval, jsonErr 65 | } 66 | } 67 | 68 | return retval, nil 69 | } 70 | 71 | // RestoreSnapshot restores a snapshot of the current state of the cluster with a specific name and for a existing repositoriy 72 | // http://www.elastic.co/guide/en/elasticsearch/reference/1.3/modules-snapshots.html 73 | func (c *Conn) RestoreSnapshot(repository, name string, args map[string]interface{}, query interface{}) (BaseResponse, error) { 74 | var url string 75 | var retval BaseResponse 76 | url = fmt.Sprintf("/_snapshot/%s/%s/_restore", repository, name) 77 | body, err := c.DoCommand("POST", url, args, query) 78 | if err != nil { 79 | return retval, err 80 | } 81 | if err == nil { 82 | fmt.Println(string(body)) 83 | jsonErr := json.Unmarshal(body, &retval) 84 | if jsonErr != nil { 85 | return retval, jsonErr 86 | } 87 | } 88 | 89 | return retval, nil 90 | } 91 | 92 | // GetSnapshots returns all snapshot of the specified name for a specific repository 93 | // http://www.elastic.co/guide/en/elasticsearch/reference/1.3/modules-snapshots.html 94 | func (c *Conn) GetSnapshotByName(repository, name string, args map[string]interface{}) (GetSnapshotsResponse, error) { 95 | return c.getSnapshots(repository, name, args) 96 | } 97 | 98 | // GetSnapshots returns all snapshot for a specific repository 99 | // http://www.elastic.co/guide/en/elasticsearch/reference/1.3/modules-snapshots.html 100 | func (c *Conn) GetSnapshots(repository string, args map[string]interface{}) (GetSnapshotsResponse, error) { 101 | return c.getSnapshots(repository, "_all", args) 102 | } 103 | 104 | func (c *Conn) getSnapshots(repository, name string, args map[string]interface{}) (GetSnapshotsResponse, error) { 105 | var url string 106 | var retval GetSnapshotsResponse 107 | url = fmt.Sprintf("/_snapshot/%s/%s", repository, name) 108 | body, err := c.DoCommand("GET", url, args, nil) 109 | if err != nil { 110 | return retval, err 111 | } 112 | if err == nil { 113 | jsonErr := json.Unmarshal(body, &retval) 114 | if jsonErr != nil { 115 | return retval, jsonErr 116 | } 117 | } 118 | 119 | return retval, nil 120 | } 121 | -------------------------------------------------------------------------------- /tutorial/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Tutorials 4 | ======================================= 5 | 6 | To run these tutorials:: 7 | 8 | 9 | go run start_1.go 10 | 11 | # etc -------------------------------------------------------------------------------- /tutorial/start_1.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 Matthew Baird 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // http://www.apache.org/licenses/LICENSE-2.0 6 | // Unless required by applicable law or agreed to in writing, software 7 | // distributed under the License is distributed on an "AS IS" BASIS, 8 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 9 | // See the License for the specific language governing permissions and 10 | // limitations under the License. 11 | 12 | package main 13 | 14 | import ( 15 | "flag" 16 | "fmt" 17 | elastigo "github.com/mattbaird/elastigo/lib" 18 | "log" 19 | "os" 20 | ) 21 | 22 | var ( 23 | host *string = flag.String("host", "localhost", "Elasticsearch Host") 24 | ) 25 | 26 | func main() { 27 | c := elastigo.NewConn() 28 | log.SetFlags(log.LstdFlags) 29 | flag.Parse() 30 | 31 | // Trace all requests 32 | c.RequestTracer = func(method, url, body string) { 33 | log.Printf("Requesting %s %s", method, url) 34 | log.Printf("Request body: %s", body) 35 | } 36 | 37 | fmt.Println("host = ", *host) 38 | // Set the Elasticsearch Host to Connect to 39 | c.Domain = *host 40 | 41 | // Index a document 42 | _, err := c.Index("testindex", "user", "docid_1", nil, `{"name":"bob"}`) 43 | exitIfErr(err) 44 | 45 | // Index a doc using a map of values 46 | _, err = c.Index("testindex", "user", "docid_2", nil, map[string]string{"name": "venkatesh"}) 47 | exitIfErr(err) 48 | 49 | // Index a doc using Structs 50 | _, err = c.Index("testindex", "user", "docid_3", nil, MyUser{"wanda", 22}) 51 | exitIfErr(err) 52 | 53 | // Search Using Raw json String 54 | searchJson := `{ 55 | "query" : { 56 | "term" : { "Name" : "wanda" } 57 | } 58 | }` 59 | out, err := c.Search("testindex", "user", nil, searchJson) 60 | if len(out.Hits.Hits) == 1 { 61 | fmt.Println("%v", out.Hits.Hits[0].Source) 62 | } 63 | exitIfErr(err) 64 | 65 | } 66 | func exitIfErr(err error) { 67 | if err != nil { 68 | fmt.Fprintf(os.Stderr, "Error: %s\n", err.Error()) 69 | os.Exit(1) 70 | } 71 | } 72 | 73 | type MyUser struct { 74 | Name string 75 | Age int 76 | } 77 | --------------------------------------------------------------------------------