├── .gitignore ├── .travis.yml ├── Dockerfile ├── README.md ├── config ├── config.exs └── test.exs ├── docker-compose.yml ├── lib ├── elastic_sync.ex ├── elastic_sync │ ├── index.ex │ ├── index │ │ └── http.ex │ ├── reindex.ex │ ├── repo.ex │ └── sync_repo.ex └── mix │ └── tasks │ └── elastic_sync │ └── reindex.ex ├── mix.exs ├── mix.lock ├── priv └── test_repo │ └── migrations │ └── 1_migrate_all.exs └── test ├── dummy.ex ├── elastic_sync ├── index │ └── http_test.exs ├── index_test.exs ├── repo_test.exs └── sync_repo_test.exs ├── elastic_sync_test.exs ├── mix └── tasks │ └── elastic_sync │ └── reindex_test.exs └── test_helper.exs /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc 12 | 13 | # If the VM crashes, it generates a dump, let's ignore it too. 14 | erl_crash.dump 15 | 16 | # Also ignore archive artifacts (built via "mix archive.build"). 17 | *.ez 18 | 19 | /cover 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | services: 4 | - docker 5 | 6 | before_install: 7 | - sudo apt-get update 8 | - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce 9 | - sudo rm /usr/local/bin/docker-compose 10 | - curl -L https://github.com/docker/compose/releases/download/1.12.0/docker-compose-`uname -s`-`uname -m` > docker-compose 11 | - chmod +x docker-compose 12 | - sudo mv docker-compose /usr/local/bin 13 | 14 | env: 15 | global: 16 | - MIX_ENV=test 17 | 18 | matrix: 19 | - ES_VERSION=1.7.6 20 | - ES_VERSION=2.4.3 21 | - ES_VERSION=5.2.2 22 | 23 | install: 24 | - docker-compose run setup 25 | 26 | script: 27 | - docker-compose run test mix coveralls.travis 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nebo15/alpine-elixir:1.4.2 2 | 3 | WORKDIR /app 4 | 5 | RUN mix local.hex --force 6 | RUN mix local.rebar --force 7 | RUN apk --no-cache add inotify-tools 8 | 9 | CMD ["mix", "test"] 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ElasticSync 2 | 3 | [![Build Status](https://travis-ci.org/promptworks/elastic_sync.svg?branch=master)](https://travis-ci.org/promptworks/elastic_sync) 4 | 5 | This project is in its infancy. So unless you're interested in contributing, you should probably move along. 6 | 7 | This project is inspired by [searchkick](https://github.com/ankane/searchkick). It aims to provide: 8 | 9 | + An Ecto-like interface for creating/updating/deleting ElasticSearch documents. 10 | + An seamless way to keep your Ecto models in synchronization with an ElasticSearch. 11 | + Mix tasks for reindexing. 12 | 13 | It is definitely *not* an Ecto adapter for ElasticSearch. 14 | 15 | ## Installation 16 | 17 | This project is not currently available on Hex, so for now, you'll have to load it from GitHub. 18 | 19 | 1. Add `elastic_sync` to your list of dependencies in `mix.exs`: 20 | 21 | ```elixir 22 | def deps do 23 | [{:elastic_sync, github: "promptworks/elastic_sync"}] 24 | end 25 | ``` 26 | 27 | 2. Ensure `elastic_sync` is started before your application: 28 | 29 | ```elixir 30 | def application do 31 | [applications: [:elastic_sync]] 32 | end 33 | ``` 34 | 35 | ## Usage 36 | 37 | ### ElasticSync.Index 38 | 39 | Like Ecto, ElasticSync has a concept of a schema and a repo. Here's how you'd configure your schema: 40 | 41 | ```elixir 42 | defmodule MyApp.Food do 43 | defstruct [:id, :name] 44 | 45 | use ElasticSync.Index, index: "foods" 46 | 47 | @doc """ 48 | Convert a struct to a plain ol' map. This will become our document. 49 | """ 50 | def to_search_document(record) do 51 | Map.take(record, [:id, :name]) 52 | end 53 | end 54 | ``` 55 | 56 | Great. Now, you can insert/update/delete some data. 57 | 58 | ```elixir 59 | alias MyApp.Food 60 | alias ElasticSync.Repo 61 | 62 | {:ok, 201, _response} = Repo.insert(%Food{id: 1}) 63 | {:ok, 200, _response} = Repo.update(%Food{id: 1, name: "meatloaf"}) 64 | {:ok, 200, _response} = Repo.delete(%Food{id: 1}) 65 | 66 | {:ok, 200, _response} = Repo.insert_all(Food, [ 67 | %Food{id: 1, name: "cheesecake"}, 68 | %Food{id: 2, name: "applesauce"}, 69 | %Food{id: 3, name: "sausage"} 70 | ]) 71 | ``` 72 | 73 | And, you can search it: 74 | 75 | ```elixir 76 | # Search with strings: 77 | 78 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(SomeModel, "meatloaf") 79 | 80 | # Search using the elasticsearch DSL: 81 | 82 | query = %{ 83 | query: %{bool: %{must: [%{match: %{name: "meatloaf"}}]}} 84 | } 85 | 86 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Food, query) 87 | 88 | # Or, use the macro provided by Tirexs: 89 | 90 | import Tirexs.Search 91 | 92 | query = search do 93 | query do 94 | bool do 95 | must do 96 | match "name", "meatloaf" 97 | end 98 | end 99 | end 100 | end 101 | 102 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Food, query) 103 | ``` 104 | 105 | ### ElasticSync.SyncRepo 106 | 107 | Imagine you're building an app that uses Ecto. You want to synchronize changes that are made in the database with your ElasticSearch index. This is where `ElasticSync.SyncRepo` comes in handy! 108 | 109 | ```elixir 110 | defmodule MyApp.SyncRepo do 111 | use ElasticSync.SyncRepo, ecto: MyApp.Repo 112 | end 113 | ``` 114 | 115 | Now, anytime you make a change to one of your models, just use the `SyncRepo` instead of your app's `Repo`. 116 | 117 | The `SyncRepo` will only push those changes to ElasticSearch if the save operation is successful. However, you might want to handle the scenario where an HTTP request fails. For example: 118 | 119 | ```elixir 120 | changeset = Foo.changeset(%Food{id: 1}, %{"name" => "poison"}) 121 | 122 | case SyncRepo.insert(changeset) do 123 | {:ok, record} -> 124 | # everything was successful! 125 | {:error, changeset} -> 126 | # ecto had an error 127 | {:error, status, response} -> 128 | # something bad happened when communicating with ElasticSearch 129 | end 130 | ``` 131 | 132 | ### Mix.Tasks.ElasticSync.Reindex 133 | 134 | Now, to reindex your models, you can simply run: 135 | 136 | ``` 137 | $ mix elastic_sync.reindex MyApp.SyncRepo MyApp.SomeModel 138 | ``` 139 | 140 | ### Configuring the Elasticsearch endpoint 141 | 142 | ElasticSync is build on top of [Tirexs](https://github.com/Zatvobor/tirexs), which offers two ways to customize the Elasticsearch endpoint: 143 | 144 | 1. By setting the `ES_URI` environment variable. 145 | 2. Using `Mix.Config`: 146 | 147 | ```elixir 148 | config :tirexs, :uri, "http://your-endpoint.com:9200" 149 | ``` 150 | 151 | ## Development 152 | 153 | The easiest way to run the tests locally is by using docker-compose. To setup your environment, run the following commands: 154 | 155 | ``` 156 | $ git clone git@github.com:promptworks/elastic_sync 157 | $ cd elastic_sync 158 | $ docker-compose run setup 159 | ``` 160 | 161 | To run the tests, just run: 162 | 163 | ``` 164 | $ docker-compose run test 165 | ``` 166 | 167 | To run the tests against a specific elasticsearch version, you can use the `ES_VERSION` environment variable. Make sure to stop all running containers after doing this: 168 | 169 | ``` 170 | $ export ES_VERSION=1.7.6 171 | $ docker-compose stop 172 | ``` 173 | 174 | ## TODO 175 | 176 | + [ ] Create indexes with a default analyzer. 177 | + [ ] Allow developers to customize mappings (#5). 178 | + [ ] Better output for the mix task. 179 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure for your application as: 12 | # 13 | # config :elastic_sync, key: :value 14 | # 15 | # And access this configuration in your application as: 16 | # 17 | # Application.get_env(:elastic_sync, :key) 18 | # 19 | # Or configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | if Mix.env == :test do 31 | import_config "#{Mix.env}.exs" 32 | end 33 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :elastic_sync, 4 | ecto_repos: [ElasticSync.TestRepo] 5 | 6 | config :elastic_sync, ElasticSync.TestRepo, 7 | username: "postgres", 8 | hostname: System.get_env("POSTGRES_HOST") || "localhost", 9 | database: "elastic_sync_test", 10 | adapter: Ecto.Adapters.Postgres, 11 | pool: Ecto.Adapters.SQL.Sandbox 12 | 13 | config :logger, level: :warn 14 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | elasticsearch: 5 | image: elasticsearch:${ES_VERSION:-5.2.2}-alpine 6 | ports: 7 | - '9200' 8 | volumes: 9 | - /usr/share/elasticsearch/data 10 | 11 | postgres: 12 | image: postgres:alpine 13 | ports: 14 | - '5432' 15 | volumes: 16 | - /var/lib/postgresql/data 17 | 18 | test: &test 19 | build: . 20 | tty: true 21 | depends_on: 22 | - elasticsearch 23 | - postgres 24 | volumes: 25 | - .:/app 26 | environment: 27 | - MIX_ENV 28 | - TRAVIS 29 | - TRAVIS_JOB_ID 30 | - POSTGRES_HOST=postgres 31 | - ES_URI=http://elasticsearch:9200 32 | 33 | setup: 34 | <<: *test 35 | command: mix do deps.get, compile 36 | environment: 37 | - MIX_ENV=test 38 | -------------------------------------------------------------------------------- /lib/elastic_sync.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync do 2 | end 3 | -------------------------------------------------------------------------------- /lib/elastic_sync/index.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Index do 2 | defstruct [ 3 | name: nil, 4 | type: nil, 5 | alias: nil, 6 | config: {ElasticSync.Index, :default_config} 7 | ] 8 | 9 | defmacro __using__(opts) do 10 | name = Keyword.get(opts, :index) 11 | type = Keyword.get(opts, :type, name) 12 | config = Keyword.get(opts, :config) 13 | 14 | quote do 15 | def __elastic_sync__ do 16 | alias ElasticSync.Index 17 | 18 | %Index{} 19 | |> Index.put(:name, unquote(name)) 20 | |> Index.put(:type, unquote(type)) 21 | |> Index.put(:config, unquote(config)) 22 | end 23 | end 24 | end 25 | 26 | def put(_index, :name, nil) do 27 | raise ArgumentError, """ 28 | You must provide an index name. For example: 29 | 30 | use ElasticSync.Index, index: "foods" 31 | """ 32 | end 33 | def put(index, :config, nil), do: index 34 | def put(_index, :config, value) when not is_tuple(value) do 35 | raise ArgumentError, """ 36 | The index config must be a tuple in the format. 37 | 38 | use ElasticSync.Index, index: "foods", config: {Food, :index_config} 39 | """ 40 | end 41 | def put(index, key, value), do: Map.put(index, key, value) 42 | 43 | def default_config do 44 | %{} 45 | end 46 | 47 | def put_alias(%__MODULE__{name: name, alias: alias_name} = index) do 48 | ms = :os.system_time(:milli_seconds) 49 | base_name = alias_name || name 50 | next_name = base_name <> "-" <> to_string(ms) 51 | %__MODULE__{index | name: next_name, alias: base_name} 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/elastic_sync/index/http.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Index.HTTP do 2 | alias ElasticSync.Index 3 | alias Tirexs.HTTP 4 | alias Tirexs.Resources.APIs, as: API 5 | 6 | @doc """ 7 | Useful for reindexing. It will: 8 | 9 | 1. Create a new index using the given alias_name. 10 | 2. Call the given function, with the alias name as an argument. 11 | 3. Refresh the index. 12 | 4. Set the newly created index to the alias. 13 | 5. Remove old indicies. 14 | """ 15 | def transition(%Index{} = index, load_data) do 16 | new_index = Index.put_alias(index) 17 | 18 | with :ok <- normalize_http(create(new_index)), 19 | :ok <- normalize_data_load(load_data.(new_index)), 20 | :ok <- normalize_http(refresh(new_index)), 21 | :ok <- normalize_http(replace_alias(new_index)), 22 | :ok <- normalize_http(clean_indicies(new_index)), 23 | do: {:ok, new_index} 24 | end 25 | 26 | def create(%Index{name: name, config: {mod, fun}}) do 27 | name 28 | |> API.index 29 | |> HTTP.put(apply(mod, fun, [])) 30 | end 31 | 32 | def remove(%Index{name: name}) do 33 | do_remove(name) 34 | end 35 | 36 | def exists?(%Index{name: name}) do 37 | case name |> API.index |> HTTP.get do 38 | {:ok, _, _} -> true 39 | {:error, _, _} -> false 40 | end 41 | end 42 | 43 | def refresh(%Index{name: name}) do 44 | name 45 | |> API._refresh 46 | |> HTTP.post 47 | end 48 | 49 | def load(%Index{name: name, type: type}, data) do 50 | import Tirexs.Bulk 51 | 52 | # Tirexs requires keyword lists... 53 | data = Enum.map data, fn 54 | doc when is_list(doc) -> doc 55 | doc -> Enum.into(doc, []) 56 | end 57 | 58 | payload = 59 | [index: name, type: type] 60 | |> bulk(do: index(data)) 61 | 62 | Tirexs.bump!(payload)._bulk() 63 | end 64 | 65 | @doc """ 66 | Attach the alias name to the newly created index. Remove 67 | all old aliases. 68 | """ 69 | def replace_alias(%Index{name: name, alias: alias_name}) do 70 | add = %{add: %{alias: alias_name, index: name}} 71 | 72 | remove = 73 | alias_name 74 | |> get_aliases() 75 | |> Enum.map(fn a -> 76 | %{remove: %{alias: alias_name, index: a}} 77 | end) 78 | 79 | API._aliases 80 | |> HTTP.post(%{actions: remove ++ [add]}) 81 | end 82 | 83 | def clean_indicies(%Index{name: name, alias: alias_name}) do 84 | alias_name 85 | |> get_aliases() 86 | |> Enum.filter(&(&1 != name)) 87 | |> case do 88 | [] -> {:ok, 200, %{acknowledged: true}} 89 | names -> do_remove(names) 90 | end 91 | end 92 | 93 | defp do_remove(names) do 94 | names 95 | |> API.index 96 | |> HTTP.delete 97 | end 98 | 99 | defp get_aliases(name) do 100 | API._aliases 101 | |> HTTP.get 102 | |> normalize_aliases() 103 | |> Enum.filter(&Regex.match?(~r/^#{name}-\d{13}$/, &1)) 104 | end 105 | 106 | defp normalize_http({:ok, _, _}), do: :ok 107 | defp normalize_http({:error, _, error}) do 108 | {:error, error["error"]["reason"]} 109 | end 110 | 111 | defp normalize_data_load(:ok), do: :ok 112 | defp normalize_data_load(other) do 113 | {:error, "Failed to load data. Expected function to return :ok, but got #{inspect other}."} 114 | end 115 | 116 | defp normalize_aliases({:error, _, _}), do: [] 117 | defp normalize_aliases({:ok, 200, aliases}) do 118 | aliases 119 | |> Map.keys() 120 | |> Enum.map(&to_string/1) 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /lib/elastic_sync/reindex.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Reindex do 2 | alias ElasticSync.{Index, Repo} 3 | 4 | @batch_size 500 5 | 6 | def run(ecto, schema, opts \\ []) do 7 | index = schema.__elastic_sync__ 8 | batch_size = Keyword.get(opts, :batch_size, @batch_size) 9 | 10 | Index.HTTP.transition(index, fn new_index -> 11 | result = ecto.transaction fn -> 12 | schema 13 | |> ecto.stream(max_rows: batch_size) 14 | |> Stream.chunk(batch_size, batch_size, []) 15 | |> Stream.each(&Repo.load(new_index, &1)) 16 | |> Stream.run 17 | end 18 | 19 | normalize_transaction(result) 20 | end) 21 | end 22 | 23 | defp normalize_transaction({:ok, :ok}), do: :ok 24 | defp normalize_transaction(other), do: other 25 | end 26 | -------------------------------------------------------------------------------- /lib/elastic_sync/repo.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Repo do 2 | alias ElasticSync.Index 3 | alias Tirexs.HTTP 4 | alias Tirexs.Resources.APIs, as: API 5 | 6 | def search(schema, query) when is_binary(query) do 7 | schema 8 | |> to_search_url() 9 | |> HTTP.get(%{q: query}) 10 | end 11 | def search(schema, [search: query]) do 12 | search(schema, query) 13 | end 14 | def search(schema, query) do 15 | schema 16 | |> to_search_url() 17 | |> HTTP.post(query) 18 | end 19 | def search(schema) do 20 | schema 21 | |> to_search_url() 22 | |> HTTP.get 23 | end 24 | 25 | def insert(record) do 26 | record.__struct__ 27 | |> to_index_url() 28 | |> HTTP.post(%{id: record.id}, to_document(record)) 29 | end 30 | 31 | def insert!(record) do 32 | record.__struct__ 33 | |> to_index_url() 34 | |> HTTP.post!(%{id: record.id}, to_document(record)) 35 | end 36 | 37 | def update(record) do 38 | record 39 | |> to_document_url() 40 | |> HTTP.put(to_document(record)) 41 | end 42 | 43 | def update!(record) do 44 | record 45 | |> to_document_url() 46 | |> HTTP.put!(to_document(record)) 47 | end 48 | 49 | def delete(record) do 50 | record 51 | |> to_document_url() 52 | |> HTTP.delete 53 | end 54 | 55 | def delete!(record) do 56 | record 57 | |> to_document_url() 58 | |> HTTP.delete! 59 | end 60 | 61 | def insert_all(schema, records) when is_list(records) do 62 | with {:ok, 200, response} <- load(schema.__elastic_sync__, records), 63 | :ok <- Index.HTTP.refresh(schema.__elastic_sync__), 64 | do: {:ok, 200, response} 65 | end 66 | 67 | @doc false 68 | def load(index, records) do 69 | Index.HTTP.load(index, Enum.map(records, &to_document/1)) 70 | end 71 | 72 | def to_search_url(schema) do 73 | url_for(:_search, schema) 74 | end 75 | 76 | def to_index_url(schema) do 77 | url_for(:index, schema) 78 | end 79 | 80 | def to_document_url(record) do 81 | url_for(:doc, record.__struct__, [record.id]) 82 | end 83 | 84 | defp url_for(fun_name, schema, paths \\ []) do 85 | index = schema.__elastic_sync__ 86 | apply(API, fun_name, [index.name, index.type] ++ paths) 87 | end 88 | 89 | def to_document(record) do 90 | record.__struct__.to_search_document(record) 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/elastic_sync/sync_repo.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.SyncRepo do 2 | alias ElasticSync.{Repo, Reindex} 3 | 4 | defmacro __using__(opts) do 5 | ecto = Keyword.fetch!(opts, :ecto) 6 | 7 | quote do 8 | @ecto unquote(ecto) 9 | 10 | def __elastic_sync__(:ecto), do: @ecto 11 | 12 | def insert(struct_or_changeset, opts \\ []) do 13 | ElasticSync.SyncRepo.insert(@ecto, struct_or_changeset, opts) 14 | end 15 | 16 | def insert!(struct_or_changeset, opts \\ []) do 17 | ElasticSync.SyncRepo.insert!(@ecto, struct_or_changeset, opts) 18 | end 19 | 20 | def update(changeset, opts \\ []) do 21 | ElasticSync.SyncRepo.update(@ecto, changeset, opts) 22 | end 23 | 24 | def update!(changeset, opts \\ []) do 25 | ElasticSync.SyncRepo.update!(@ecto, changeset, opts) 26 | end 27 | 28 | def delete(struct_or_changeset, opts \\ []) do 29 | ElasticSync.SyncRepo.delete(@ecto, struct_or_changeset, opts) 30 | end 31 | 32 | def delete!(struct_or_changeset, opts \\ []) do 33 | ElasticSync.SyncRepo.delete!(@ecto, struct_or_changeset, opts) 34 | end 35 | 36 | def insert_all(schema_or_source, entries, opts \\ []) do 37 | ElasticSync.SyncRepo.insert_all(@ecto, schema_or_source, entries, opts) 38 | end 39 | 40 | def reindex(schema, opts \\ []) do 41 | ElasticSync.SyncRepo.reindex(@ecto, schema, opts) 42 | end 43 | end 44 | end 45 | 46 | def insert(ecto, struct_or_changeset, opts \\ []) do 47 | sync_one(ecto, :insert, [struct_or_changeset, opts]) 48 | end 49 | 50 | def insert!(ecto, struct_or_changeset, opts \\ []) do 51 | sync_one!(ecto, :insert!, [struct_or_changeset, opts]) 52 | end 53 | 54 | def update(ecto, changeset, opts \\ []) do 55 | sync_one(ecto, :update, [changeset, opts]) 56 | end 57 | 58 | def update!(ecto, changeset, opts \\ []) do 59 | sync_one!(ecto, :update!, [changeset, opts]) 60 | end 61 | 62 | def delete(ecto, struct_or_changeset, opts \\ []) do 63 | sync_one(ecto, :delete, [struct_or_changeset, opts]) 64 | end 65 | 66 | def delete!(ecto, struct_or_changeset, opts \\ []) do 67 | sync_one!(ecto, :delete!, [struct_or_changeset, opts]) 68 | end 69 | 70 | def insert_all(ecto, schema_or_source, entries, opts \\ []) do 71 | with {:ok, records} <- ecto.insert_all(schema_or_source, entries, opts), 72 | {:ok, _, _} <- Repo.insert_all(schema_or_source, records), 73 | do: {:ok, records} 74 | end 75 | 76 | def reindex(ecto, schema, opts \\ []) do 77 | Reindex.run(ecto, schema, opts) 78 | end 79 | 80 | defp sync_one(ecto, action, args) do 81 | with {:ok, record} <- apply(ecto, action, args), 82 | {:ok, _, _} <- apply(Repo, action, [record]), 83 | do: {:ok, record} 84 | end 85 | 86 | defp sync_one!(ecto, action, args) do 87 | result = apply(ecto, action, args) 88 | apply(Repo, action, [result]) 89 | result 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /lib/mix/tasks/elastic_sync/reindex.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.ElasticSync.Reindex do 2 | import Mix.Ecto 3 | 4 | @switches [batch_size: :integer] 5 | 6 | def run(args) do 7 | Mix.Task.run "loadpaths", args 8 | 9 | unless "--no-compile" in args do 10 | Mix.Project.compile([]) 11 | end 12 | 13 | {sync_repo, schema, opts} = parse!(args) 14 | repo = sync_repo.__elastic_sync__(:ecto) 15 | 16 | ensure_started(repo, []) 17 | 18 | case sync_repo.reindex(schema, opts) do 19 | {:ok, _} -> 20 | :ok 21 | error -> 22 | Mix.raise "The following error occurred:\n #{inspect error}" 23 | end 24 | end 25 | 26 | def parse!(args) when length(args) < 2 do 27 | Mix.raise "Wrong number of arguments." 28 | end 29 | def parse!([sync_repo_name, schema_name | args]) do 30 | {opts, _, _} = OptionParser.parse(args, strict: @switches) 31 | {compile!(sync_repo_name), compile!(schema_name), opts} 32 | end 33 | 34 | defp compile!(name) do 35 | case [name] |> Module.concat() |> Code.ensure_compiled() do 36 | {:module, mod} -> 37 | mod 38 | error -> 39 | Mix.raise "Invalid module name '#{name}', error: #{inspect error}." 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :elastic_sync, 6 | version: "0.1.0", 7 | elixir: "~> 1.3", 8 | elixirc_paths: elixirc_paths(Mix.env), 9 | build_embedded: Mix.env == :prod, 10 | start_permanent: Mix.env == :prod, 11 | preferred_cli_env: preferred_cli_env(), 12 | test_coverage: [tool: ExCoveralls], 13 | deps: deps()] 14 | end 15 | 16 | # Configuration for the OTP application 17 | # 18 | # Type "mix help compile.app" for more information 19 | def application() do 20 | [applications: app_list(Mix.env)] 21 | end 22 | 23 | # Dependencies can be Hex packages: 24 | # 25 | # {:mydep, "~> 0.3.0"} 26 | # 27 | # Or git/path repositories: 28 | # 29 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 30 | # 31 | # Type "mix help deps" for more examples and options 32 | defp deps do 33 | [{:tirexs, "~> 0.8"}, 34 | {:ecto, "~> 2.1", optional: true}, 35 | {:ecto, "~> 2.1", only: [:dev, :test]}, 36 | {:postgrex, ">= 0.0.0", only: :test}, 37 | {:excoveralls, "~> 0.6", only: :test}, 38 | {:mix_test_watch, "~> 0.3", only: :dev, runtime: false}] 39 | end 40 | 41 | defp preferred_cli_env do 42 | ["coveralls": :test, 43 | "coveralls.detail": :test, 44 | "coveralls.post": :test, 45 | "coveralls.html": :test] 46 | end 47 | 48 | defp elixirc_paths(:test), do: elixirc_paths() ++ ["test/dummy.ex"] 49 | defp elixirc_paths(_), do: elixirc_paths() 50 | defp elixirc_paths(), do: ["lib"] 51 | 52 | defp app_list(:test), do: app_list() ++ [:ecto, :postgrex] 53 | defp app_list(_), do: app_list() 54 | defp app_list(), do: [:logger, :tirexs] 55 | end 56 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"certifi": {:hex, :certifi, "1.0.0", "1c787a85b1855ba354f0b8920392c19aa1d06b0ee1362f9141279620a5be2039", [:rebar3], []}, 2 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], []}, 3 | "db_connection": {:hex, :db_connection, "1.1.2", "2865c2a4bae0714e2213a0ce60a1b12d76a6efba0c51fbda59c9ab8d1accc7a8", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, 4 | "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, 5 | "ecto": {:hex, :ecto, "2.1.4", "d1ba932813ec0e0d9db481ef2c17777f1cefb11fc90fa7c142ff354972dfba7e", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, 6 | "excoveralls": {:hex, :excoveralls, "0.6.3", "894bf9254890a4aac1d1165da08145a72700ff42d8cb6ce8195a584cb2a4b374", [:mix], [{:exjsx, "~> 3.0", [hex: :exjsx, optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, optional: false]}]}, 7 | "exjsx": {:hex, :exjsx, "3.2.1", "1bc5bf1e4fd249104178f0885030bcd75a4526f4d2a1e976f4b428d347614f0f", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, optional: false]}]}, 8 | "fs": {:hex, :fs, "2.12.0", "ad631efacc9a5683c8eaa1b274e24fa64a1b8eb30747e9595b93bec7e492e25e", [:rebar3], []}, 9 | "hackney": {:hex, :hackney, "1.7.1", "e238c52c5df3c3b16ce613d3a51c7220a784d734879b1e231c9babd433ac1cb4", [:rebar3], [{:certifi, "1.0.0", [hex: :certifi, optional: false]}, {:idna, "4.0.0", [hex: :idna, optional: false]}, {:metrics, "1.0.1", [hex: :metrics, optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, optional: false]}, {:ssl_verify_fun, "1.1.1", [hex: :ssl_verify_fun, optional: false]}]}, 10 | "idna": {:hex, :idna, "4.0.0", "10aaa9f79d0b12cf0def53038547855b91144f1bfcc0ec73494f38bb7b9c4961", [:rebar3], []}, 11 | "jsx": {:hex, :jsx, "2.8.2", "7acc7d785b5abe8a6e9adbde926a24e481f29956dd8b4df49e3e4e7bcc92a018", [:mix, :rebar3], []}, 12 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], []}, 13 | "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], []}, 14 | "mix_test_watch": {:hex, :mix_test_watch, "0.4.0", "7e44b681b0238999d4c39b5beed77b4ac45aef1c112a763aae414bdb5bc34523", [:mix], [{:fs, "~> 2.12", [hex: :fs, optional: false]}]}, 15 | "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], []}, 16 | "postgrex": {:hex, :postgrex, "0.13.2", "2b88168fc6a5456a27bfb54ccf0ba4025d274841a7a3af5e5deb1b755d95154e", [:mix], [{:connection, "~> 1.0", [hex: :connection, optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, optional: false]}]}, 17 | "progress_bar": {:hex, :progress_bar, "1.6.1", "4a82aa4f2b04708dca7472febf5e73477f0e09c81fec98fc502c8d5e3d833172", [:mix], [], "hexpm"}, 18 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}, 19 | "tirexs": {:hex, :tirexs, "0.8.13", "71c6832cecdb7e2675e9d1c13effc101ad553b60316a90042d3062b1ea2ce3b3", [:mix], [{:exjsx, "~> 3.2.0", [hex: :exjsx, optional: false]}]}} 20 | -------------------------------------------------------------------------------- /priv/test_repo/migrations/1_migrate_all.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.TestRepo.Migrations.MigrateAll do 2 | use Ecto.Migration 3 | 4 | def change do 5 | create table(:things) do 6 | add :name, :string 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /test/dummy.ex: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.TestRepo do 2 | use Ecto.Repo, otp_app: :elastic_sync 3 | end 4 | 5 | defmodule ElasticSync.TestSyncRepo do 6 | use ElasticSync.SyncRepo, 7 | ecto: ElasticSync.TestRepo 8 | end 9 | 10 | defmodule ElasticSync.Thing do 11 | use Ecto.Schema 12 | 13 | use ElasticSync.Index, 14 | index: "elastic_sync_thing", 15 | config: {ElasticSync.Thing, :index_config} 16 | 17 | import Ecto.Changeset 18 | 19 | schema "things" do 20 | field :name, :string 21 | end 22 | 23 | def to_search_document(record) do 24 | Map.take(record, [:id, :name]) 25 | end 26 | 27 | def changeset(struct, params \\ %{}) do 28 | struct 29 | |> cast(params, [:name]) 30 | |> validate_required([:name]) 31 | end 32 | 33 | def index_config do 34 | %{ 35 | settings: %{}, 36 | mappings: %{ 37 | test: %{ 38 | properties: %{ 39 | name: %{type: "string"}, 40 | id: %{type: "string"} 41 | } 42 | } 43 | } 44 | } 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /test/elastic_sync/index/http_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.Index.HTTPTest do 2 | use ExUnit.Case, async: false 3 | 4 | alias Tirexs.HTTP 5 | alias Tirexs.Resources.APIs, as: API 6 | alias ElasticSync.Index 7 | 8 | @index %Index{ 9 | name: "elastic_sync_index_test", 10 | type: "elastic_sync_index_test" 11 | } 12 | 13 | @search "#{@index.name}/_search" 14 | 15 | setup do 16 | HTTP.delete!("*") 17 | :ok 18 | end 19 | 20 | test "create and remove" do 21 | assert {:ok, _, _} = Index.HTTP.create(@index) 22 | assert Index.HTTP.exists?(@index) 23 | 24 | assert {:ok, _, _} = Index.HTTP.remove(@index) 25 | refute Index.HTTP.exists?(@index) 26 | end 27 | 28 | test "load/2 and refresh/1" do 29 | assert {:ok, _, _} = Index.HTTP.create(@index) 30 | assert {:ok, _, _} = Index.HTTP.load(@index, [%{id: 1, name: "foo"}, %{id: 2, name: "bar"}]) 31 | 32 | {:ok, _, %{hits: %{hits: hits}}} = HTTP.get(@search) 33 | assert length(hits) == 0 34 | 35 | Index.HTTP.refresh(@index) 36 | {:ok, _, %{hits: %{hits: hits}}} = HTTP.get(@search) 37 | assert length(hits) == 2 38 | end 39 | 40 | test "replace_alias/1" do 41 | # Generate a new alias name 42 | index1 = Index.put_alias(@index) 43 | 44 | # Replace alias should point "elastic_sync_index_test" 45 | # at the timestamped index name. 46 | Index.HTTP.create(index1) 47 | Index.HTTP.replace_alias(index1) 48 | assert {:ok, _, resp} = HTTP.get(@index.name) 49 | assert Map.keys(resp) == [String.to_atom(index1.name)] 50 | 51 | # Swap to the next alias 52 | index2 = Index.put_alias(index1) 53 | assert {:ok, _, _} = Index.HTTP.create(index2) 54 | assert {:ok, _, _} = Index.HTTP.replace_alias(index2) 55 | 56 | assert {:ok, _, resp} = HTTP.get(@index.name) 57 | assert Map.keys(resp) == [String.to_atom(index2.name)] 58 | end 59 | 60 | test "clean_indicies/1" do 61 | index1 = Index.put_alias(@index) 62 | Index.HTTP.create(index1) 63 | 64 | index2 = Index.put_alias(index1) 65 | Index.HTTP.create(index2) 66 | 67 | {:ok, _, resp} = HTTP.get(API._aliases()) 68 | assert Map.keys(resp) == [String.to_atom(index1.name), String.to_atom(index2.name)] 69 | 70 | Index.HTTP.clean_indicies(index2) 71 | 72 | {:ok, _, resp} = HTTP.get(API._aliases()) 73 | assert Map.keys(resp) == [String.to_atom(index2.name)] 74 | end 75 | 76 | test "transition/1" do 77 | assert {:ok, index1} = Index.HTTP.transition(@index, fn _i -> :ok end) 78 | assert Index.HTTP.exists?(index1) 79 | 80 | assert {:ok, index2} = Index.HTTP.transition(@index, fn _i -> :ok end) 81 | refute Index.HTTP.exists?(index1) 82 | assert Index.HTTP.exists?(index2) 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /test/elastic_sync/index_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.IndexTest do 2 | use ExUnit.Case, async: false 3 | 4 | alias Tirexs.HTTP 5 | alias ElasticSync.Index 6 | 7 | @index %Index{ 8 | name: "elastic_sync_index_test", 9 | type: "elastic_sync_index_test" 10 | } 11 | 12 | @config {Index, :default_config} 13 | 14 | setup do 15 | HTTP.delete!("*") 16 | :ok 17 | end 18 | 19 | test "put/3 with a name" do 20 | assert Index.put(@index, :name, "foo").name == "foo" 21 | end 22 | 23 | test "put/3 with a nil name" do 24 | assert_raise ArgumentError, fn -> 25 | Index.put(@index, :name, nil) 26 | end 27 | end 28 | 29 | test "put/3 with config tuple" do 30 | assert Index.put(@index, :config, @config).config == @config 31 | end 32 | 33 | test "put/3 with nil config" do 34 | assert Index.put(@index, :config, nil) == @index 35 | end 36 | 37 | test "put/3 with non-tuple" do 38 | assert_raise ArgumentError, fn -> 39 | Index.put(@index, :config, "foo") 40 | end 41 | end 42 | 43 | test "put_alias/1" do 44 | index1 = Index.put_alias(@index) 45 | assert index1.alias == @index.name 46 | assert Regex.match?(~r/^#{@index.name}-\d{13}$/, index1.name) 47 | 48 | index2 = Index.put_alias(index1) 49 | assert index2.alias == @index.name 50 | assert Regex.match?(~r/^#{@index.name}-\d{13}$/, index2.name) 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /test/elastic_sync/repo_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.RepoTest do 2 | use ExUnit.Case, async: false 3 | 4 | alias Tirexs.HTTP 5 | alias ElasticSync.Repo 6 | 7 | doctest ElasticSync.Repo 8 | 9 | defmodule Thing do 10 | defstruct [:id, :name] 11 | 12 | use ElasticSync.Index, 13 | index: "elastic_sync_repo_test", 14 | type: "things" 15 | 16 | def to_search_document(struct) do 17 | Map.take(struct, [:id, :name]) 18 | end 19 | end 20 | 21 | defp find(id) do 22 | HTTP.get("/elastic_sync_repo_test/things/#{id}") 23 | end 24 | 25 | setup do 26 | HTTP.delete!("*") 27 | HTTP.put!("elastic_sync_test") 28 | :ok 29 | end 30 | 31 | test "to_index_url/1 generates a valid url" do 32 | assert Repo.to_index_url(Thing) == "elastic_sync_repo_test/things" 33 | end 34 | 35 | test "to_document_url/1 generates a valid url" do 36 | assert Repo.to_document_url(%Thing{id: 1}) == "elastic_sync_repo_test/things/1" 37 | end 38 | 39 | test "insert/1" do 40 | assert {:ok, 201, _} = Repo.insert(%Thing{id: 1}) 41 | assert {:ok, 200, _} = find(1) 42 | end 43 | 44 | test "insert!" do 45 | Repo.insert!(%Thing{id: 1}) 46 | assert {:ok, 200, _} = find(1) 47 | end 48 | 49 | test "update/1" do 50 | Repo.insert!(%Thing{id: 1}) 51 | assert {:ok, 200, _} = Repo.update(%Thing{id: 1, name: "pasta"}) 52 | {:ok, 200, %{_source: source}} = find(1) 53 | assert source == %{id: 1, name: "pasta"} 54 | end 55 | 56 | test "update!/1" do 57 | Repo.insert!(%Thing{id: 1}) 58 | Repo.update!(%Thing{id: 1, name: "pasta"}) 59 | {:ok, 200, %{_source: source}} = find(1) 60 | assert source == %{id: 1, name: "pasta"} 61 | end 62 | 63 | test "delete/1" do 64 | Repo.insert!(%Thing{id: 1}) 65 | assert {:ok, 200, _} = Repo.delete(%Thing{id: 1, name: "pasta"}) 66 | assert {:error, 404, _} = find(1) 67 | end 68 | 69 | test "delete!/1" do 70 | Repo.insert!(%Thing{id: 1}) 71 | Repo.delete!(%Thing{id: 1, name: "pasta"}) 72 | {:error, 404, _} = find(1) 73 | end 74 | 75 | test "insert_all/1" do 76 | Repo.insert_all Thing, [ 77 | %Thing{id: 1, name: "meatloaf"}, 78 | %Thing{id: 2, name: "pizza"}, 79 | %Thing{id: 3, name: "sausage"}, 80 | ] 81 | 82 | {:ok, 200, %{hits: %{hits: hits}}} = HTTP.get("/elastic_sync_repo_test/things/_search") 83 | assert length(hits) == 3 84 | end 85 | 86 | test "search/2 with a binary" do 87 | Repo.insert_all Thing, [ 88 | %Thing{id: 1, name: "meatloaf"}, 89 | %Thing{id: 2, name: "pizza"} 90 | ] 91 | 92 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Thing) 93 | assert length(hits) == 2 94 | 95 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Thing, "meatloaf") 96 | assert length(hits) == 1 97 | end 98 | 99 | test "search/2 with map" do 100 | Repo.insert_all Thing, [ 101 | %Thing{id: 1, name: "meatloaf"}, 102 | %Thing{id: 2, name: "pizza"} 103 | ] 104 | 105 | query = %{query: %{bool: %{must: [%{match: %{name: "meatloaf"}}]}}} 106 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Thing, query) 107 | assert length(hits) == 1 108 | end 109 | 110 | test "search/2 with DSL" do 111 | import Tirexs.Search 112 | 113 | Repo.insert_all Thing, [ 114 | %Thing{id: 1, name: "meatloaf"}, 115 | %Thing{id: 2, name: "pizza"} 116 | ] 117 | 118 | query = search do 119 | query do 120 | bool do 121 | must do 122 | match "name", "meatloaf" 123 | end 124 | end 125 | end 126 | end 127 | 128 | {:ok, 200, %{hits: %{hits: hits}}} = Repo.search(Thing, query) 129 | assert length(hits) == 1 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /test/elastic_sync/sync_repo_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSync.SyncRepoTest do 2 | use ExUnit.Case, async: false 3 | 4 | doctest ElasticSync.SyncRepo 5 | 6 | alias ElasticSync.{Thing, TestRepo, TestSyncRepo} 7 | 8 | setup do 9 | Tirexs.HTTP.delete!("*") 10 | Tirexs.HTTP.put!("elastic_sync_test") 11 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(ElasticSync.TestRepo) 12 | end 13 | 14 | test "insert with a struct" do 15 | assert {:ok, %Thing{id: _}} = TestSyncRepo.insert(%Thing{name: "meatloaf"}) 16 | end 17 | 18 | test "insert with a changeset" do 19 | changeset = Thing.changeset(%Thing{name: "meatloaf"}) 20 | assert {:ok, %Thing{id: _}} = TestSyncRepo.insert(changeset) 21 | end 22 | 23 | test "insert with invalid changeset" do 24 | changeset = Thing.changeset(%Thing{}) 25 | assert {:error, _} = TestSyncRepo.insert(changeset) 26 | end 27 | 28 | test "update with a changeset" do 29 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 30 | changeset = Thing.changeset(thing, %{"name" => "pears"}) 31 | assert {:ok, %Thing{name: "pears"}} = TestSyncRepo.update(changeset) 32 | end 33 | 34 | test "update with invalid changeset" do 35 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 36 | changeset = Thing.changeset(thing, %{"name" => ""}) 37 | assert {:error, _} = TestSyncRepo.update(changeset) 38 | end 39 | 40 | test "update! with a changeset" do 41 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 42 | changeset = Thing.changeset(thing, %{"name" => "pears"}) 43 | assert %Thing{name: "pears"} = TestSyncRepo.update!(changeset) 44 | end 45 | 46 | test "update! with an invalid changeset" do 47 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 48 | changeset = Thing.changeset(thing, %{"name" => ""}) 49 | 50 | assert_raise Ecto.InvalidChangesetError, fn -> 51 | TestSyncRepo.update!(changeset) 52 | end 53 | end 54 | 55 | test "delete" do 56 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 57 | assert {:ok, _} = TestSyncRepo.delete(thing) 58 | end 59 | 60 | test "delete with a changeset" do 61 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 62 | assert {:ok, _} = TestSyncRepo.delete(Thing.changeset(thing)) 63 | end 64 | 65 | test "delete!" do 66 | thing = TestSyncRepo.insert!(%Thing{name: "meatloaf"}) 67 | assert %Thing{name: "meatloaf"} = TestSyncRepo.delete!(thing) 68 | end 69 | 70 | test "reindex" do 71 | TestRepo.insert!(%Thing{name: "one"}) 72 | TestRepo.insert!(%Thing{name: "two"}) 73 | TestRepo.insert!(%Thing{name: "three"}) 74 | 75 | assert {:ok, _} = TestSyncRepo.reindex(Thing) 76 | assert {:ok, _, %{hits: %{hits: hits}}} = ElasticSync.Repo.search(Thing) 77 | assert length(hits) == 3 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /test/elastic_sync_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ElasticSyncTest do 2 | use ExUnit.Case, async: false 3 | end 4 | -------------------------------------------------------------------------------- /test/mix/tasks/elastic_sync/reindex_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.ElasticSync.ReindexTest do 2 | use ExUnit.Case, async: false 3 | 4 | alias Tirexs.HTTP 5 | alias Mix.Tasks.ElasticSync.Reindex 6 | alias ElasticSync.{TestRepo, TestSyncRepo, Thing} 7 | 8 | @index "elastic_sync_thing" 9 | @search "/#{@index}/_search" 10 | @aliases "/_aliases" 11 | 12 | setup do 13 | Tirexs.HTTP.delete!("*") 14 | :ok = Ecto.Adapters.SQL.Sandbox.checkout(ElasticSync.TestRepo) 15 | end 16 | 17 | setup do 18 | TestRepo.insert!(%Thing{name: "one"}) 19 | TestRepo.insert!(%Thing{name: "two"}) 20 | TestRepo.insert!(%Thing{name: "three"}) 21 | :ok 22 | end 23 | 24 | defp reindex! do 25 | ExUnit.CaptureIO.capture_io fn -> 26 | assert :ok == Reindex.run(["ElasticSync.TestSyncRepo", "ElasticSync.Thing"]) 27 | end 28 | end 29 | 30 | test "parse!" do 31 | flags = [batch_size: 5000] 32 | 33 | assert {TestSyncRepo, Thing, ^flags} = Reindex.parse!([ 34 | "ElasticSync.TestSyncRepo", 35 | "ElasticSync.Thing", 36 | "--batch-size", "5000", 37 | "--extra", "20" 38 | ]) 39 | end 40 | 41 | describe "the first reindex" do 42 | test "creates an index" do 43 | reindex!() 44 | 45 | assert {:ok, _, _} = HTTP.get(@index) 46 | end 47 | 48 | test "loads the data" do 49 | reindex!() 50 | assert {:ok, _, %{hits: %{hits: hits}}} = HTTP.get(@search) 51 | assert length(hits) == 3 52 | end 53 | end 54 | 55 | describe "after the first reindex" do 56 | test "creates a new alias for the index" do 57 | reindex!() 58 | assert {:ok, _, phase1} = HTTP.get(@aliases) 59 | 60 | reindex!() 61 | assert {:ok, _, phase2} = HTTP.get(@aliases) 62 | refute Map.keys(phase1) == Map.keys(phase2) 63 | end 64 | 65 | test "reloads the data" do 66 | reindex!() 67 | assert {:ok, _, %{hits: %{hits: hits}}} = HTTP.get(@search) 68 | assert length(hits) == 3 69 | 70 | TestRepo.insert!(%Thing{name: "four"}) 71 | 72 | reindex!() 73 | assert {:ok, _, %{hits: %{hits: hits}}} = HTTP.get(@search) 74 | assert length(hits) == 4 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | Mix.Task.run "ecto.drop", ["--quiet", "-r", "ElasticSync.TestRepo"] 2 | Mix.Task.run "ecto.create", ["--quiet", "-r", "ElasticSync.TestRepo"] 3 | Mix.Task.run "ecto.migrate", ["-r", "ElasticSync.TestRepo"] 4 | 5 | ElasticSync.TestRepo.start_link 6 | 7 | ExUnit.start() 8 | 9 | Ecto.Adapters.SQL.Sandbox.mode(ElasticSync.TestRepo, :manual) 10 | --------------------------------------------------------------------------------