├── .gitignore ├── Guardfile ├── LICENCE ├── README.md ├── lib └── parallel.ex ├── mix.exs └── test ├── parallel_test.exs └── test_helper.exs /.gitignore: -------------------------------------------------------------------------------- 1 | /deps 2 | erl_crash.dump 3 | _build 4 | 5 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # Add files and commands to this file, like the example: 2 | # watch(%r{file/path}) { `command(s)` } 3 | # 4 | guard 'shell' do 5 | watch(/(.*).exs?/) {|m| `mix test` } 6 | end 7 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Adam Lindberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Parallel 2 | 3 | A library that implements Elixir's `Enum` interface but parallelized. 4 | 5 | ```elixir 6 | :timer.tc fn -> Enum.map 1..10, fn i -> :timer.sleep(1000) end end 7 | # -> {10010674, [:ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok]} 8 | # Run time is 10 seconds 9 | :timer.tc fn -> Parallel.map 1..10, fn i -> :timer.sleep(1000) end, size: 10 end 10 | # -> {1001653, [:ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok, :ok]} 11 | # Run time is 1 second 12 | ``` 13 | 14 | # TODO 15 | 16 | * [ ] Implement all Enum functions 17 | * [x] `all?(collection, fun \\ fn x -> x end)` 18 | * [x] `any?(collection, fun \\ fn x -> x end)` 19 | * [ ] `at(collection, n, default \\ nil)` 20 | * [ ] `chunk(collection, n)` 21 | * [ ] `chunk(collection, n, step, pad \\ nil)` 22 | * [ ] `chunk_by(collection, fun)` 23 | * [ ] `concat(enumerables)` 24 | * [ ] `concat(left, right)` 25 | * [ ] `count(collection)` 26 | * [ ] `count(collection, fun)` 27 | * [ ] `dedup(collection)` 28 | * [ ] `dedup_by(collection, fun)` 29 | * [ ] `drop(collection, count)` 30 | * [ ] `drop_while(collection, fun)` 31 | * [x] `each(collection, fun)` 32 | * [ ] `empty?(collection)` 33 | * [ ] `fetch(collection, n)` 34 | * [ ] `fetch!(collection, n)` 35 | * [ ] `filter(collection, fun)` 36 | * [ ] `filter_map(collection, filter, mapper)` 37 | * [ ] `find(collection, default \\ nil, fun)` 38 | * [ ] `find_index(collection, fun)` 39 | * [ ] `find_value(collection, default \\ nil, fun)` 40 | * [ ] `flat_map(collection, fun)` 41 | * [ ] `flat_map_reduce(collection, acc, fun)` 42 | * [ ] `group_by(collection, dict \\ %{}, fun)` 43 | * [ ] `intersperse(collection, element)` 44 | * [ ] `into(collection, list)` 45 | * [ ] `into(collection, list, transform)` 46 | * [ ] `join(collection, joiner \\ "")` 47 | * [x] `map(collection, fun)` 48 | * [ ] `map_join(collection, joiner \\ "", mapper)` 49 | * [ ] `map_reduce(collection, acc, fun)` 50 | * [ ] `max(collection)` 51 | * [ ] `max_by(collection, fun)` 52 | * [ ] `member?(collection, value)` 53 | * [ ] `min(collection)` 54 | * [ ] `min_by(collection, fun)` 55 | * [ ] `min_max(collection)` 56 | * [ ] `min_max_by(collection, fun)` 57 | * [ ] `partition(collection, fun)` 58 | * [ ] `random(collection)` 59 | * [ ] `reduce(collection, fun)` 60 | * [ ] `reduce(collection, acc, fun)` 61 | * [ ] `reduce_while(collection, acc, fun)` 62 | * [ ] `reject(collection, fun)` 63 | * [ ] `reverse(collection)` 64 | * [ ] `reverse(collection, tail)` 65 | * [ ] `reverse_slice(collection, start, count)` 66 | * [ ] `scan(enum, fun)` 67 | * [ ] `scan(enum, acc, fun)` 68 | * [ ] `shuffle(collection)` 69 | * [ ] `slice(collection, range)` 70 | * [ ] `slice(collection, start, count)` 71 | * [ ] `sort(collection)` 72 | * [ ] `sort(collection, fun)` 73 | * [ ] `sort_by(collection, mapper, sorter \\ &<=/2)` 74 | * [ ] `split(collection, count)` 75 | * [ ] `split_while(collection, fun)` 76 | * [ ] `sum(collection)` 77 | * [ ] `take(collection, count)` 78 | * [ ] `take_every(collection, nth)` 79 | * [ ] `take_random(collection, count)` 80 | * [ ] `take_while(collection, fun)` 81 | * [ ] `to_list(collection)` 82 | * [ ] `uniq(collection)` 83 | * [ ] `uniq_by(collection, fun)` 84 | * [ ] `unzip(collection)` 85 | * [ ] `with_index(collection)` 86 | * [ ] `zip(collection1, collection2)` 87 | * [ ] Short circuit relevant functions (`any?/2`, `find/3` etc.) 88 | * [ ] Investigate [Stream](http://elixir-lang.org/docs/v1.1/elixir/Stream.html) 89 | compatibility (option `stream: true`?) 90 | * [ ] Decide on implementing all functions or just relevant ones (e.g. `first/1`) 91 | * [ ] Add documentation 92 | * [ ] Add `link: false` option 93 | * [ ] Add `sorted: true` option 94 | -------------------------------------------------------------------------------- /lib/parallel.ex: -------------------------------------------------------------------------------- 1 | defmodule Parallel do 2 | 3 | def map(collection, fun, options \\ []) do 4 | run(collection, fun, options, [], fn item, acc -> [item|acc] end) 5 | end 6 | 7 | def each(collection, fun, options \\ []) do 8 | run(collection, fun, options, nil, fn _item, nil -> nil end) 9 | end 10 | 11 | def any?(collection, fun, options \\ []) do 12 | run(collection, fun, options, false, fn item, value -> item || value end) 13 | end 14 | 15 | def all?(collection, fun, options \\ []) do 16 | run(collection, fun, options, true, fn item, value -> item && value end) 17 | end 18 | 19 | # Private 20 | 21 | defp run(collection, fun, options, acc, update) do 22 | state = {pool(fun, options), [], acc, update} 23 | {_, busy, acc, _} = Enum.reduce(collection, state, &execute/2) 24 | consume(busy, acc, update) 25 | end 26 | 27 | defp execute(item, {free = [], busy, acc, update}) do 28 | receive do 29 | {ref, from, result} -> 30 | send(from, {ref, self(), item}) 31 | {free, busy, update.(result, acc), update} 32 | end 33 | end 34 | defp execute(item, {[worker = {ref, pid}|free], busy, acc, update}) do 35 | send(pid, {ref, self(), item}) 36 | {free, [worker|busy], acc, update} 37 | end 38 | 39 | defp consume(pool, acc, update) do 40 | Enum.reduce(pool, acc, fn {ref, pid}, acc -> 41 | receive do 42 | {^ref, ^pid, result} -> update.(result, acc) 43 | end 44 | end) 45 | end 46 | 47 | def worker(fun) do 48 | receive do 49 | {ref, sender, item} -> 50 | send(sender, {ref, self, fun.(item)}) 51 | worker(fun) 52 | :exit -> 53 | :ok 54 | end 55 | end 56 | 57 | defp pool(fun, options) do 58 | size = Keyword.get(options, :size) || :erlang.system_info(:schedulers) * 2 59 | spawn_worker = fn -> {make_ref(), spawn_link(fn -> worker(fun) end)} end 60 | Stream.repeatedly(spawn_worker) |> Enum.take(size) 61 | end 62 | 63 | end 64 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Parallel.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ app: :parallel, 6 | version: "0.0.1", 7 | deps: deps ] 8 | end 9 | 10 | # Configuration for the OTP application 11 | def application do 12 | [] 13 | end 14 | 15 | # Returns the list of dependencies in the format: 16 | # { :foobar, "0.1", git: "https://github.com/elixir-lang/foobar.git" } 17 | defp deps do 18 | [] 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/parallel_test.exs: -------------------------------------------------------------------------------- 1 | Code.require_file "test_helper.exs", __DIR__ 2 | 3 | defmodule ParallelTest do 4 | use ExUnit.Case 5 | doctest Parallel 6 | 7 | import EnumCompare 8 | import Parallel 9 | 10 | test :map do 11 | assert_enum :map, 1..10, &(&1 + 1), sort: true 12 | end 13 | 14 | test :random_map do 15 | :random.seed(:erlang.now()) 16 | Enum.each 1..50, fn _ -> 17 | list = Enum.map 1..50, &:random.uniform/1 18 | assert_enum :map, list, &(&1 + 1), sort: true 19 | end 20 | end 21 | 22 | test :each do 23 | pid = self() 24 | collection = 1..10 25 | each(collection, fn i -> send(pid, {:test, i}) end) 26 | Enum.each(collection, fn i -> 27 | receive do 28 | {:test, ^i} -> :ok 29 | after 100 -> 30 | assert false, "No result received" 31 | end 32 | end) 33 | end 34 | 35 | test :any? do 36 | assert_enum :any?, [false, true], fn b -> b end 37 | end 38 | 39 | test :all? do 40 | assert_enum :all?, [false, true], fn b -> b end 41 | end 42 | 43 | end 44 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | defmodule EnumCompare do 4 | use ExUnit.Case 5 | 6 | defmacro assert_enum(function, collection, fun, options \\ []) do 7 | parallel = quote do 8 | Parallel.unquote(function)(unquote(collection), unquote(fun)) 9 | end 10 | enum = quote do 11 | Enum.unquote(function)(unquote(collection), unquote(fun)) 12 | end 13 | if Keyword.get(options, :sort) do 14 | parallel = quote do: Enum.sort(unquote(parallel)) 15 | enum = quote do: Enum.sort(unquote(enum)) 16 | end 17 | quote do: assert unquote(enum) == unquote(parallel) 18 | end 19 | end 20 | --------------------------------------------------------------------------------