├── .gitignore ├── README.md ├── config └── config.exs ├── lib ├── streamz.ex └── streamz │ ├── merge.ex │ ├── merge │ └── helpers.ex │ └── task.ex ├── mix.exs ├── perf ├── merge_perf_test.exs ├── nested_merge_perf_test.exs └── task_stream_test.exs └── test ├── streamz ├── merge_test.exs ├── task_test.exs └── time_test.exs ├── streamz_test.exs └── test_helper.exs /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | erl_crash.dump 4 | *.ez 5 | *.beam 6 | .DS_Store 7 | *.swp 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Streamz 2 | ======= 3 | 4 | NOTE: Many of the stream sources previously in this repo have been moved into 5 | 6 | # Dataflow and Reactive Programming 7 | 8 | ## Highlights 9 | 10 | ### `Streamz.merge/1` 11 | `Streamz.merge/1` accepts an array of streams and merges them into a single stream. This differs from `Stream.zip/2` in that the resulting order is a function of execution order rather than simply alternating between the two streams. 12 | 13 | #### Merge two GenEvent streams and take the first 100 events from either 14 | 15 | ```elixir 16 | {:ok, event_one} = GenEvent.start_link 17 | {:ok, event_two} = GenEvent.start_link 18 | combined_stream = Streamz.merge [ 19 | GenEvent.stream(event_one), 20 | GenEvent.stream(event_two) 21 | ] 22 | combined_stream |> Enum.take(100) 23 | ``` 24 | 25 | ### `Streamz.Task.stream/1` 26 | `Streamz.Task.stream/1` accepts an array of functions and launches them all as Tasks. The returned stream will emit the results of the Tasks in the order in which execution completes. 27 | 28 | ```elixir 29 | stream = Streamz.Task.stream [ 30 | fn -> 31 | :timer.sleep(10) 32 | 1 33 | end, 34 | fn -> 2 end 35 | ] 36 | result = stream |> Enum.to_list 37 | assert result == [2,1] 38 | ``` 39 | 40 | This enables a lot of cool functionality, such as processing just the first response: 41 | 42 | ```elixir 43 | result = stream |> Enum.take(1) |> hd 44 | ``` 45 | 46 | The above example is so useful it exists as `Streamz.Task.first_completed_of/1` 47 | 48 | More ideas are welcome. Feedback on code quality is welcomed, especially when it comes to OTP fundamentals (eg. monitoring vs linking). 49 | 50 | ## Production readiness 51 | This is largely a playground at the moment, but the plan is to get this mature enough to be used in production. Some more tests and a bit more use will help with stability and my ability to commit to the APIs of the library. 52 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies. The Mix.Config module provides functions 3 | # to aid in doing so. 4 | use Mix.Config 5 | 6 | # Note this file is loaded before any dependency and is restricted 7 | # to this project. If another project depends on this project, this 8 | # file won't be loaded nor affect the parent project. 9 | 10 | # Sample configuration: 11 | # 12 | # config :my_dep, 13 | # key: :value, 14 | # limit: 42 15 | 16 | # It is also possible to import configuration files, relative to this 17 | # directory. For example, you can emulate configuration per environment 18 | # by uncommenting the line below and defining dev.exs, test.exs and such. 19 | # Configuration from the imported file will override the ones defined 20 | # here (which is why it is important to import them last). 21 | # 22 | # import_config "#{Mix.env}.exs" 23 | -------------------------------------------------------------------------------- /lib/streamz.ex: -------------------------------------------------------------------------------- 1 | defmodule Streamz do 2 | @moduledoc """ 3 | Module for creating and composing streams. 4 | 5 | Streamz is meant to complement the Stream module. 6 | """ 7 | 8 | @doc """ 9 | Creates a composite stream. The stream emits values from the underlying streams, in the 10 | order they are produced. This differs from `Stream.zip/2`, where the order of values is 11 | strictly alternating between the source streams. 12 | 13 | iex(1)> s1 = Stream.repeatedly fn -> 14 | ...(1)> :timer.sleep(100) 15 | ...(1)> :slow 16 | ...(1)> end 17 | #Function<18.77372642/2 in Stream.repeatedly/1> 18 | iex(2)> s2 = Stream.repeatedly fn -> 19 | ...(2)> :timer.sleep(40) 20 | ...(2)> :fast 21 | ...(2)> end 22 | #Function<18.77372642/2 in Stream.repeatedly/1> 23 | iex(3)> Streamz.merge([s1,s2]) |> Enum.take(3) 24 | [:fast, :fast, :slow] 25 | 26 | """ 27 | @spec merge(Enumerable.t) :: Enumerable.t 28 | def merge(streams) do 29 | Streamz.Merge.new(streams) 30 | end 31 | 32 | @doc """ 33 | Takes elements from the first stream until the second stream produces data. 34 | """ 35 | @spec take_until(Enumerable.t, Enumerable.t) :: Enumerable.t 36 | def take_until(stream, cutoff) do 37 | ref = make_ref 38 | wrapped_cutoff = cutoff |> Stream.map fn (_) -> 39 | ref 40 | end 41 | Streamz.merge([stream, wrapped_cutoff]) 42 | |> Stream.take_while &( &1 != ref ) 43 | end 44 | 45 | @doc """ 46 | Takes two streams and creates a combined stream of form {left, right} using the most recent 47 | elements emitted by the two input streams. No elements are emitted from the returned stream 48 | until both input streams have emitted elements. 49 | """ 50 | @spec combine_latest(Enumerable.t, Enumerable.t) :: Enumerable.t 51 | def combine_latest(stream1, stream2) do 52 | left = Stream.map(stream1, &({:left, &1})) 53 | right = Stream.map(stream2, &({:right, &1})) 54 | Streamz.merge([left, right]) 55 | |> Stream.scan({nil,nil}, fn(el, {left, right}) -> 56 | case el do 57 | {:left,_} -> {el,right} 58 | {:right,_} -> {left,el} 59 | end 60 | end) 61 | |> Stream.drop_while(fn {a,b} -> a == nil or b == nil end) 62 | |> Stream.map(fn {{_,a},{_,b}} -> {a,b} end) 63 | end 64 | 65 | @doc """ 66 | Creates a stream that emits one element. 67 | 68 | The element is the result of executing the passed in function. 69 | 70 | iex(1)> Streamz.once(fn -> "foo" end) |> Enum.take(2) 71 | ["foo"] 72 | 73 | """ 74 | @spec once(fun) :: Enumerable.t 75 | def once(fun) do 76 | Stream.unfold true, fn 77 | (true) -> {fun.(), false} 78 | (false) -> nil 79 | end 80 | end 81 | 82 | @doc """ 83 | Take `n` elements and pass them into the given function before continuing to enumerate the stream. 84 | 85 | iex(1)> 1..100 |> Streamz.take_and_continue(5, &IO.inspect/1) |> Enum.take(10) 86 | [1, 2, 3, 4, 5] 87 | [6, 7, 8, 9, 10, 11, 12, 13, 14, 15] 88 | """ 89 | @spec take_and_continue(Enumerable.t, non_neg_integer, (Enumerable.t -> term)) :: Enumerable.t 90 | def take_and_continue(stream, count, fun) do 91 | stream |> Stream.transform [], fn 92 | (el, acc) when is_list(acc) and length(acc) == count -> 93 | fun.(Enum.reverse(acc)) 94 | {[el], nil} 95 | (el, nil) -> 96 | {[el], nil} 97 | (el, acc) -> 98 | {[], [el | acc]} 99 | end 100 | end 101 | 102 | @doc """ 103 | Remove repeated elements. 104 | 105 | iex(1)> Streamz.dedupe([1,1,2,3,2,3,4,5,5]) 106 | [1,2,3,2,3,4,5] 107 | """ 108 | @spec dedupe(Enumerable.t) :: Enumerable.t 109 | def dedupe(stream) do 110 | stream |> Stream.transform nil, fn 111 | (last, last) -> {[], last} 112 | (el, _) -> {[el], el} 113 | end 114 | end 115 | 116 | @doc """ 117 | Spawn `n` processes, one for each element, apply `fun` to each fo them and collect the results. 118 | 119 | Does not preserve order. If order is important, the recommended approach is to tag the elements 120 | with the index and then sort. 121 | 122 | iex(1)> Streamz.pmap(1..5, &(&1 * 2)) |> Enum.to_list 123 | [2,6,4,8,10] 124 | 125 | iex(1)> Stream.with_index(1..5)) 126 | ...(1)> |> Streamz.pmap(fn ({el, i}) -> {el * 2, i} end) 127 | ...(1)> |> Enum.sort_by( &elem(&1,1) ) 128 | ...(1)> |> Enum.map( &elem(&1, 0) ) 129 | [2,4,6,8,10] 130 | """ 131 | def pmap(stream, fun) do 132 | stream 133 | |> Stream.map(fn (el) -> 134 | Streamz.once(fn -> fun.(el) end) 135 | end) 136 | |> Streamz.merge 137 | end 138 | end 139 | -------------------------------------------------------------------------------- /lib/streamz/merge.ex: -------------------------------------------------------------------------------- 1 | defmodule Streamz.Merge do 2 | @moduledoc false 3 | 4 | defstruct streams: [] 5 | 6 | require Streamz.Merge.Helpers 7 | 8 | alias Streamz.Merge.Helpers, as: H 9 | 10 | def new(streams) do 11 | %__MODULE__{streams: streams} 12 | end 13 | 14 | def build_merge_stream(stream) do 15 | Stream.resource( 16 | fn -> start_stream(stream.streams) end, 17 | &next/1, 18 | &stop/1 19 | ) 20 | end 21 | 22 | @type merge_resource :: {reference, pid} 23 | 24 | @spec start_stream([Enumerable.t]) :: merge_resource 25 | defp start_stream(streams) do 26 | ref = make_ref 27 | parent = self 28 | agent = create_state 29 | spawner = spawn_link fn -> 30 | ids = streams |> Enum.flat_map fn (stream) -> 31 | Mergeable.merge(stream, parent, ref) 32 | end 33 | set_sources(agent, ids) 34 | receive do 35 | H.pattern(from, ref, :cleanup) -> 36 | Agent.get(agent, fn (%{sources: sources, completed: completed}) -> 37 | Set.difference(sources, completed) 38 | end) |> Enum.each fn({id, stream}) -> 39 | Mergeable.cleanup(stream, id) 40 | end 41 | :gen.reply(from, :ack) 42 | end 43 | end 44 | {ref, {spawner, agent}} 45 | end 46 | 47 | @spec next(merge_resource) :: {term, merge_resource} | nil 48 | defp next({ref, state = {_, agent}}) do 49 | receive do 50 | H.pattern(from, ref, value) -> 51 | :gen.reply from, :ack 52 | case value do 53 | {:value, value} -> 54 | {[value], {ref, state}} 55 | {:done, id} -> 56 | case add_completed_and_check_state(agent, id) do 57 | true -> {:halt, {ref, state}} 58 | false -> next({ref, state}) 59 | end 60 | end 61 | end 62 | end 63 | 64 | @spec stop(merge_resource) :: :ok 65 | defp stop({ref, {spawner, _}}) do 66 | {:ok, :ack} = H.call(spawner, ref, :cleanup) 67 | H.clear_mailbox(H.pattern(_, ref, _)) 68 | :ok 69 | end 70 | 71 | defp create_state do 72 | {:ok, pid} = Agent.start_link fn -> 73 | %{started: false, sources: HashSet.new, completed: HashSet.new} 74 | end 75 | pid 76 | end 77 | 78 | defp set_sources(pid, sources) do 79 | Agent.update pid, fn(state) -> 80 | %{state | :started => true, :sources => Enum.into(sources, HashSet.new)} 81 | end 82 | end 83 | 84 | defp add_completed_and_check_state(pid, stream) do 85 | Agent.get_and_update pid, fn(state) -> 86 | new_state = %{state | :completed => Set.put(state.completed, stream)} 87 | done = new_state[:started] && Set.equal?(new_state[:completed], new_state[:sources]) 88 | {done, new_state} 89 | end 90 | end 91 | end 92 | 93 | defprotocol Mergeable do 94 | @fallback_to_any true 95 | @type id :: term 96 | @spec merge(Enumerable.t, pid, reference) :: [id] 97 | def merge(stream, target, ref) 98 | 99 | @spec cleanup(Enumerable.t, id) :: :ok 100 | def cleanup(stream, id) 101 | end 102 | 103 | defimpl Mergeable, for: Any do 104 | 105 | require Streamz.Merge.Helpers 106 | 107 | alias Streamz.Merge.Helpers, as: H 108 | 109 | def merge(stream, target, ref) do 110 | id = spawn_link fn -> 111 | stream |> Enum.each fn (el) -> 112 | {:ok, :ack} = H.call(target, ref, {:value, el}, :infinity) 113 | end 114 | {:ok, :ack} = H.call(target, ref, {:done, {self, stream}}, :infinity) 115 | end 116 | [{id, stream}] 117 | end 118 | 119 | def cleanup(_, id) do 120 | mref = Process.monitor(id) 121 | Process.unlink(id) 122 | Process.exit(id, :kill) 123 | receive do 124 | {:DOWN, ^mref, _, _, :killed} -> 125 | end 126 | end 127 | end 128 | 129 | defimpl Mergeable, for: Streamz.Merge do 130 | 131 | defdelegate cleanup(stream, id), to: Mergeable.Any 132 | 133 | def merge(stream, target, ref) do 134 | stream.streams |> Enum.flat_map fn(str) -> 135 | Mergeable.merge(str, target, ref) 136 | end 137 | end 138 | end 139 | 140 | defimpl Enumerable, for: Streamz.Merge do 141 | def reduce(stream, acc, fun) do 142 | Streamz.Merge.build_merge_stream(stream).(acc, fun) 143 | end 144 | 145 | def count(_), do: {:error, __MODULE__} 146 | 147 | def member?(_, _), do: {:error, __MODULE__} 148 | end 149 | -------------------------------------------------------------------------------- /lib/streamz/merge/helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule Streamz.Merge.Helpers do 2 | def call(target, ref, message, timeout \\ 5000) do 3 | :gen.call(target, '$merge', {ref, message}, timeout) 4 | end 5 | 6 | defmacro pattern(from, ref, message) do 7 | quote do 8 | {'$merge', unquote(from), {^unquote(ref), unquote(message)}} 9 | end 10 | end 11 | 12 | @doc """ 13 | A helper macro for clearing out messages that match a certain pattern from an inbox. 14 | """ 15 | defmacro clear_mailbox(pattern) do 16 | quote do 17 | fun1 = fn(fun2, count) -> 18 | receive do 19 | unquote(pattern) -> fun2.(fun2, count+1) 20 | after 21 | 0 -> count 22 | end 23 | end 24 | fun1.(fun1, 0) 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/streamz/task.ex: -------------------------------------------------------------------------------- 1 | defmodule Streamz.Task do 2 | 3 | @spec stream(Enumerable.t) :: Enumerable.t 4 | def stream(funs) do 5 | Stream.map(funs, fn(fun) -> 6 | Streamz.once(fun) 7 | end) |> Streamz.merge 8 | end 9 | 10 | def first_completed_of(funs) do 11 | stream(funs) |> Enum.take(1) |> hd 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Streamz.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :streamz, 7 | version: "0.0.1", 8 | elixir: "~> 1.0.0-rc2", 9 | deps: deps 10 | ] 11 | end 12 | 13 | # Configuration for the OTP application 14 | # 15 | # Type `mix help compile.app` for more information 16 | def application do 17 | [applications: []] 18 | end 19 | 20 | # Dependencies can be hex.pm packages: 21 | # 22 | # {:mydep, "~> 0.3.0"} 23 | # 24 | # Or git/path repositories: 25 | # 26 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1"} 27 | # 28 | # Type `mix help deps` for more examples and options 29 | defp deps do 30 | [] 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /perf/merge_perf_test.exs: -------------------------------------------------------------------------------- 1 | scale = Stream.iterate({1,1}, fn ({old, cur}) -> 2 | {cur, old + cur} 3 | end) |> Stream.map &elem(&1, 1) 4 | 5 | Stream.take(scale, 20) |> Stream.map(fn(count) -> 6 | streams = Stream.cycle([1..1_000_000_000]) |> Stream.take(count) 7 | results = 1..5 |> Enum.map(fn(_) -> 8 | :timer.tc(fn -> 9 | Streamz.merge(streams) |> Enum.take(1_000_000) 10 | end) |> elem(0) 11 | end) 12 | {results, count} 13 | end) |> Enum.each fn({times, index}) -> IO.puts "#{index}\t#{Enum.join(times, "\t")}" end 14 | -------------------------------------------------------------------------------- /perf/nested_merge_perf_test.exs: -------------------------------------------------------------------------------- 1 | scale = Stream.iterate({1,1}, fn ({old, cur}) -> 2 | {cur, old + cur} 3 | end) |> Stream.map &elem(&1, 1) 4 | 5 | Stream.take(scale, 20) |> Stream.map(fn(count) -> 6 | streams = Stream.cycle([1..1_000_000_000]) |> Stream.take(32) |> Streamz.merge 7 | nested_streams = 1..count |> Enum.drop(1) |> Enum.reduce streams, fn 8 | (_, acc) -> Streamz.merge([acc]) 9 | end 10 | results = 1..5 |> Enum.map(fn(_) -> 11 | :timer.tc(fn -> 12 | nested_streams |> Enum.take(1_000_000) 13 | end) |> elem(0) 14 | end) 15 | {results, count} 16 | end) |> Enum.each fn({times, index}) -> IO.puts "#{index}\t#{Enum.join(times, "\t")}" end 17 | -------------------------------------------------------------------------------- /perf/task_stream_test.exs: -------------------------------------------------------------------------------- 1 | scale = Stream.iterate({1,1}, fn ({old, cur}) -> 2 | {cur, old + cur} 3 | end) |> Stream.map(&elem(&1, 1)) |> Stream.drop(1_000_00) 4 | 5 | IO.puts "New" 6 | 7 | [1,2,3,5,8,13,21,34,55,89,144,233,377] |> Enum.each fn (count) -> 8 | {time, _} = :timer.tc fn -> 9 | funs = Stream.cycle([fn -> Enum.take(scale, 10) end]) |> Enum.take(count) 10 | Streamz.tasks(funs) |> Enum.take(count) 11 | end 12 | IO.puts "#{count}\t#{time}" 13 | end 14 | 15 | IO.puts "Old" 16 | 17 | [1,2,3,5,8,13,21,34,55,89,144,233,377] |> Enum.each fn (count) -> 18 | {time, _} = :timer.tc fn -> 19 | funs = Stream.cycle([fn -> Enum.take(scale, 10) end]) |> Enum.take(count) 20 | Streamz.Task.stream(funs) |> Enum.take(count) 21 | end 22 | IO.puts "#{count}\t#{time}" 23 | end 24 | -------------------------------------------------------------------------------- /test/streamz/merge_test.exs: -------------------------------------------------------------------------------- 1 | defmodule MergeTest do 2 | use ExUnit.Case 3 | 4 | test "infinite streams" do 5 | stream_one = Stream.cycle [1,2,3] 6 | stream_two = Stream.cycle [3,4,5] 7 | stream_three = Streamz.merge([stream_one, stream_two]) 8 | values = stream_three |> Stream.drop(100000) |> Enum.take(20) 9 | assert values |> Enum.member?(1) 10 | assert values |> Enum.member?(5) 11 | assert Enum.count(values) == 20 12 | end 13 | 14 | test "finite streams" do 15 | stream_one = [1,2,3] 16 | stream_two = [3,4,5] 17 | stream_three = Streamz.merge([stream_one, stream_two]) 18 | values = stream_three |> Enum.take(10) 19 | assert values |> Enum.member?(1) 20 | assert values |> Enum.member?(5) 21 | assert Enum.count(values) == 6 22 | end 23 | 24 | test "mixed streams" do 25 | stream_one = [1,2,3] 26 | stream_two = Stream.cycle [3,4,5] 27 | stream_three = Streamz.merge([stream_one, stream_two]) 28 | values = stream_three |> Enum.take(10) 29 | assert values |> Enum.member?(1) 30 | assert values |> Enum.member?(5) 31 | assert Enum.count(values) == 10 32 | end 33 | 34 | test "GenEvent streams" do 35 | {:ok, event_one} = GenEvent.start_link 36 | {:ok, event_two} = GenEvent.start_link 37 | events = [event_one, event_two] 38 | task = Task.async fn -> 39 | stream = Streamz.merge(events |> Enum.map &GenEvent.stream(&1) ) 40 | stream |> Enum.take(10) 41 | end 42 | :timer.sleep(50) 43 | 1..10 |> Enum.each fn(x) -> 44 | :timer.sleep(10) # sufficient sleep to preserve ordering 45 | event = Enum.shuffle(events) |> hd 46 | GenEvent.notify(event, x) 47 | end 48 | results = Task.await(task) 49 | assert results == [1,2,3,4,5,6,7,8,9,10] 50 | end 51 | 52 | test "nested merges" do 53 | stream_one = Stream.cycle [1,2,3] 54 | stream_two = Stream.cycle [3,4,5] 55 | stream_three = Streamz.merge([stream_one, stream_two]) 56 | stream_four = Streamz.merge([stream_three, stream_one]) 57 | values = stream_four |> Stream.drop(10000) |> Enum.take(10) 58 | assert values |> Enum.member?(1) 59 | assert values |> Enum.member?(5) 60 | assert Enum.count(values) == 10 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /test/streamz/task_test.exs: -------------------------------------------------------------------------------- 1 | defmodule TaskTest do 2 | use ExUnit.Case 3 | 4 | test "stream - tasks should all execute" do 5 | stream = Streamz.Task.stream [ 6 | fn -> 1 end, 7 | fn -> 2 end, 8 | fn -> 3 end, 9 | fn -> 4 end 10 | ] 11 | results = stream |> Enum.sort 12 | assert results == [1,2,3,4] 13 | end 14 | 15 | test "stream - tasks should return in the order of completion" do 16 | stream = Streamz.Task.stream [ 17 | fn -> 18 | :timer.sleep(10) 19 | 1 20 | end, 21 | fn -> 2 end 22 | ] 23 | result = stream |> Enum.to_list 24 | assert result == [2,1] 25 | end 26 | 27 | test "Stream - TaskStream pid should ignore stray messages" do 28 | stream = Streamz.Task.stream [ 29 | fn -> 30 | :timer.sleep(100) 31 | 1 32 | end, 33 | fn -> 2 end 34 | ] 35 | task = Task.async fn -> 36 | stream |> Enum.to_list 37 | end 38 | #send stream.pid, {:hi, 12} 39 | result = Task.await(task) 40 | assert result == [2,1] 41 | end 42 | 43 | test "first_completed_of - should return value of first completed function" do 44 | value = Streamz.Task.first_completed_of [ 45 | fn -> 46 | :timer.sleep(100) 47 | 1 48 | end, 49 | fn -> 2 end 50 | ] 51 | assert value == 2 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /test/streamz/time_test.exs: -------------------------------------------------------------------------------- 1 | defmodule TimeTest do 2 | use ExUnit.Case 3 | 4 | test "interval" do 5 | stream = Streamz.Time.interval(50) 6 | task = Task.async fn -> 7 | stream |> Enum.take(5) 8 | end 9 | result = Task.await(task, 300) 10 | assert result == [:ok, :ok, :ok, :ok, :ok] 11 | end 12 | 13 | test "timer" do 14 | stream = Streamz.Time.timer(50) 15 | task = Task.async fn -> 16 | stream |> Enum.to_list 17 | end 18 | result = Task.await(task, 150) 19 | assert result == [:ok] 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /test/streamz_test.exs: -------------------------------------------------------------------------------- 1 | defmodule StreamzTest do 2 | use ExUnit.Case 3 | 4 | test "take_until/2" do 5 | {:ok, event_one} = GenEvent.start_link 6 | {:ok, event_two} = GenEvent.start_link 7 | stream = GenEvent.stream(event_one) 8 | cutoff = GenEvent.stream(event_two) 9 | task = Task.async fn -> 10 | Streamz.take_until(stream, cutoff) |> Enum.to_list 11 | end 12 | :timer.sleep(50) 13 | GenEvent.sync_notify(event_one, 1) 14 | GenEvent.sync_notify(event_one, 2) 15 | GenEvent.sync_notify(event_one, 3) 16 | GenEvent.sync_notify(event_two, 4) 17 | GenEvent.sync_notify(event_one, 5) 18 | assert Task.await(task) == [1,2,3] 19 | end 20 | 21 | test "combine_latest/2" do 22 | {:ok, left_event} = GenEvent.start_link 23 | {:ok, right_event} = GenEvent.start_link 24 | left_stream = GenEvent.stream(left_event) 25 | right_stream = GenEvent.stream(right_event) 26 | 27 | combined = Task.async fn -> 28 | Streamz.combine_latest(left_stream, right_stream) 29 | |> Stream.map(fn {a,b} -> %{:color => a.color, :shape => b.shape} end) 30 | |> Enum.take(4) 31 | end 32 | 33 | :timer.sleep(50) 34 | 35 | GenEvent.sync_notify(left_event, %{:color => "red", :shape => "triangle"}) 36 | GenEvent.sync_notify(right_event, %{:color => "yellow", :shape => "square"}) 37 | GenEvent.sync_notify(left_event, %{:color => "blue", :shape => "triangle"}) 38 | GenEvent.sync_notify(right_event, %{:color => "red", :shape => "circle"}) 39 | GenEvent.sync_notify(left_event, %{:color => "green", :shape => "rectangle"}) 40 | 41 | assert Task.await(combined) == [ 42 | %{color: "red", shape: "square"}, 43 | %{color: "blue", shape: "square"}, 44 | %{color: "blue", shape: "circle"}, 45 | %{color: "green", shape: "circle"} 46 | ] 47 | end 48 | 49 | test "once/1" do 50 | s = Streamz.once fn -> 51 | :foo 52 | end 53 | assert Enum.take(s,2) == [:foo] 54 | end 55 | 56 | test "dedupe/1" do 57 | result = Streamz.dedupe([1,2,2,3,2,3,3,4,1,1]) |> Enum.to_list 58 | assert result == [1,2,3,2,3,4,1] 59 | end 60 | 61 | test "take_and_continue/3" do 62 | s = 1..100 |> Streamz.take_and_continue(10, fn(list) -> 63 | assert list == Enum.to_list(1..10) 64 | end) |> Enum.take(10) 65 | assert s == Enum.to_list(11..20) 66 | end 67 | 68 | test "pmap/2" do 69 | start = 1..50 70 | result = Stream.map(start, &(&1 * 2)) |> Enum.into(HashSet.new) 71 | parallel_result = Streamz.pmap(start, &(&1 * 2)) |> Enum.into(HashSet.new) 72 | assert Set.equal?(result, parallel_result) 73 | end 74 | 75 | test "sorting pmap/2" do 76 | start = 1..50 77 | result = Stream.map(start, &(&1 * 2)) |> Enum.to_list 78 | parallel_result = Stream.with_index(start) 79 | |> Streamz.pmap(fn ({el, i}) -> {el*2, i} end) 80 | |> Enum.sort_by( &elem(&1, 1) ) 81 | |> Enum.map &elem(&1, 0) 82 | assert result == parallel_result 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------