├── .credo.exs ├── .dialyzer_ignore.exs ├── .formatter.exs ├── .github └── workflows │ └── main.yml ├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── bin ├── functions ├── lint └── setup ├── config ├── config.exs ├── dev.exs ├── prod.exs └── test.exs ├── lib ├── http_stream.ex └── http_stream │ ├── adapter.ex │ ├── adapter │ ├── httpoison.ex │ └── mint.ex │ └── request.ex ├── mix.exs ├── mix.lock └── test ├── fixtures └── large.tif ├── http_stream └── request_test.exs ├── http_stream_test.exs ├── integration └── large_file_test.exs ├── support ├── http_case.ex ├── http_helpers.ex ├── http_server.ex └── http_server │ ├── endpoint.ex │ ├── respond_with_fixture.ex │ └── respond_with_request.ex └── test_helper.exs /.credo.exs: -------------------------------------------------------------------------------- 1 | checks = [ 2 | {Credo.Check.Design.TagTODO, [exit_status: 0]}, 3 | {Credo.Check.Design.TagFIXME, [exit_status: 0]} 4 | ] 5 | 6 | %{ 7 | configs: [ 8 | %{ 9 | name: "default", 10 | strict: true, 11 | checks: checks, 12 | files: %{ 13 | excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/", "test/"] 14 | } 15 | }, 16 | %{ 17 | name: "test", 18 | strict: true, 19 | files: %{ 20 | included: ["test/"] 21 | }, 22 | checks: [ 23 | {Credo.Check.Readability.ModuleDoc, false} 24 | | checks 25 | ] 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /.dialyzer_ignore.exs: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | import_deps: [], 3 | inputs: ["*.{ex,exs}", "priv/*/seeds.exs", "{config,lib,test}/**/*.{ex,exs}"], 4 | subdirectories: ["priv/*/migrations"], 5 | line_length: 80, 6 | locals_without_parens: [ 7 | # elixir 8 | defstruct: :*, 9 | defmodule: :*, 10 | send: :*, 11 | spawn: :*, 12 | import_if_available: :*, 13 | 14 | # ecto 15 | create: :*, 16 | drop: :*, 17 | remove: :*, 18 | field: :*, 19 | schema: :*, 20 | add: :*, 21 | rename: :*, 22 | modify: :*, 23 | execute: :*, 24 | from: :*, 25 | 26 | # vex 27 | validates: :*, 28 | 29 | # plug 30 | plug: :*, 31 | 32 | # phoenix 33 | pipe_through: :*, 34 | forward: :*, 35 | get: :*, 36 | post: :*, 37 | patch: :*, 38 | put: :*, 39 | delete: :*, 40 | resources: :*, 41 | pipeline: :*, 42 | scope: :*, 43 | socket: :*, 44 | adapter: :*, 45 | action_fallback: :*, 46 | 47 | # absinthe 48 | import_types: :*, 49 | object: :*, 50 | field: :*, 51 | arg: :*, 52 | resolve: :* 53 | ] 54 | ] 55 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - "*" 9 | pull_request: 10 | types: [opened, synchronize] 11 | 12 | jobs: 13 | test: 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | otp: ['22.2', '23.1'] 18 | elixir: ['1.8.2', '1.9.4', '1.10.3'] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - uses: erlef/setup-elixir@v1 23 | with: 24 | otp-version: ${{ matrix.otp }} 25 | elixir-version: ${{ matrix.elixir }} 26 | - run: mix deps.get 27 | - run: mix test 28 | 29 | lint: 30 | runs-on: ubuntu-latest 31 | steps: 32 | - uses: actions/checkout@v2 33 | - uses: erlef/setup-elixir@v1 34 | with: 35 | elixir-version: 1.10.4 36 | otp-version: 23.1 37 | - run: mix deps.get 38 | - run: mix format --check-formatted 39 | - run: mix credo --strict 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | http_stream-*.tar 24 | 25 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at contact@subvisual.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2019, Subvisual 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HTTPStream 2 | 3 | [![Build][build-badge]][build] 4 | 5 | HTTPStream is a tiny, tiny package that wraps HTTP requests into a `Stream` so 6 | that you can manage data on the fly, without keeping everything in memory. 7 | 8 | Downloading an image: 9 | 10 | ```elixir 11 | HTTPStream.get(large_image_url) 12 | |> Stream.into(File.stream!("large_image.png")) 13 | |> Stream.run() 14 | ``` 15 | 16 | Streaming multiple images into a ZIP archive (using [zstream][zstream]) 17 | 18 | ```elixir 19 | [ 20 | Zstream.entry("a.png", HTTPStream.get(a_url)) 21 | Zstream.entry("b.png", HTTPStream.get(b_url)) 22 | ] 23 | |> Zstream.zip() 24 | |> Stream.into(File.stream!("archive.zip")) 25 | |> Stream.run() 26 | ``` 27 | 28 | ## Table of Contents 29 | 30 | * [Installation](#installation) 31 | * [Development](#development) 32 | * [Contributing](#contributing) 33 | * [About](#about) 34 | 35 | ## Installation 36 | 37 | First, you need to add `http_stream` to your list of dependencies on `mix.exs`: 38 | 39 | ```elixir 40 | def deps do 41 | [ 42 | {:http_stream, "~> 1.0.0"}, 43 | 44 | # if using the Mint adapter: 45 | {:castore, "~> 0.1.7"}, 46 | {:mint, "~> 1.1.0"} 47 | 48 | # if using the HTTPoison adapter: 49 | {:httpoison, "~> 1.7.0"} 50 | ] 51 | end 52 | ``` 53 | 54 | HTTPStream comes with two adapters: [`Mint`][mint] and [`HTTPoison`][httpoison]. 55 | By default `Mint` is configured but you need to include it in your dependencies. 56 | 57 | To use `HTTPoison`, set in your `config/config.exs`: 58 | 59 | ```elixir 60 | config :http_stream, adapter: HTTPStream.Adapter.HTTPoison 61 | ``` 62 | 63 | That's it! For more intricate API details, refer to the [documentation][docs]. 64 | 65 | ## Development 66 | 67 | If you want to setup the project for local development, you can just run the 68 | following commands. 69 | 70 | ``` 71 | git clone git@github.com:subvisual/http_stream.git 72 | cd http_stream 73 | bin/setup 74 | ``` 75 | 76 | PRs and issues welcome. 77 | 78 | ## Contributing 79 | 80 | Feel free to contribute! 81 | 82 | If you found a bug, open an issue. You can also open a PR for bugs or new 83 | features. Your PRs will be reviewed and subjected to our styleguide and linters. 84 | 85 | All contributions **must** follow the [Code of Conduct][coc] 86 | and [Subvisual's guides][subvisual-guides]. 87 | 88 | ## About 89 | 90 | HTTPStream is maintained with ❤️ by [Subvisual][subvisual]. 91 | 92 |
93 | 94 | ![Subvisual][subvisual-logo] 95 | 96 | [build-badge]: https://github.com/subvisual/http_stream/workflows/build/badge.svg 97 | [build]: https://github.com/subvisual/http_stream/actions?query=workflow%3Abuild 98 | [zstream]: https://github.com/ananthakumaran/zstream 99 | [mint]: https://github.com/elixir-mint/mint 100 | [httpoison]: https://github.com/edgurgel/httpoison 101 | [docs]: https://hexdocs.pm/http_stream 102 | [subvisual]: https://subvisual.com 103 | [subvisual-guides]: https://github.com/subvisual/guides 104 | [subvisual-logo]: https://raw.githubusercontent.com/subvisual/guides/master/github/templates/logos/blue.png 105 | [coc]: https://github.com/subvisual/http_stream/blob/master/CODE_OF_CONDUCT.md 106 | -------------------------------------------------------------------------------- /bin/functions: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | BLUE='\033[1;34m' 4 | GREEN='\033[1;32m' 5 | YELLOW='\033[1;99m' 6 | RED='\033[1;91m' 7 | RESET='\033[0m' 8 | 9 | pp() { 10 | printf "$1[$2]: $3${RESET}\n" 11 | } 12 | 13 | pp_info() { 14 | pp $BLUE "$1" "$2" 15 | } 16 | 17 | pp_success() { 18 | pp $GREEN "$1" "$2" 19 | } 20 | 21 | pp_error() { 22 | pp $RED "$1" "$2" 23 | } 24 | 25 | pp_warn() { 26 | pp $YELLOW "$1" "$2" 27 | } 28 | 29 | not_installed() { 30 | [ ! -x "$(command -v "$@")" ] 31 | } 32 | 33 | ensure_confirmation() { 34 | read -r "confirmation?please confirm you want to continue [y/n] (default: y)" 35 | confirmation=${confirmation:-"y"} 36 | 37 | if [ "$confirmation" != "y" ]; then 38 | exit 1 39 | fi 40 | } 41 | 42 | -------------------------------------------------------------------------------- /bin/lint: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | set -e 4 | 5 | source "./bin/functions" 6 | 7 | pp_info "lint" "running the elixir formatter..." 8 | mix format --check-formatted --dot-formatter .formatter.exs 9 | 10 | pp_info "lint" "running elixir credo..." 11 | mix credo --strict 12 | 13 | pp_success "lint" "no problems found!" 14 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | set -e 4 | source "./bin/functions" 5 | 6 | env=${1:-"dev"} 7 | 8 | pp_info "setup" "Installing required languages..." 9 | 10 | if not_installed "asdf"; then 11 | pp_error "setup" " 12 | We are using asdf (https://github.com/asdf-vm/asdf) to manage tool 13 | dependencies, since it was not found on your system we cannot ensure that you 14 | are using the correct versions of all the tools. Please install it and run 15 | this script again, or proceed at your own peril. 16 | " 17 | 18 | ensure_confirmation 19 | else 20 | set +e 21 | asdf plugin-add erlang https://github.com/asdf-vm/asdf-erlang.git 2>/dev/null 22 | asdf plugin-add elixir https://github.com/asdf-vm/asdf-elixir.git 2>/dev/null 23 | set -e 24 | 25 | asdf install 26 | fi 27 | 28 | echo "" 29 | pp_info "setup" "Installing elixir dependencies..." 30 | MIX_ENV=$env mix local.hex --force 31 | MIX_ENV=$env mix local.rebar --force 32 | MIX_ENV=$env mix deps.get 33 | 34 | echo "" 35 | pp_success "setup" "You're good to go! Run bin/server to get the development server running." 36 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :http_stream, adapter: HTTPStream.Adapter.Mint 4 | 5 | import_config "#{Mix.env()}.exs" 6 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :http_stream, HTTPStream.HTTPServer, port: 3000 4 | -------------------------------------------------------------------------------- /lib/http_stream.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream do 2 | @moduledoc """ 3 | Main API interface. 4 | 5 | HTTPStream is a tiny tiny library for streaming big big files. It works by 6 | wrapping HTTP requests onto a Stream. You can use it with Flow, write it to 7 | disk through regular streams and more! 8 | 9 | ``` 10 | HTTPStream.get(large_image_url) 11 | |> Stream.into(File.stream!("large_image.png")) 12 | |> Stream.run() 13 | ``` 14 | 15 | The adapter can be configured by setting in your `config/config.exs`: 16 | 17 | ``` 18 | config :http_stream, adapter: HTTPStream.Adapter.Mint 19 | ``` 20 | 21 | At the moment, only two adapters are supported: Mint (default) and HTTPoison. 22 | """ 23 | 24 | alias HTTPStream.Adapter 25 | alias HTTPStream.Request 26 | 27 | @type method :: String.t() 28 | 29 | @doc """ 30 | Performs a GET request. 31 | 32 | Supported options: 33 | 34 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 35 | * `:query` (optional) - Keyword list of query params to add to the request. 36 | """ 37 | 38 | @spec get(String.t(), keyword()) :: Enumerable.t() 39 | def get(url, opts \\ []) do 40 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 41 | query = Keyword.get(opts, :query, []) 42 | 43 | request("GET", url, headers, query) 44 | end 45 | 46 | @doc """ 47 | Performs a DELETE request. 48 | 49 | Supported options: 50 | 51 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 52 | * `:query` (optional) - Keyword list of query params to add to the request. 53 | """ 54 | 55 | @spec delete(String.t(), keyword()) :: Enumerable.t() 56 | def delete(url, opts \\ []) do 57 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 58 | query = Keyword.get(opts, :query, []) 59 | 60 | request("DELETE", url, headers, query) 61 | end 62 | 63 | @doc """ 64 | Performs a OPTIONS request. 65 | 66 | Supported options: 67 | 68 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 69 | * `:query` (optional) - Keyword list of query params to add to the request. 70 | """ 71 | 72 | @spec options(String.t(), keyword()) :: Enumerable.t() 73 | def options(url, opts \\ []) do 74 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 75 | query = Keyword.get(opts, :query, []) 76 | 77 | request("OPTIONS", url, headers, query) 78 | end 79 | 80 | @doc """ 81 | Performs a TRACE request. 82 | 83 | Supported options: 84 | 85 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 86 | * `:query` (optional) - Keyword list of query params to add to the request. 87 | """ 88 | 89 | @spec trace(String.t(), keyword()) :: Enumerable.t() 90 | def trace(url, opts \\ []) do 91 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 92 | query = Keyword.get(opts, :query, []) 93 | 94 | request("TRACE", url, headers, query) 95 | end 96 | 97 | @doc """ 98 | Performs a HEAD request. 99 | 100 | Supported options: 101 | 102 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 103 | * `:query` (optional) - Keyword list of query params to add to the request. 104 | """ 105 | 106 | @spec head(String.t(), keyword()) :: Enumerable.t() 107 | def head(url, opts \\ []) do 108 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 109 | query = Keyword.get(opts, :query, []) 110 | 111 | request("HEAD", url, headers, query) 112 | end 113 | 114 | @doc """ 115 | Performs a POST request. 116 | 117 | Supported options: 118 | 119 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 120 | * `:params` (optional) - Keyword list of query params to add to the request. 121 | """ 122 | 123 | @spec post(String.t(), keyword()) :: Enumerable.t() 124 | def post(url, opts \\ []) do 125 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 126 | params = Keyword.get(opts, :params, "") |> to_json() 127 | 128 | request("POST", url, headers, params) 129 | end 130 | 131 | @doc """ 132 | Performs a PUT request. 133 | 134 | Supported options: 135 | 136 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 137 | * `:params` (optional) - Keyword list of query params to add to the request. 138 | """ 139 | 140 | @spec put(String.t(), keyword()) :: Enumerable.t() 141 | def put(url, opts \\ []) do 142 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 143 | params = Keyword.get(opts, :params, "") |> to_json() 144 | 145 | request("PUT", url, headers, params) 146 | end 147 | 148 | @doc """ 149 | Performs a PATCH request. 150 | 151 | Supported options: 152 | 153 | * `:headers` (optional) - Keyword list of HTTP headers to add to the request. 154 | * `:params` (optional) - Keyword list of query params to add to the request. 155 | """ 156 | 157 | @spec patch(String.t(), keyword()) :: Enumerable.t() 158 | def patch(url, opts \\ []) do 159 | headers = Keyword.get(opts, :headers, []) |> to_keyword() 160 | params = Keyword.get(opts, :params, "") |> to_json() 161 | 162 | request("PATCH", url, headers, params) 163 | end 164 | 165 | @doc """ 166 | Performs an HTTP request. 167 | 168 | Supported methods: GET, OPTIONS, HEAD, TRACE, POST, PUT, PATCH and DELETE 169 | """ 170 | 171 | @spec request(method(), String.t(), keyword(), binary()) :: Enumerable.t() 172 | def request(method, url, headers \\ [], body \\ "") do 173 | Request.new(method, url, headers: headers, body: body) 174 | |> do_request() 175 | end 176 | 177 | defp do_request(%Request{} = request) do 178 | Stream.resource( 179 | fn -> adapter().request(request) end, 180 | &adapter().parse_chunks/1, 181 | &adapter().close/1 182 | ) 183 | end 184 | 185 | defp to_keyword(enum) do 186 | Stream.map(enum, fn {k, v} -> {to_string(k), v} end) 187 | |> Enum.to_list() 188 | end 189 | 190 | defp to_json(term), do: Jason.encode!(term) 191 | 192 | defp adapter, do: Application.get_env(:http_stream, :adapter, Adapter.Mint) 193 | end 194 | -------------------------------------------------------------------------------- /lib/http_stream/adapter.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.Adapter do 2 | @moduledoc """ 3 | Adapter behaviour for HTTPStream compatible clients. 4 | 5 | An adapter must implement the following callbacks: 6 | 7 | * `request/1` - Receives an `HTTPStream.Request.t()` and initiates the HTTP 8 | connection to the endpoint. 9 | * `parse_chunks/1` - Receives values of any type returned by `request/1` and 10 | reads a chunk of data from the connection. 11 | * `close/1` - Called when the final value from `parse_chunks/1` is read. 12 | Should close the connection. 13 | 14 | Currently supported adapters: `HTTPStream.Adapter.Mint` 15 | """ 16 | 17 | alias HTTPStream.Request 18 | 19 | @callback request(Request.t()) :: any() 20 | @callback parse_chunks(any()) :: any() 21 | @callback close(any()) :: :ok | {:error, term()} 22 | end 23 | -------------------------------------------------------------------------------- /lib/http_stream/adapter/httpoison.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(HTTPoison) do 2 | defmodule HTTPStream.Adapter.HTTPoison do 3 | @moduledoc """ 4 | Implements `HTTPStream.Adapter` for the HTTPoison client. 5 | 6 | HTTPoison performs async requests through messages. This means there is an implicit timeout if, for some reason, the server stops responding. By default this timeout is `30_000` (in milliseconds) but can be configured by setting in your `config/config.exs` file: 7 | 8 | ``` 9 | config :http_stream, HTTPStream.Adapter.HTTPoison, timeout: 60_000 10 | 11 | ``` 12 | """ 13 | 14 | alias HTTPStream.Request 15 | 16 | @behaviour HTTPStream.Adapter 17 | 18 | @default_timeout 30_000 19 | 20 | @impl true 21 | def request(%Request{} = request) do 22 | HTTPoison.request( 23 | request.method, 24 | Request.url_for(request), 25 | request.body, 26 | request.headers, 27 | async: :once, 28 | stream_to: self() 29 | ) 30 | end 31 | 32 | @impl true 33 | def parse_chunks({:ok, %HTTPoison.AsyncResponse{} = response}) do 34 | parse_chunks(response) 35 | end 36 | 37 | def parse_chunks(%HTTPoison.AsyncResponse{id: id} = response) do 38 | receive do 39 | %HTTPoison.AsyncChunk{id: ^id, chunk: chunk} -> 40 | HTTPoison.stream_next(response) 41 | {[chunk], response} 42 | 43 | %HTTPoison.AsyncEnd{id: ^id} -> 44 | {:halt, response} 45 | 46 | _ -> 47 | HTTPoison.stream_next(response) 48 | {[], response} 49 | after 50 | timeout() -> 51 | {:halt, response} 52 | end 53 | end 54 | 55 | # See: https://github.com/edgurgel/httpoison/issues/103 56 | @impl true 57 | def close(%HTTPoison.AsyncResponse{id: id}) do 58 | :hackney.stop_async(id) 59 | end 60 | 61 | defp timeout do 62 | config = Application.get_env(:http_stream, __MODULE__) 63 | config[:timeout] || @default_timeout 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/http_stream/adapter/mint.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(Mint.HTTP) do 2 | defmodule HTTPStream.Adapter.Mint do 3 | @moduledoc """ 4 | Implements `HTTPStream.Adapter` for the Mint HTTP Client. 5 | """ 6 | 7 | alias HTTPStream.Request 8 | 9 | @behaviour HTTPStream.Adapter 10 | 11 | @impl true 12 | def request(%Request{} = request) do 13 | with {:ok, conn} <- connect(request), 14 | {:ok, conn, ref} <- do_request(conn, request) do 15 | {conn, ref, :continue} 16 | end 17 | end 18 | 19 | @impl true 20 | def parse_chunks({conn, ref, :halt}) do 21 | {:halt, {conn, ref}} 22 | end 23 | 24 | def parse_chunks({conn, ref, :continue}) do 25 | case Mint.HTTP.recv(conn, 0, :infinity) do 26 | {:ok, conn, chunks} -> 27 | handle_chunks(conn, ref, chunks) 28 | 29 | {:error, conn, _error, chunks} -> 30 | {chunks, {conn, ref, :halt}} 31 | end 32 | end 33 | 34 | @impl true 35 | def close({conn, _ref}), do: do_close(conn) 36 | def close({conn, _ref, :halt}), do: do_close(conn) 37 | 38 | defp connect(%Request{scheme: scheme, host: host, port: port}) do 39 | Mint.HTTP.connect(scheme, host, port, mode: :passive) 40 | end 41 | 42 | defp do_request(conn, request) do 43 | Mint.HTTP.request( 44 | conn, 45 | request.method, 46 | request.path, 47 | request.headers, 48 | request.body 49 | ) 50 | end 51 | 52 | defp handle_chunks(conn, ref, chunks) do 53 | next = 54 | if Enum.any?(chunks, &done?/1) do 55 | :halt 56 | else 57 | :continue 58 | end 59 | 60 | {filter_data(chunks), {conn, ref, next}} 61 | end 62 | 63 | defp do_close(conn) do 64 | Mint.HTTP.close(conn) 65 | :ok 66 | end 67 | 68 | defp filter_data(chunks) do 69 | Stream.filter(chunks, &data?/1) 70 | |> Enum.map(fn {:data, _ref, chunk} -> chunk end) 71 | end 72 | 73 | defp done?(message) do 74 | elem(message, 0) == :done 75 | end 76 | 77 | defp data?(message) do 78 | elem(message, 0) == :data 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/http_stream/request.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.Request do 2 | @moduledoc """ 3 | Struct that represents a request. 4 | 5 | Fields: 6 | 7 | * `scheme`: `atom()` - e.g. `:http` 8 | * `host`: `binary()` - e.g. `"localhost"` 9 | * `port`: `integer()` - e.g. `80` 10 | * `path`: `binary()` - e.g `"/users/1/avatar.png"` 11 | * `method`: `String.t()` - e.g. `"GET"` 12 | * `headers`: `keyword()` - e.g. `[authorization: "Bearer 123"]` 13 | * `body`: `binary()` - e.g. `{ "id": "1" }` 14 | """ 15 | 16 | @supported_methods ~w(GET OPTIONS HEAD TRACE POST PUT PATCH DELETE) 17 | 18 | defstruct scheme: nil, 19 | host: nil, 20 | port: 80, 21 | path: "/", 22 | method: "GET", 23 | headers: [], 24 | body: "" 25 | 26 | @type t :: %__MODULE__{ 27 | scheme: atom() | nil, 28 | host: binary() | nil, 29 | port: integer(), 30 | path: binary(), 31 | method: binary(), 32 | headers: keyword(), 33 | body: binary() 34 | } 35 | 36 | @doc """ 37 | Parses a given URL and uses a given method to generate a valid 38 | `HTTPStream.Request` struct. 39 | 40 | Supported options: 41 | 42 | * `headers` - HTTP headers to be sent. 43 | * `body` - Body of the HTTP request. This will be the request `query` field 44 | if the method is one of "GET", "TRACE", "HEAD", "OPTIONS" and "DELETE". 45 | 46 | This function raises an `ArgumentError` if the HTTP method is unsupported or 47 | the `url` argument isn't a string. 48 | """ 49 | @spec new(String.t(), String.t(), keyword()) :: t() | no_return() 50 | def new(method, url, opts \\ []) 51 | 52 | def new(method, url, opts) 53 | when is_binary(url) and method in @supported_methods do 54 | uri = URI.parse(url) 55 | scheme = String.to_atom(uri.scheme) 56 | headers = Keyword.get(opts, :headers, []) 57 | {body, query} = body_and_query_from_method(method, opts) 58 | path = encode_query_params(uri.path || "/", query) 59 | 60 | %__MODULE__{ 61 | scheme: scheme, 62 | host: uri.host, 63 | port: uri.port, 64 | path: path, 65 | method: method, 66 | headers: headers, 67 | body: body 68 | } 69 | end 70 | 71 | def new(method, _, _) when method not in @supported_methods do 72 | supported_methods = Enum.join(@supported_methods, ", ") 73 | msg = "#{method} is not supported. Supported methods: #{supported_methods}" 74 | 75 | raise ArgumentError, msg 76 | end 77 | 78 | def new(_, _, _) do 79 | raise ArgumentError, "URL must be a string" 80 | end 81 | 82 | def url_for(%__MODULE__{scheme: scheme, host: host, port: port, path: path}) do 83 | [ 84 | scheme, 85 | "://", 86 | host, 87 | ":", 88 | port, 89 | path 90 | ] 91 | |> Enum.join("") 92 | end 93 | 94 | defp encode_query_params(path, []), do: path 95 | 96 | defp encode_query_params(path, query) do 97 | path <> "?" <> URI.encode_query(query) 98 | end 99 | 100 | defp body_and_query_from_method(method, opts) 101 | when method in ~w(GET OPTIONS HEAD TRACE DELETE) do 102 | query = Keyword.get(opts, :body, []) 103 | {"", query} 104 | end 105 | 106 | defp body_and_query_from_method(_, opts) do 107 | body = Keyword.get(opts, :body, "") 108 | {body, []} 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.MixProject do 2 | use Mix.Project 3 | 4 | @version "1.0.0" 5 | @source_url "https://github.com/subvisual/http_stream" 6 | 7 | def project do 8 | [ 9 | app: :http_stream, 10 | version: @version, 11 | elixir: "~> 1.8", 12 | start_permanent: Mix.env() == :prod, 13 | elixirc_paths: elixirc_paths(Mix.env()), 14 | description: description(), 15 | docs: docs(), 16 | deps: deps(), 17 | package: package(), 18 | name: "HTTPStream", 19 | source_url: @source_url, 20 | dialyzer: [ 21 | plt_add_apps: [:mix, :ex_unit], 22 | ignore_warnings: ".dialyzer_ignore.exs" 23 | ] 24 | ] 25 | end 26 | 27 | def application do 28 | [ 29 | extra_applications: [:logger] 30 | ] 31 | end 32 | 33 | defp elixirc_paths(:test), do: ["lib", "test/support"] 34 | defp elixirc_paths(_), do: ["lib"] 35 | 36 | defp deps do 37 | [ 38 | {:castore, "~> 0.1.7", optional: true}, 39 | {:mint, "~> 1.1.0", optional: true}, 40 | {:httpoison, "~> 1.7.0", optional: true}, 41 | {:credo, "~> 1.5.0-rc.2", only: [:dev, :test], runtime: false}, 42 | {:dialyxir, "~> 1.0.0", only: :dev, runtime: false}, 43 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 44 | {:jason, "~> 1.2", only: [:dev, :test]}, 45 | {:plug_cowboy, "~> 2.0", only: :test} 46 | ] 47 | end 48 | 49 | defp package do 50 | [ 51 | licenses: ["ISC"], 52 | links: %{"GitHub" => @source_url}, 53 | files: ~w(.formatter.exs mix.exs README.md CODE_OF_CONDUCT.md lib LICENSE) 54 | ] 55 | end 56 | 57 | defp description do 58 | "A tiny, tiny library to stream big big files. HTTPStream wraps HTTP requests into a Stream." 59 | end 60 | 61 | defp docs do 62 | [ 63 | extras: ["README.md"], 64 | main: "readme", 65 | source_url: @source_url, 66 | source_ref: "v#{@version}" 67 | ] 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, 3 | "castore": {:hex, :castore, "0.1.7", "1ca19eee705cde48c9e809e37fdd0730510752cc397745e550f6065a56a701e9", [:mix], [], "hexpm", "a2ae2c13d40e9c308387f1aceb14786dca019ebc2a11484fb2a9f797ea0aa0d8"}, 4 | "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, 5 | "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"}, 6 | "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"}, 7 | "credo": {:hex, :credo, "1.5.0-rc.2", "41d80beb23da88578ff8fce3f34760f1deab33547fbdd08a232f7c6d8822e405", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "925b0c24a33f915b7a6296f907379156b638b65c33430e5dc16f6ef5ab58ab50"}, 8 | "dialyxir": {:hex, :dialyxir, "1.0.0", "6a1fa629f7881a9f5aaf3a78f094b2a51a0357c843871b8bc98824e7342d00a5", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "aeb06588145fac14ca08d8061a142d52753dbc2cf7f0d00fc1013f53f8654654"}, 9 | "earmark": {:hex, :earmark, "1.4.0", "397e750b879df18198afc66505ca87ecf6a96645545585899f6185178433cc09", [:mix], [], "hexpm", "4bedcec35de03b5f559fd2386be24d08f7637c374d3a85d3fe0911eecdae838a"}, 10 | "earmark_parser": {:hex, :earmark_parser, "1.4.10", "6603d7a603b9c18d3d20db69921527f82ef09990885ed7525003c7fe7dc86c56", [:mix], [], "hexpm", "8e2d5370b732385db2c9b22215c3f59c84ac7dda7ed7e544d7c459496ae519c0"}, 11 | "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, 12 | "ex_doc": {:hex, :ex_doc, "0.22.6", "0fb1e09a3e8b69af0ae94c8b4e4df36995d8c88d5ec7dbd35617929144b62c00", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "1e0aceda15faf71f1b0983165e6e7313be628a460e22a031e32913b98edbd638"}, 13 | "file_system": {:hex, :file_system, "0.2.9", "545b9c9d502e8bfa71a5315fac2a923bd060fd9acb797fe6595f54b0f975fd32", [:mix], [], "hexpm", "3cf87a377fe1d93043adeec4889feacf594957226b4f19d5897096d6f61345d8"}, 14 | "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, 15 | "httpoison": {:hex, :httpoison, "1.7.0", "abba7d086233c2d8574726227b6c2c4f6e53c4deae7fe5f6de531162ce9929a0", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "975cc87c845a103d3d1ea1ccfd68a2700c211a434d8428b10c323dc95dc5b980"}, 16 | "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, 17 | "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"}, 18 | "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"}, 19 | "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"}, 20 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 21 | "mime": {:hex, :mime, "1.4.0", "5066f14944b470286146047d2f73518cf5cca82f8e4815cf35d196b58cf07c47", [:mix], [], "hexpm", "75fa42c4228ea9a23f70f123c74ba7cece6a03b1fd474fe13f6a7a85c6ea4ff6"}, 22 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, 23 | "mint": {:hex, :mint, "1.1.0", "1fd0189edd9e3ffdbd7fcd8bc3835902b987a63ec6c4fd1aa8c2a56e2165f252", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bfd316c3789340b682d5679a8116bcf2112e332447bdc20c1d62909ee45f48d"}, 24 | "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"}, 25 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, 26 | "plug": {:hex, :plug, "1.10.4", "41eba7d1a2d671faaf531fa867645bd5a3dce0957d8e2a3f398ccff7d2ef017f", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad1e233fe73d2eec56616568d260777b67f53148a999dc2d048f4eb9778fe4a0"}, 27 | "plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"}, 28 | "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"}, 29 | "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, 30 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, 31 | "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"}, 32 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, 33 | } 34 | -------------------------------------------------------------------------------- /test/fixtures/large.tif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/subvisual/http_stream/081fb87a7eadc4680560291fb3e39670d47fd115/test/fixtures/large.tif -------------------------------------------------------------------------------- /test/http_stream/request_test.exs: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.RequestTest do 2 | use ExUnit.Case 3 | 4 | alias HTTPStream.Request 5 | 6 | describe "new/3" do 7 | test "generates the correct structure" do 8 | url = "http://localhost:4000" 9 | 10 | assert %Request{ 11 | scheme: :http, 12 | host: "localhost", 13 | port: 4000, 14 | path: "/", 15 | method: "GET", 16 | headers: [], 17 | body: "" 18 | } == Request.new("GET", url) 19 | end 20 | 21 | test "correctly parses query params" do 22 | url = "http://localhost:4000" 23 | query = [id: 1, filter: true] 24 | 25 | assert %Request{ 26 | scheme: :http, 27 | host: "localhost", 28 | port: 4000, 29 | path: "/?id=1&filter=true", 30 | method: "GET", 31 | headers: [], 32 | body: "" 33 | } == Request.new("GET", url, body: query) 34 | end 35 | 36 | test "correctly parses body params" do 37 | url = "http://localhost:4000" 38 | params = %{id: 1, filter: true} 39 | 40 | assert %Request{ 41 | scheme: :http, 42 | host: "localhost", 43 | port: 4000, 44 | path: "/", 45 | method: "POST", 46 | headers: [], 47 | body: params 48 | } == Request.new("POST", url, body: params) 49 | end 50 | end 51 | 52 | describe "url_for/1" do 53 | test "generates the correct URL" do 54 | url = "http://localhost:4000" 55 | query = [id: 1, filter: true] 56 | request = Request.new("GET", url, body: query) 57 | 58 | expected_url = "http://localhost:4000/?id=1&filter=true" 59 | assert expected_url == Request.url_for(request) 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /test/http_stream_test.exs: -------------------------------------------------------------------------------- 1 | defmodule HTTPStreamTest do 2 | use HTTPStream.HTTPCase 3 | doctest HTTPStream 4 | 5 | alias HTTPStream.Adapter 6 | 7 | for adapter <- [Adapter.Mint, Adapter.Poison] do 8 | Application.put_env(HTTPStream, :adapter, adapter) 9 | 10 | for method <- ~w(get options delete)a do 11 | describe "[#{adapter}]: #{method}/2" do 12 | upcased_method = method |> to_string() |> String.upcase() 13 | 14 | test "makes a #{upcased_method} request" do 15 | %{"method" => method} = 16 | apply(HTTPStream, unquote(method), ["http://localhost:3000"]) 17 | |> parse_response() 18 | 19 | assert method == unquote(upcased_method) 20 | end 21 | 22 | test "sets the correct headers" do 23 | headers = [authorization: "Bearer 123"] 24 | 25 | %{"headers" => headers} = 26 | apply(HTTPStream, unquote(method), [ 27 | "http://localhost:3000", 28 | [headers: headers] 29 | ]) 30 | |> parse_response() 31 | 32 | assert headers["authorization"] == "Bearer 123" 33 | end 34 | 35 | test "sets the query params" do 36 | params = [email: "user@example.org"] 37 | 38 | %{"params" => params} = 39 | apply(HTTPStream, unquote(method), [ 40 | "http://localhost:3000", 41 | [query: params] 42 | ]) 43 | |> parse_response() 44 | 45 | assert params["email"] == "user@example.org" 46 | end 47 | end 48 | end 49 | 50 | for method <- ~w(head trace)a do 51 | describe "#{adapter}: #{method}/2" do 52 | upcased_method = method |> to_string() |> String.upcase() 53 | 54 | test "makes a #{upcased_method} request" do 55 | response = 56 | apply(HTTPStream, unquote(method), ["http://localhost:3000"]) 57 | |> Enum.join("") 58 | 59 | assert response == "" 60 | end 61 | 62 | test "sets the correct headers" do 63 | headers = [authorization: "Bearer 123"] 64 | 65 | response = 66 | apply(HTTPStream, unquote(method), [ 67 | "http://localhost:3000", 68 | [headers: headers] 69 | ]) 70 | |> Enum.join("") 71 | 72 | assert response == "" 73 | end 74 | 75 | test "sets the query params" do 76 | params = [email: "user@example.org"] 77 | 78 | response = 79 | apply(HTTPStream, unquote(method), [ 80 | "http://localhost:3000", 81 | [query: params] 82 | ]) 83 | |> Enum.join("") 84 | 85 | assert response == "" 86 | end 87 | end 88 | end 89 | 90 | for method <- ~w(post put patch)a do 91 | describe "#{adapter}: #{method}/2" do 92 | upcased_method = method |> to_string() |> String.upcase() 93 | 94 | test "makes a #{upcased_method} request" do 95 | headers = [ 96 | authorization: "Bearer 123", 97 | "content-type": "application/json" 98 | ] 99 | 100 | %{"method" => method} = 101 | apply(HTTPStream, unquote(method), [ 102 | "http://localhost:3000", 103 | [headers: headers] 104 | ]) 105 | |> parse_response() 106 | 107 | assert method == unquote(upcased_method) 108 | end 109 | 110 | test "sets the correct headers" do 111 | headers = [ 112 | authorization: "Bearer 123", 113 | "content-type": "application/json" 114 | ] 115 | 116 | %{"headers" => headers} = 117 | apply(HTTPStream, unquote(method), [ 118 | "http://localhost:3000", 119 | [headers: headers] 120 | ]) 121 | |> parse_response() 122 | 123 | assert headers["authorization"] == "Bearer 123" 124 | end 125 | 126 | test "sets the params" do 127 | params = %{email: "user@example.org"} 128 | headers = ["content-type": "application/json"] 129 | 130 | %{"params" => params} = 131 | apply(HTTPStream, unquote(method), [ 132 | "http://localhost:3000", 133 | [headers: headers, params: params] 134 | ]) 135 | |> parse_response() 136 | 137 | assert params["email"] == "user@example.org" 138 | end 139 | end 140 | end 141 | end 142 | end 143 | -------------------------------------------------------------------------------- /test/integration/large_file_test.exs: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.Integration.LargeFileTest do 2 | use HTTPStream.HTTPCase 3 | 4 | alias HTTPStream.HTTPServer.RespondWithFixture 5 | 6 | @output_path "out.tif" 7 | 8 | describe "large file download" do 9 | @describetag respond_with: RespondWithFixture 10 | 11 | test "streaming to another process" do 12 | pid = self() 13 | 14 | spawn fn -> 15 | HTTPStream.get("http://localhost:3000") 16 | |> Stream.map(fn chunk -> 17 | send pid, {:data, chunk} 18 | end) 19 | |> Stream.run() 20 | end 21 | 22 | assert_receive {:data, _chunk} 23 | assert_receive {:data, _chunk} 24 | end 25 | 26 | test "streaming to the file system" do 27 | on_exit(fn -> 28 | File.rm!(@output_path) 29 | end) 30 | 31 | spawn fn -> 32 | HTTPStream.get("http://localhost:3000") 33 | |> Stream.into(File.stream!(@output_path)) 34 | |> Stream.run() 35 | end 36 | 37 | # Ugly but needed to wait for the file to be created 38 | Process.sleep(100) 39 | 40 | {:ok, %{size: size_1}} = File.stat(@output_path) 41 | Process.sleep(100) 42 | {:ok, %{size: size_2}} = File.stat(@output_path) 43 | Process.sleep(100) 44 | {:ok, %{size: size_3}} = File.stat(@output_path) 45 | 46 | assert size_2 > size_1 47 | assert size_3 > size_2 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /test/support/http_case.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPCase do 2 | use ExUnit.CaseTemplate, async: false 3 | 4 | alias HTTPStream.HTTPServer 5 | 6 | using do 7 | quote do 8 | import HTTPStream.HTTPHelpers 9 | end 10 | end 11 | 12 | setup tags do 13 | config = Application.get_env(:http_stream, HTTPServer) 14 | 15 | response_module = 16 | case tags[:respond_with] do 17 | nil -> HTTPServer.RespondWithRequest 18 | module -> module 19 | end 20 | 21 | Application.put_env(:http_stream, HTTPServer, 22 | port: config[:port], 23 | respond_with: response_module 24 | ) 25 | 26 | {:ok, _pid} = HTTPServer.start() 27 | 28 | on_exit(fn -> 29 | HTTPServer.stop() 30 | Application.put_env(:http_stream, HTTPServer, config) 31 | end) 32 | 33 | :ok 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /test/support/http_helpers.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPHelpers do 2 | def parse_response(stream) do 3 | stream 4 | |> Enum.join("") 5 | |> Jason.decode!() 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /test/support/http_server.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPServer do 2 | alias HTTPStream.HTTPServer.Endpoint 3 | 4 | @default_port 3000 5 | 6 | def start(port \\ @default_port) do 7 | Plug.Cowboy.http(Endpoint, [], port: port) 8 | end 9 | 10 | def stop, do: Plug.Cowboy.shutdown(Endpoint.HTTP) 11 | end 12 | -------------------------------------------------------------------------------- /test/support/http_server/endpoint.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPServer.Endpoint do 2 | use Plug.Router 3 | 4 | alias HTTPStream.HTTPServer 5 | 6 | plug Plug.Parsers, parsers: [:json], json_decoder: Jason 7 | plug :match 8 | plug :dispatch 9 | 10 | match _ do 11 | response_module = 12 | Application.get_env(:http_stream, HTTPServer)[:respond_with] 13 | 14 | response_module.call(conn) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /test/support/http_server/respond_with_fixture.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPServer.RespondWithFixture do 2 | @fixture Path.dirname(__DIR__) 3 | |> Path.join("../fixtures/large.tif") 4 | |> Path.expand() 5 | 6 | import Plug.Conn 7 | 8 | def call(conn) do 9 | conn = 10 | conn 11 | |> put_resp_content_type("image/event-stream") 12 | |> put_resp_header( 13 | "Content-disposition", 14 | "attachment; filename=\"large.tif\"" 15 | ) 16 | |> put_resp_header("Content-Type", "application/octet-stream") 17 | |> send_chunked(200) 18 | 19 | File.stream!(@fixture) 20 | |> Stream.map(&chunk(conn, &1)) 21 | |> Stream.run() 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/support/http_server/respond_with_request.ex: -------------------------------------------------------------------------------- 1 | defmodule HTTPStream.HTTPServer.RespondWithRequest do 2 | def call(conn) do 3 | headers = Enum.into(conn.req_headers, %{}) 4 | params = Enum.into(conn.params, %{}) 5 | method = conn.method 6 | 7 | request = %{ 8 | headers: headers, 9 | params: params, 10 | method: method 11 | } 12 | 13 | response = Jason.encode!(request) 14 | 15 | Plug.Conn.send_resp(conn, 200, response) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------