├── .formatter.exs ├── .github └── workflows │ ├── build.yml │ └── publish.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── bin ├── publish ├── setup └── test ├── config └── config.exs ├── docker-compose.yml ├── lib ├── file_store.ex └── file_store │ ├── adapters │ ├── disk.ex │ ├── memory.ex │ ├── null.ex │ └── s3.ex │ ├── config.ex │ ├── error.ex │ ├── middleware.ex │ ├── middleware │ ├── errors.ex │ ├── logger.ex │ └── prefix.ex │ ├── stat.ex │ └── utils.ex ├── mix.exs ├── mix.lock └── test ├── file_store ├── adapters │ ├── disk_test.exs │ ├── memory_test.exs │ ├── null_test.exs │ └── s3_test.exs ├── config_test.exs ├── error_test.exs ├── middleware │ ├── errors_test.exs │ ├── logger_test.exs │ └── prefix_test.exs ├── stat_test.exs └── utils_test.exs ├── file_store_test.exs ├── fixtures └── test.txt ├── support ├── adapter_case.ex └── error_adapter.ex └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | on: [push] 3 | jobs: 4 | # Lint should only run against the latest versions. 5 | lint: 6 | name: Lint 7 | runs-on: ${{ matrix.os }} 8 | env: 9 | MIX_ENV: dev 10 | strategy: 11 | matrix: 12 | os: ["ubuntu-20.04"] 13 | elixir: ["1.16"] 14 | otp: ["26"] 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v2 18 | 19 | - name: Install Elixir 20 | uses: erlef/setup-beam@v1 21 | with: 22 | otp-version: ${{ matrix.otp }} 23 | elixir-version: ${{ matrix.elixir }} 24 | 25 | - name: Cache dependencies 26 | uses: actions/cache@v2 27 | with: 28 | path: deps 29 | key: ${{ matrix.os }}-otp_${{ matrix.otp }}-elixir_${{ matrix.elixir }}-mix_${{ hashFiles('**/mix.lock') }} 30 | restore-keys: ${{ matrix.os }}-otp_${{ matrix.otp }}-elixir_${{ matrix.elixir }}-mix_ 31 | 32 | - name: Install depenencies 33 | run: mix do deps.get, deps.compile 34 | 35 | - name: Check formatting 36 | run: mix format --check-formatted 37 | 38 | - name: Check for unused dependencies 39 | run: mix deps.unlock --check-unused 40 | 41 | - name: Cache dialyzer 42 | uses: actions/cache@v2 43 | with: 44 | path: priv/plts 45 | key: plts-otp_${{ matrix.otp }}-elixir_${{ matrix.elixir }} 46 | 47 | - name: Dialyzer 48 | run: mix dialyzer 49 | 50 | - name: Credo 51 | run: mix credo --all 52 | 53 | # Build phase to check against multiple versions of OTP and elixir for as 54 | # much compatibility as we can stand. 55 | build: 56 | name: Test Elixir ${{ matrix.elixir }}, OTP ${{ matrix.otp }}, OS ${{ matrix.os }} 57 | runs-on: ${{ matrix.os }} 58 | env: 59 | MIX_ENV: test 60 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 61 | strategy: 62 | fail-fast: false 63 | matrix: 64 | os: ["ubuntu-20.04"] 65 | elixir: ["1.16", "1.15", "1.14"] 66 | otp: ["26", "25", "24"] 67 | steps: 68 | - name: Checkout 69 | uses: actions/checkout@v2 70 | 71 | - name: Start services 72 | run: docker compose up -d --wait 73 | 74 | - name: Install Elixir 75 | uses: erlef/setup-beam@v1 76 | with: 77 | otp-version: ${{ matrix.otp }} 78 | elixir-version: ${{ matrix.elixir }} 79 | 80 | - name: Cache dependencies 81 | uses: actions/cache@v2 82 | with: 83 | path: deps 84 | key: ${{ matrix.os }}-otp_${{ matrix.otp }}-elixir_${{ matrix.elixir }}-mix_${{ hashFiles('**/mix.lock') }} 85 | restore-keys: ${{ matrix.os }}-otp_${{ matrix.otp }}-elixir_${{ matrix.elixir }}-mix_ 86 | 87 | - name: Install dependencies 88 | run: mix do deps.get, deps.compile 89 | 90 | - name: Compile 91 | run: mix compile --force --warnings-as-errors 92 | 93 | - name: Test 94 | run: mix coveralls.github 95 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | publish: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v2 11 | 12 | - name: Setup Elixir 13 | uses: erlef/setup-beam@v1 14 | with: 15 | otp-version: "24" 16 | elixir-version: "1.13" 17 | 18 | - name: Dependencies 19 | run: mix deps.get 20 | 21 | - name: Publish 22 | run: bin/publish "${GITHUB_REF:11}" 23 | env: 24 | HEX_API_KEY: ${{ secrets.HEX_TOKEN }} 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | file_store-*.tar 24 | 25 | # Ignore ElixirLS temporary files 26 | /.elixir_ls 27 | 28 | # Ignore Dialyzer temporary files 29 | /priv/plts 30 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Please see [GitHub Releases](https://github.com/rzane/file_store/releases). 4 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Ray Zane 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FileStore 2 | 3 | [![github.com](https://img.shields.io/github/workflow/status/rzane/file_store/Build.svg)](https://github.com/rzane/file_store/actions?query=workflow%3ABuild) 4 | [![coveralls.io](https://img.shields.io/coveralls/github/rzane/file_store.svg)](https://coveralls.io/github/rzane/file_store) 5 | [![hex.pm](https://img.shields.io/hexpm/v/file_store.svg)](https://hex.pm/packages/file_store) 6 | [![hex.pm](https://img.shields.io/hexpm/dt/file_store.svg)](https://hex.pm/packages/file_store) 7 | [![hex.pm](https://img.shields.io/hexpm/l/file_store.svg)](https://hex.pm/packages/file_store) 8 | [![github.com](https://img.shields.io/github/last-commit/rzane/file_store.svg)](https://github.com/rzane/file_store/commits/master) 9 | 10 | FileStore allows you to read, write, upload, download, and interact with files, regardless of where they are stored. 11 | 12 | It includes adapters for the following storage backends: 13 | 14 | - [Disk](https://hexdocs.pm/file_store/FileStore.Adapters.Disk.html) 15 | - [S3](https://hexdocs.pm/file_store/FileStore.Adapters.S3.html) 16 | - [Memory](https://hexdocs.pm/file_store/FileStore.Adapters.Memory.html) 17 | - [Null](https://hexdocs.pm/file_store/FileStore.Adapters.Null.html) 18 | 19 | > [**View the documentation**](https://hexdocs.pm/file_store) 20 | 21 | ## Installation 22 | 23 | The package can be installed by adding `file_store` to your list of dependencies in `mix.exs`: 24 | 25 | ```elixir 26 | def deps do 27 | [{:file_store, "~> 0.3"}] 28 | end 29 | ``` 30 | 31 | ## Usage 32 | 33 | Configure a new store: 34 | 35 | ```elixir 36 | store = FileStore.Adapters.Disk.new( 37 | storage_path: "/path/to/store/files", 38 | base_url: "http://example.com/files/" 39 | ) 40 | ``` 41 | 42 | Now, you can manipulate files in your store: 43 | 44 | ```elixir 45 | iex> FileStore.upload(store, "hello.txt", "world.txt") 46 | :ok 47 | 48 | iex> FileStore.read(store, "world.txt") 49 | {:ok, "hello world"} 50 | 51 | iex> FileStore.stat(store, "world.txt") 52 | {:ok, 53 | %FileStore.Stat{ 54 | etag: "5eb63bbbe01eeed093cb22bb8f5acdc3", 55 | key: "hello.txt", 56 | size: 11, 57 | type: "application/octet-stream" 58 | }} 59 | 60 | iex> FileStore.get_public_url(store, "world.txt") 61 | "http://localhost:4000/world.txt" 62 | ``` 63 | 64 | [Click here to see all available operations.](https://hexdocs.pm/file_store/FileStore.html#summary) 65 | 66 | ## Middleware 67 | 68 | #### Logger 69 | 70 | To enable logging, just wrap your store with the logging middleware: 71 | 72 | ```elixir 73 | iex> store 74 | ...> |> FileStore.Middleware.Logger.new() 75 | ...> |> FileStore.read("test.txt") 76 | # 02:37:30.724 [debug] READ OK key="test.txt" 77 | {:ok, "hello"} 78 | ``` 79 | 80 | #### Errors 81 | 82 | The errors middleware will wrap error values: 83 | 84 | ```elixir 85 | iex> store 86 | ...> |> FileStore.Middleware.Errors.new() 87 | ...> |> FileStore.read("bizcorp.jpg") 88 | {:error, %FileStore.Error{...}} 89 | ``` 90 | 91 | One of the following structs will be returned: 92 | 93 | - `FileStore.Error` 94 | - `FileStore.UploadError` 95 | - `FileStore.DownloadError` 96 | - `FileStore.CopyError` 97 | - `FileStore.RenameError` 98 | 99 | Because the error implements the `Exception` behaviour, you can `raise` it. 100 | 101 | #### Prefix 102 | 103 | The prefix middleware allows you to prepend a prefix to all operations. 104 | 105 | ```elixir 106 | iex> store 107 | ...> |> FileStore.Middleware.Prefix.new(prefix: "company/logos") 108 | ...> |> FileStore.read("bizcorp.jpg") 109 | ``` 110 | 111 | In the example above, `bizcorp.jpg` was translated to `companies/logos/bizcorp.jpg`. 112 | 113 | ## Creating a store 114 | 115 | You can also create a dedicated store in your application. 116 | 117 | ```elixir 118 | defmodule MyApp.Storage do 119 | use FileStore.Config, otp_app: :my_app 120 | end 121 | ``` 122 | 123 | You'll need to provide configuration for this module: 124 | 125 | ```elixir 126 | config :my_app, MyApp.Storage, 127 | adapter: FileStore.Adapters.Null, 128 | middleware: [FileStore.Middleware.Errors] 129 | ``` 130 | 131 | Now, you can interact with your store more conveniently: 132 | 133 | ```elixir 134 | iex> MyApp.Storage.write("foo", "hello world") 135 | :ok 136 | 137 | iex> MyApp.Storage.read("foo") 138 | {:ok, "hello world"} 139 | ``` 140 | 141 | ## Contributing 142 | 143 | In order to run the test suite, you'll need [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/). Docker is used to run services like Minio locally, so that we can integration test the S3 adapter. 144 | 145 | To install dependencies and start services: 146 | 147 | $ bin/setup 148 | 149 | Run the test suite: 150 | 151 | $ bin/test 152 | -------------------------------------------------------------------------------- /bin/publish: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | if [ $# -ne 1 ]; then 6 | echo "Missing required argument: version" 7 | exit 1 8 | fi 9 | 10 | perl -pi -e "s/@version \"0.0.0\"/@version \"$1\"/" mix.exs 11 | mix hex.publish --yes 12 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | say() { 6 | printf "\e[33m$1\e[0m\n" 7 | } 8 | 9 | say "==>> Installing dependencies..." 10 | mix deps.get 11 | 12 | say "\n==>> Starting services..." 13 | docker compose up -d --wait 14 | -------------------------------------------------------------------------------- /bin/test: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | 5 | say() { 6 | printf "\e[33m$1\e[0m\n" 7 | } 8 | 9 | say "==>> Compiling..." 10 | mix compile --force --warnings-as-errors 11 | 12 | say "\n==>> Checking for proper formatting..." 13 | mix format --check-formatted 14 | 15 | say "\n==>> Checking for unused deps in the lockfile..." 16 | mix deps.unlock --check-unused 17 | 18 | say "\n==>> Running credo..." 19 | mix credo --all 20 | 21 | say "\n==>> Running the test suite..." 22 | mix test 23 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | import Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # third-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure your application as: 12 | # 13 | # config :file_store, key: :value 14 | # 15 | # and access this configuration in your application as: 16 | # 17 | # Application.get_env(:file_store, :key) 18 | # 19 | # You can also configure a third-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env()}.exs" 31 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: minio/minio:RELEASE.2020-10-18T21-54-12Z 4 | command: ["server", "/data"] 5 | ports: ["9000:9000"] 6 | environment: 7 | MINIO_ACCESS_KEY: development 8 | MINIO_SECRET_KEY: development 9 | healthcheck: 10 | test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] 11 | -------------------------------------------------------------------------------- /lib/file_store.ex: -------------------------------------------------------------------------------- 1 | defprotocol FileStore do 2 | @moduledoc """ 3 | FileStore allows you to read, write, upload, download, and interact 4 | with files, regardless of where they are stored. 5 | 6 | ## Adapters 7 | 8 | This package ships with the following adapters: 9 | 10 | * `FileStore.Adapters.Disk` 11 | * `FileStore.Adapters.S3` 12 | * `FileStore.Adapters.Memory` 13 | * `FileStore.Adapters.Null` 14 | 15 | The documentation for each adapter includes an example that demonstrates 16 | it's usage. 17 | """ 18 | 19 | @type key :: binary() 20 | @type list_opts :: [{:prefix, binary()}] 21 | @type delete_all_opts :: [{:prefix, binary()}] 22 | @type write_opts :: [ 23 | {:content_type, binary()} 24 | | {:disposition, binary()} 25 | ] 26 | 27 | @type public_url_opts :: [ 28 | {:content_type, binary()} 29 | | {:disposition, binary()} 30 | ] 31 | 32 | @type signed_url_opts :: [ 33 | {:content_type, binary()} 34 | | {:disposition, binary()} 35 | | {:expires_in, integer()} 36 | ] 37 | 38 | @doc """ 39 | Write a file to the store. If a file with the given `key` 40 | already exists, it will be overwritten. 41 | 42 | ## Options 43 | 44 | * `:content_type` - Sets the content type hint for the adapter. 45 | * `:disposition` - Sets the content disposition hint for the adapter. 46 | 47 | ## Examples 48 | 49 | iex> FileStore.write(store, "foo", "hello world") 50 | :ok 51 | 52 | """ 53 | @spec write(t, key, binary, write_opts) :: :ok | {:error, term} 54 | def write(store, key, content, opts \\ []) 55 | 56 | @doc """ 57 | Read the contents of a file in store into memory. 58 | 59 | ## Examples 60 | 61 | iex> FileStore.read(store, "foo") 62 | {:ok, "hello world"} 63 | 64 | """ 65 | @spec read(t, key) :: {:ok, binary} | {:error, term} 66 | def read(store, key) 67 | 68 | @doc """ 69 | Upload a file to the store. If a file with the given `key` 70 | already exists, it will be overwritten. 71 | 72 | ## Examples 73 | 74 | iex> FileStore.upload(store, "/path/to/bar.txt", "foo") 75 | :ok 76 | 77 | """ 78 | @spec upload(t, Path.t(), key) :: :ok | {:error, term} 79 | def upload(store, source, key) 80 | 81 | @doc """ 82 | Download a file from the store and save it to the given `path`. 83 | 84 | ## Examples 85 | 86 | iex> FileStore.download(store, "foo", "/path/to/bar.txt") 87 | :ok 88 | 89 | """ 90 | @spec download(t, key, Path.t()) :: :ok | {:error, term} 91 | def download(store, key, destination) 92 | 93 | @doc """ 94 | Retrieve information about a file from the store. 95 | 96 | ## Examples 97 | 98 | iex> FileStore.stat(store, "foo") 99 | {:ok, %FileStore.Stat{key: "foo", etag: "2e5pd429", size: 24}} 100 | 101 | """ 102 | @spec stat(t, key) :: {:ok, FileStore.Stat.t()} | {:error, term} 103 | def stat(store, key) 104 | 105 | @doc """ 106 | Delete a file from the store. 107 | 108 | ## Examples 109 | 110 | iex> FileStore.delete(store, "foo") 111 | :ok 112 | 113 | """ 114 | @spec delete(t, key) :: :ok | {:error, term} 115 | def delete(store, key) 116 | 117 | @doc """ 118 | Delete files in bulk. 119 | 120 | ## Options 121 | 122 | * `:prefix` - Only delete keys matching the given prefix. 123 | 124 | ## Examples 125 | 126 | iex> FileStore.delete_all(store) 127 | :ok 128 | 129 | iex> FileStore.delete_all(store, prefix: "foo/") 130 | :ok 131 | 132 | """ 133 | @spec delete_all(t, delete_all_opts) :: :ok | {:error, term} 134 | def delete_all(store, opts \\ []) 135 | 136 | @doc """ 137 | Copy a file to a new location. 138 | 139 | ## Examples 140 | 141 | iex> FileStore.copy(store, "path/foo.txt", "path/bar.txt") 142 | :ok 143 | 144 | """ 145 | @spec copy(t(), key(), key()) :: :ok | {:error, term()} 146 | def copy(store, src, dest) 147 | 148 | @doc """ 149 | Renames a file from one name to another. 150 | 151 | **Note**: Some underlying adapters can not do this in an atomic fashion. 152 | 153 | ## Examples 154 | 155 | iex> FileStore.rename(store, "path/foo.txt", "path/bar.txt") 156 | :ok 157 | 158 | """ 159 | @spec rename(t(), key(), key()) :: :ok | {:error, term()} 160 | def rename(store, src, dest) 161 | 162 | @doc """ 163 | Get URL for your file, assuming that the file is publicly accessible. 164 | 165 | ## Options 166 | 167 | * `:content_type` - Force the `Content-Type` of the response. 168 | * `:disposition` - Force the `Content-Disposition` of the response. 169 | 170 | ## Examples 171 | 172 | iex> FileStore.get_public_url(store, "foo") 173 | "https://mybucket.s3-us-east-1.amazonaws.com/foo" 174 | 175 | """ 176 | @spec get_public_url(t, key, public_url_opts) :: binary 177 | def get_public_url(store, key, opts \\ []) 178 | 179 | @doc """ 180 | Generate a signed URL for your file. Any user with this URL should be able 181 | to access the file. 182 | 183 | ## Options 184 | 185 | * `:expires_in` - The number of seconds before the URL expires. 186 | * `:content_type` - Force the `Content-Type` of the response. 187 | * `:disposition` - Force the `Content-Disposition` of the response. 188 | 189 | ## Examples 190 | 191 | iex> FileStore.get_signed_url(store, "foo") 192 | {:ok, "https://s3.amazonaws.com/mybucket/foo?X-AMZ-Expires=3600&..."} 193 | 194 | """ 195 | @spec get_signed_url(t, key, signed_url_opts) :: {:ok, binary} | {:error, term} 196 | def get_signed_url(store, key, opts \\ []) 197 | 198 | @doc """ 199 | List files in the store. 200 | 201 | ## Options 202 | 203 | * `:prefix` - Only return keys matching the given prefix. 204 | 205 | ## Examples 206 | 207 | iex> Enum.to_list(FileStore.list!(store)) 208 | ["bar", "foo/bar"] 209 | 210 | iex> Enum.to_list(FileStore.list!(store, prefix: "foo")) 211 | ["foo/bar"] 212 | 213 | """ 214 | @spec list!(t, list_opts) :: Enumerable.t() 215 | def list!(store, opts \\ []) 216 | end 217 | -------------------------------------------------------------------------------- /lib/file_store/adapters/disk.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.Disk do 2 | @moduledoc """ 3 | Stores files on the local disk. This is primarily intended for development. 4 | 5 | ### Configuration 6 | 7 | * `storage_path` - The path on disk where files are 8 | stored. This option is required. 9 | 10 | * `base_url` - The base URL that should be used for 11 | generating URLs to your files. 12 | 13 | ### Example 14 | 15 | iex> store = FileStore.Adapters.Disk.new( 16 | ...> storage_path: "/path/to/store/files", 17 | ...> base_url: "http://example.com/files/" 18 | ...> ) 19 | %FileStore.Adapters.Disk{...} 20 | 21 | iex> FileStore.write(store, "foo", "hello world") 22 | :ok 23 | 24 | iex> FileStore.read(store, "foo") 25 | {:ok, "hello world"} 26 | 27 | """ 28 | 29 | @enforce_keys [:storage_path, :base_url] 30 | defstruct [:storage_path, :base_url] 31 | 32 | @doc "Create a new disk adapter" 33 | @spec new(keyword) :: FileStore.t() 34 | def new(opts) do 35 | if is_nil(opts[:storage_path]) do 36 | raise "missing configuration: :storage_path" 37 | end 38 | 39 | if is_nil(opts[:base_url]) do 40 | raise "missing configuration: :base_url" 41 | end 42 | 43 | struct(__MODULE__, opts) 44 | end 45 | 46 | @doc "Get an the path for a given key." 47 | @spec join(FileStore.t(), binary) :: Path.t() 48 | def join(store, key) do 49 | Path.join(store.storage_path, key) 50 | end 51 | 52 | defimpl FileStore do 53 | alias FileStore.Stat 54 | alias FileStore.Utils 55 | alias FileStore.Adapters.Disk 56 | 57 | def get_public_url(store, key, opts) do 58 | query = Keyword.take(opts, [:content_type, :disposition]) 59 | 60 | store.base_url 61 | |> URI.parse() 62 | |> Utils.append_path(key) 63 | |> Utils.put_query(query) 64 | |> URI.to_string() 65 | end 66 | 67 | def get_signed_url(store, key, opts) do 68 | {:ok, get_public_url(store, key, opts)} 69 | end 70 | 71 | def stat(store, key) do 72 | with path <- Disk.join(store, key), 73 | {:ok, stat} <- File.stat(path), 74 | {:ok, etag} <- FileStore.Stat.checksum_file(path) do 75 | { 76 | :ok, 77 | %Stat{ 78 | key: key, 79 | size: stat.size, 80 | etag: etag, 81 | type: "application/octet-stream" 82 | } 83 | } 84 | end 85 | end 86 | 87 | def delete(store, key) do 88 | case File.rm(Disk.join(store, key)) do 89 | :ok -> :ok 90 | {:error, reason} when reason in [:enoent, :enotdir] -> :ok 91 | {:error, reason} -> {:error, reason} 92 | end 93 | end 94 | 95 | def delete_all(store, opts) do 96 | prefix = Keyword.get(opts, :prefix, "") 97 | 98 | store.storage_path 99 | |> Path.join(prefix) 100 | |> File.rm_rf() 101 | |> case do 102 | {:ok, _} -> :ok 103 | {:error, reason, _file} -> {:error, reason} 104 | end 105 | end 106 | 107 | def write(store, key, content, _opts \\ []) do 108 | with {:ok, path} <- expand(store, key) do 109 | File.write(path, content) 110 | end 111 | end 112 | 113 | def read(store, key) do 114 | store |> Disk.join(key) |> File.read() 115 | end 116 | 117 | def copy(store, src, dest) do 118 | with {:ok, src} <- expand(store, src), 119 | {:ok, dest} <- expand(store, dest), 120 | {:ok, _} <- File.copy(src, dest), 121 | do: :ok 122 | end 123 | 124 | def rename(store, src, dest) do 125 | with {:ok, src} <- expand(store, src), 126 | {:ok, dest} <- expand(store, dest), 127 | do: File.rename(src, dest) 128 | end 129 | 130 | def upload(store, source, key) do 131 | with {:ok, dest} <- expand(store, key), 132 | {:ok, _} <- File.copy(source, dest), 133 | do: :ok 134 | end 135 | 136 | def download(store, key, dest) do 137 | with {:ok, source} <- expand(store, key), 138 | {:ok, _} <- File.copy(source, dest), 139 | do: :ok 140 | end 141 | 142 | def list!(store, opts) do 143 | prefix = Keyword.get(opts, :prefix, "") 144 | 145 | store.storage_path 146 | |> Path.join(prefix) 147 | |> Path.join("**/*") 148 | |> Path.wildcard(match_dot: true) 149 | |> Stream.reject(&File.dir?/1) 150 | |> Stream.map(&Path.relative_to(&1, store.storage_path)) 151 | end 152 | 153 | defp expand(store, key) do 154 | with path <- Disk.join(store, key), 155 | dir <- Path.dirname(path), 156 | :ok <- File.mkdir_p(dir), 157 | do: {:ok, path} 158 | end 159 | end 160 | end 161 | -------------------------------------------------------------------------------- /lib/file_store/adapters/memory.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.Memory do 2 | @moduledoc """ 3 | Stores files in memory. This adapter is particularly 4 | useful in tests. 5 | 6 | ### Configuration 7 | 8 | * `name` - The name used to register the process. 9 | 10 | * `base_url` - The base URL that should be used for 11 | generating URLs to your files. 12 | 13 | ### Example 14 | 15 | iex> store = FileStore.Adapters.Memory.new(base_url: "http://example.com/files/") 16 | %FileStore.Adapters.Memory{...} 17 | 18 | iex> FileStore.write(store, "foo", "hello world") 19 | :ok 20 | 21 | iex> FileStore.read(store, "foo") 22 | {:ok, "hello world"} 23 | 24 | ### Usage in tests 25 | 26 | defmodule MyTest do 27 | use ExUnit.Case 28 | 29 | setup do 30 | start_supervised!(FileStore.Adapters.Memory) 31 | :ok 32 | end 33 | 34 | test "writes a file" do 35 | store = FileStore.Adapters.Memory.new() 36 | assert :ok = FileStore.write(store, "foo", "bar") 37 | assert {:ok, "bar"} = FileStore.read(store, "foo") 38 | end 39 | end 40 | 41 | """ 42 | 43 | use Agent 44 | 45 | @enforce_keys [:base_url] 46 | defstruct [:base_url, name: __MODULE__] 47 | 48 | @doc "Creates a new memory adapter" 49 | @spec new(keyword) :: FileStore.t() 50 | def new(opts) do 51 | if is_nil(opts[:base_url]) do 52 | raise "missing configuration: :base_url" 53 | end 54 | 55 | struct(__MODULE__, opts) 56 | end 57 | 58 | @doc "Starts an agent for the test adapter." 59 | def start_link(opts) do 60 | name = Keyword.get(opts, :name, __MODULE__) 61 | Agent.start_link(fn -> %{} end, name: name) 62 | end 63 | 64 | @doc "Stops the agent for the test adapter." 65 | def stop(store, reason \\ :normal, timeout \\ :infinity) do 66 | Agent.stop(store.name, reason, timeout) 67 | end 68 | 69 | defimpl FileStore do 70 | alias FileStore.Stat 71 | alias FileStore.Utils 72 | 73 | def get_public_url(store, key, opts) do 74 | query = Keyword.take(opts, [:content_type, :disposition]) 75 | 76 | store.base_url 77 | |> URI.parse() 78 | |> Utils.append_path(key) 79 | |> Utils.put_query(query) 80 | |> URI.to_string() 81 | end 82 | 83 | def get_signed_url(store, key, opts) do 84 | {:ok, get_public_url(store, key, opts)} 85 | end 86 | 87 | def stat(store, key) do 88 | store.name 89 | |> Agent.get(&Map.fetch(&1, key)) 90 | |> case do 91 | {:ok, data} -> 92 | { 93 | :ok, 94 | %Stat{ 95 | key: key, 96 | size: byte_size(data), 97 | etag: Stat.checksum(data), 98 | type: "application/octet-stream" 99 | } 100 | } 101 | 102 | :error -> 103 | {:error, :enoent} 104 | end 105 | end 106 | 107 | def delete(store, key) do 108 | Agent.update(store.name, &Map.delete(&1, key)) 109 | end 110 | 111 | def delete_all(store, opts) do 112 | prefix = Keyword.get(opts, :prefix, "") 113 | 114 | Agent.update(store.name, fn state -> 115 | state 116 | |> Enum.reject(fn {key, _} -> String.starts_with?(key, prefix) end) 117 | |> Map.new() 118 | end) 119 | end 120 | 121 | def write(store, key, content, _opts \\ []) do 122 | Agent.update(store.name, &Map.put(&1, key, content)) 123 | end 124 | 125 | def read(store, key) do 126 | Agent.get(store.name, fn state -> 127 | with :error <- Map.fetch(state, key) do 128 | {:error, :enoent} 129 | end 130 | end) 131 | end 132 | 133 | def copy(store, src, dest) do 134 | Agent.get_and_update(store.name, fn state -> 135 | case Map.fetch(state, src) do 136 | {:ok, value} -> 137 | {:ok, Map.put(state, dest, value)} 138 | 139 | :error -> 140 | {{:error, :enoent}, state} 141 | end 142 | end) 143 | end 144 | 145 | def rename(store, src, dest) do 146 | Agent.get_and_update(store.name, fn state -> 147 | case Map.fetch(state, src) do 148 | {:ok, value} -> 149 | {:ok, state |> Map.delete(src) |> Map.put(dest, value)} 150 | 151 | :error -> 152 | {{:error, :enoent}, state} 153 | end 154 | end) 155 | end 156 | 157 | def upload(store, source, key) do 158 | with {:ok, data} <- File.read(source) do 159 | write(store, key, data) 160 | end 161 | end 162 | 163 | def download(store, key, destination) do 164 | with {:ok, data} <- read(store, key) do 165 | File.write(destination, data) 166 | end 167 | end 168 | 169 | def list!(store, opts) do 170 | prefix = Keyword.get(opts, :prefix, "") 171 | 172 | store.name 173 | |> Agent.get(&Map.keys/1) 174 | |> Stream.filter(&String.starts_with?(&1, prefix)) 175 | end 176 | end 177 | end 178 | -------------------------------------------------------------------------------- /lib/file_store/adapters/null.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.Null do 2 | @moduledoc """ 3 | Does not attempt to store files. 4 | 5 | ### Example 6 | 7 | iex> store = FileStore.Adapters.Null.new() 8 | %FileStore.Adapters.Null{...} 9 | 10 | iex> FileStore.write(store, "foo", "hello world") 11 | :ok 12 | 13 | iex> FileStore.read(store, "foo") 14 | {:ok, "hello world"} 15 | 16 | """ 17 | 18 | defstruct [] 19 | 20 | @doc "Creates a new null adapter" 21 | @spec new(keyword) :: FileStore.t() 22 | def new(opts \\ []) do 23 | struct(__MODULE__, opts) 24 | end 25 | 26 | defimpl FileStore do 27 | alias FileStore.Stat 28 | 29 | def get_public_url(_store, key, _opts), do: key 30 | def get_signed_url(_store, key, _opts), do: {:ok, key} 31 | 32 | def stat(_store, key) do 33 | { 34 | :ok, 35 | %Stat{ 36 | key: key, 37 | size: 0, 38 | etag: Stat.checksum(""), 39 | type: "application/octet-stream" 40 | } 41 | } 42 | end 43 | 44 | def delete(_store, _key), do: :ok 45 | def delete_all(_store, _opts), do: :ok 46 | def upload(_store, _source, _key), do: :ok 47 | def download(_store, _key, _destination), do: :ok 48 | def write(_store, _key, _content, _opts \\ []), do: :ok 49 | def read(_store, _key), do: {:ok, ""} 50 | def copy(_store, _src, _dest), do: :ok 51 | def rename(_store, _src, _dest), do: :ok 52 | def list!(_store, _opts), do: Stream.into([], []) 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/file_store/adapters/s3.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(ExAws.S3) do 2 | defmodule FileStore.Adapters.S3 do 3 | @moduledoc """ 4 | Stores files using Amazon S3. 5 | 6 | ### Dependencies 7 | 8 | To use this adapter, you'll need to install and configure `ExAws.S3`. 9 | 10 | ### Configuration 11 | 12 | * `bucket` - The name of your S3 bucket. This option 13 | is required. 14 | 15 | * `ex_aws` - A keyword list of options that can be 16 | used to override the default configuration for `ExAws`. 17 | 18 | ### Example 19 | 20 | iex> store = FileStore.Adapters.S3.new( 21 | ...> bucket: "mybucket" 22 | ...> ) 23 | %FileStore.Adapters.S3{...} 24 | 25 | iex> FileStore.write(store, "foo", "hello world") 26 | :ok 27 | 28 | iex> FileStore.read(store, "foo") 29 | {:ok, "hello world"} 30 | 31 | """ 32 | 33 | @enforce_keys [:bucket] 34 | defstruct [:bucket, ex_aws: []] 35 | 36 | @doc "Create a new S3 adapter" 37 | @spec new(keyword) :: FileStore.t() 38 | def new(opts) do 39 | if is_nil(opts[:bucket]) do 40 | raise "missing configuration: :bucket" 41 | end 42 | 43 | struct(__MODULE__, opts) 44 | end 45 | 46 | defimpl FileStore do 47 | alias FileStore.Stat 48 | alias FileStore.Utils 49 | 50 | @query_params [ 51 | content_type: "response-content-type", 52 | disposition: "response-content-disposition" 53 | ] 54 | 55 | def get_public_url(store, key, opts) do 56 | config = get_config(store) 57 | host = store.bucket <> "." <> config[:host] 58 | query = Utils.encode_query(get_url_query(opts)) 59 | scheme = String.trim_trailing(config[:scheme], "://") 60 | 61 | uri = %URI{ 62 | scheme: scheme, 63 | host: host, 64 | port: config[:port], 65 | path: "/" <> key, 66 | query: query 67 | } 68 | 69 | URI.to_string(uri) 70 | end 71 | 72 | def get_signed_url(store, key, opts) do 73 | config = get_config(store) 74 | 75 | opts = 76 | opts 77 | |> Keyword.take([:expires_in]) 78 | |> Keyword.put(:virtual_host, true) 79 | |> Keyword.put(:query_params, get_url_query(opts)) 80 | 81 | ExAws.S3.presigned_url(config, :get, store.bucket, key, opts) 82 | end 83 | 84 | def stat(store, key) do 85 | store.bucket 86 | |> ExAws.S3.head_object(key) 87 | |> request(store) 88 | |> case do 89 | {:ok, %{headers: headers}} -> 90 | headers = Enum.into(headers, %{}) 91 | etag = headers |> Map.get("ETag") |> unwrap_etag() 92 | size = headers |> Map.get("Content-Length") |> to_integer() 93 | type = headers |> Map.get("Content-Type") 94 | {:ok, %Stat{key: key, etag: etag, size: size, type: type}} 95 | 96 | {:error, reason} -> 97 | {:error, reason} 98 | end 99 | end 100 | 101 | def delete(store, key) do 102 | store.bucket 103 | |> ExAws.S3.delete_object(key) 104 | |> acknowledge(store) 105 | end 106 | 107 | def delete_all(store, opts) do 108 | store.bucket 109 | |> ExAws.S3.delete_all_objects(list!(store, opts)) 110 | |> acknowledge(store) 111 | rescue 112 | error -> {:error, error} 113 | end 114 | 115 | def write(store, key, content, opts \\ []) do 116 | opts = 117 | opts 118 | |> Keyword.take([:content_type, :disposition]) 119 | |> Utils.rename_key(:disposition, :content_disposition) 120 | 121 | store.bucket 122 | |> ExAws.S3.put_object(key, content, opts) 123 | |> acknowledge(store) 124 | end 125 | 126 | def read(store, key) do 127 | store.bucket 128 | |> ExAws.S3.get_object(key) 129 | |> request(store) 130 | |> case do 131 | {:ok, %{body: body}} -> {:ok, body} 132 | {:error, reason} -> {:error, reason} 133 | end 134 | end 135 | 136 | def upload(store, source, key) do 137 | source 138 | |> ExAws.S3.Upload.stream_file() 139 | |> ExAws.S3.upload(store.bucket, key) 140 | |> acknowledge(store) 141 | rescue 142 | error in [File.Error] -> {:error, error.reason} 143 | end 144 | 145 | def download(store, key, destination) do 146 | store.bucket 147 | |> ExAws.S3.download_file(key, destination) 148 | |> acknowledge(store) 149 | end 150 | 151 | def list!(store, opts) do 152 | opts = Keyword.take(opts, [:prefix]) 153 | 154 | store.bucket 155 | |> ExAws.S3.list_objects(opts) 156 | |> ExAws.stream!(store.ex_aws) 157 | |> Stream.map(fn file -> file.key end) 158 | end 159 | 160 | def copy(store, src, dest) do 161 | store.bucket 162 | |> ExAws.S3.put_object_copy(dest, store.bucket, src) 163 | |> acknowledge(store) 164 | end 165 | 166 | def rename(store, src, dest) do 167 | with :ok <- copy(store, src, dest) do 168 | delete(store, src) 169 | end 170 | end 171 | 172 | defp request(op, store) do 173 | ExAws.request(op, store.ex_aws) 174 | end 175 | 176 | defp acknowledge(op, store) do 177 | case request(op, store) do 178 | {:ok, _} -> :ok 179 | {:error, reason} -> {:error, reason} 180 | end 181 | end 182 | 183 | defp get_url_query(opts) do 184 | for {key, query_param} <- @query_params, 185 | value = Keyword.get(opts, key), 186 | into: [], 187 | do: {query_param, value} 188 | end 189 | 190 | defp get_config(store), do: ExAws.Config.new(:s3, store.ex_aws) 191 | 192 | defp unwrap_etag(nil), do: nil 193 | defp unwrap_etag(etag), do: String.trim(etag, ~s(")) 194 | 195 | defp to_integer(nil), do: nil 196 | defp to_integer(value) when is_integer(value), do: value 197 | defp to_integer(value) when is_binary(value), do: String.to_integer(value) 198 | end 199 | end 200 | end 201 | -------------------------------------------------------------------------------- /lib/file_store/config.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Config do 2 | @moduledoc """ 3 | Define a configurable store. 4 | 5 | ### Usage 6 | 7 | First, define a new module: 8 | 9 | defmodule MyApp.Storage do 10 | use FileStore.Config, otp_app: :my_app 11 | end 12 | 13 | In your config files, you'll need to configure your adapter: 14 | 15 | config :my_app, MyApp.Storage, 16 | adapter: FileStore.Adapters.Disk, 17 | storage_path: "/path/to/files", 18 | base_url: "https://localhost:4000" 19 | 20 | You can also configure any `FileStore.Middleware` here: 21 | 22 | config :my_app, MyApp.Storage, 23 | adapter: FileStore.Adapters.Disk, 24 | # ...etc... 25 | middleware: [FileStore.Middleware.Errors] 26 | 27 | If you need to dynamically configure your store at runtime, 28 | you can implement the `init/1` callback. 29 | 30 | def init(opts) do 31 | Keyword.put(opts, :foo, "bar") 32 | end 33 | 34 | ### Example 35 | 36 | iex> MyApp.Storage.write("foo", "hello world") 37 | :ok 38 | 39 | iex> MyApp.Storage.read("foo") 40 | {:ok, "hello world"} 41 | 42 | """ 43 | 44 | defmacro __using__(opts) do 45 | {otp_app, opts} = Keyword.pop(opts, :otp_app) 46 | 47 | quote location: :keep do 48 | @spec init(keyword) :: keyword 49 | def init(opts), do: opts 50 | defoverridable init: 1 51 | 52 | @spec new() :: FileStore.t() 53 | def new do 54 | config = Application.get_env(unquote(otp_app), __MODULE__, []) 55 | config = unquote(opts) |> Keyword.merge(config) |> init() 56 | {middlewares, config} = Keyword.pop(config, :middleware, []) 57 | 58 | case Keyword.pop(config, :adapter) do 59 | {nil, _} -> 60 | raise "Adapter not specified in #{__MODULE__} configuration" 61 | 62 | {adapter, config} -> 63 | Enum.reduce(middlewares, adapter.new(config), fn 64 | {middleware, args}, store -> middleware.new(store, args) 65 | middleware, store -> middleware.new(store) 66 | end) 67 | end 68 | end 69 | 70 | @spec stat(binary()) :: {:ok, FileStore.Stat.t()} | {:error, term()} 71 | def stat(key) do 72 | FileStore.stat(new(), key) 73 | end 74 | 75 | @spec read(binary()) :: {:ok, binary()} | {:error, term()} 76 | def read(key) do 77 | FileStore.read(new(), key) 78 | end 79 | 80 | @spec write(FileStore.key(), binary(), FileStore.write_opts()) :: :ok | {:error, term()} 81 | def write(key, content, opts \\ []) do 82 | FileStore.write(new(), key, content, opts) 83 | end 84 | 85 | @spec delete(FileStore.key()) :: :ok | {:error, term()} 86 | def delete(key) do 87 | FileStore.delete(new(), key) 88 | end 89 | 90 | @spec delete_all(FileStore.delete_all_opts()) :: :ok | {:error, term()} 91 | def delete_all(opts \\ []) do 92 | FileStore.delete_all(new(), opts) 93 | end 94 | 95 | @spec copy(FileStore.key(), FileStore.key()) :: :ok | {:error, term()} 96 | def copy(src, dest) do 97 | FileStore.copy(new(), src, dest) 98 | end 99 | 100 | @spec rename(FileStore.key(), FileStore.key()) :: :ok | {:error, term()} 101 | def rename(src, dest) do 102 | FileStore.rename(new(), src, dest) 103 | end 104 | 105 | @spec upload(Path.t(), binary()) :: :ok | {:error, term()} 106 | def upload(source, key) do 107 | FileStore.upload(new(), source, key) 108 | end 109 | 110 | @spec download(FileStore.key(), Path.t()) :: :ok | {:error, term()} 111 | def download(key, destination) do 112 | FileStore.download(new(), key, destination) 113 | end 114 | 115 | @spec get_public_url(FileStore.key(), Keyword.t()) :: binary() 116 | def get_public_url(key, opts \\ []) do 117 | FileStore.get_public_url(new(), key, opts) 118 | end 119 | 120 | @spec get_signed_url(FileStore.key(), Keyword.t()) :: {:ok, binary()} | {:error, term()} 121 | def get_signed_url(key, opts \\ []) do 122 | FileStore.get_signed_url(new(), key, opts) 123 | end 124 | 125 | @spec list! :: Enumerable.t() 126 | def list!(opts \\ []) do 127 | FileStore.list!(new(), opts) 128 | end 129 | end 130 | end 131 | end 132 | -------------------------------------------------------------------------------- /lib/file_store/error.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Error do 2 | defexception [:reason, :key, :action] 3 | 4 | @common_posix ~w(eacces eexist enoent enospc enotdir)a 5 | 6 | @impl true 7 | def message(%{action: action, key: nil, reason: reason}) do 8 | "could not #{action}: #{format(reason)}" 9 | end 10 | 11 | def message(%{action: action, reason: reason, key: key}) do 12 | "could not #{action} #{inspect(key)}: #{format(reason)}" 13 | end 14 | 15 | @doc false 16 | def format(reason) when reason in @common_posix do 17 | reason |> :file.format_error() |> IO.iodata_to_binary() 18 | end 19 | 20 | def format(reason) do 21 | inspect(reason) 22 | end 23 | end 24 | 25 | defmodule FileStore.UploadError do 26 | defexception [:reason, :path, :key] 27 | 28 | @impl true 29 | def message(%{reason: reason, path: path, key: key}) do 30 | reason = FileStore.Error.format(reason) 31 | "could not upload file #{inspect(path)} to key #{inspect(key)}: #{reason}" 32 | end 33 | end 34 | 35 | defmodule FileStore.DownloadError do 36 | defexception [:reason, :path, :key] 37 | 38 | @impl true 39 | def message(%{reason: reason, path: path, key: key}) do 40 | reason = FileStore.Error.format(reason) 41 | "could not download key #{inspect(key)} to file #{inspect(path)}: #{reason}" 42 | end 43 | end 44 | 45 | defmodule FileStore.CopyError do 46 | defexception [:reason, :src, :dest] 47 | 48 | @impl true 49 | def message(%{reason: reason, src: src, dest: dest}) do 50 | reason = FileStore.Error.format(reason) 51 | "could not copy #{inspect(src)} to #{inspect(dest)}: #{reason}" 52 | end 53 | end 54 | 55 | defmodule FileStore.RenameError do 56 | defexception [:reason, :src, :dest] 57 | 58 | @impl true 59 | def message(%{reason: reason, src: src, dest: dest}) do 60 | reason = FileStore.Error.format(reason) 61 | "could not rename #{inspect(src)} to #{inspect(dest)}: #{reason}" 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/file_store/middleware.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware do 2 | @moduledoc """ 3 | Middleware allows you to enhance your store with additional functionality. 4 | 5 | The following middlewares ship with this library. 6 | 7 | * `FileStore.Middleware.Logger` 8 | * `FileStore.Middleware.Errors` 9 | * `FileStore.Middleware.Prefix` 10 | 11 | To use a middleware, simply wrap your existing store with the middleware: 12 | 13 | iex> store = FileStore.Adapters.Disk.new([...]) 14 | %FileStore.Adapters.Disk{...} 15 | 16 | iex> store = FileStore.Middleware.Logger.new(store) 17 | %FileStore.Middleware.Logger{...} 18 | 19 | iex> FileStore.read(store, "test.txt") 20 | # 02:37:30.724 [debug] READ OK key="test.txt" 21 | {:ok, "hello"} 22 | 23 | You can compose multiple middlewares, but order _does_ matter. The following 24 | order is recommended: 25 | 26 | FileStore.Adapters.Null.new() 27 | |> FileStore.Middleware.Errors.new() 28 | |> FileStore.Middleware.Prefix.new(prefix: "foo") 29 | |> FileStore.Middleware.Logger.new() 30 | """ 31 | end 32 | -------------------------------------------------------------------------------- /lib/file_store/middleware/errors.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.Errors do 2 | @moduledoc """ 3 | By default, each adapter will return errors in a different format. This 4 | middleware attempts to make the errors returned by this library a little 5 | more useful by wrapping them in exception structs: 6 | 7 | * `FileStore.Error` 8 | * `FileStore.UploadError` 9 | * `FileStore.DownloadError` 10 | * `FileStore.CopyError` 11 | * `FileStore.RenameError` 12 | 13 | Each of these structs contain `reason` field, where you'll find the original 14 | error that was returned by the underlying adapter. 15 | 16 | One nice feature of this middleware is that it makes it easy to raise: 17 | 18 | store 19 | |> FileStore.Middleware.Errors.new() 20 | |> FileStore.read("example.jpg") 21 | |> case do 22 | {:ok, data} -> data 23 | {:error, error} -> raise error 24 | end 25 | 26 | See the documentation for `FileStore.Middleware` for more information. 27 | """ 28 | 29 | @enforce_keys [:__next__] 30 | defstruct [:__next__] 31 | 32 | def new(store) do 33 | %__MODULE__{__next__: store} 34 | end 35 | 36 | defimpl FileStore do 37 | alias FileStore.Error 38 | alias FileStore.UploadError 39 | alias FileStore.DownloadError 40 | alias FileStore.RenameError 41 | alias FileStore.CopyError 42 | 43 | def stat(store, key) do 44 | store.__next__ 45 | |> FileStore.stat(key) 46 | |> wrap(action: "read stats for key", key: key) 47 | end 48 | 49 | def write(store, key, content, opts) do 50 | store.__next__ 51 | |> FileStore.write(key, content, opts) 52 | |> wrap(action: "write to key", key: key) 53 | end 54 | 55 | def read(store, key) do 56 | store.__next__ 57 | |> FileStore.read(key) 58 | |> wrap(action: "read key", key: key) 59 | end 60 | 61 | def copy(store, src, dest) do 62 | store.__next__ 63 | |> FileStore.copy(src, dest) 64 | |> wrap(CopyError, src: src, dest: dest) 65 | end 66 | 67 | def rename(store, src, dest) do 68 | store.__next__ 69 | |> FileStore.rename(src, dest) 70 | |> wrap(RenameError, src: src, dest: dest) 71 | end 72 | 73 | def upload(store, path, key) do 74 | store.__next__ 75 | |> FileStore.upload(path, key) 76 | |> wrap(UploadError, path: path, key: key) 77 | end 78 | 79 | def download(store, key, path) do 80 | store.__next__ 81 | |> FileStore.download(key, path) 82 | |> wrap(DownloadError, path: path, key: key) 83 | end 84 | 85 | def delete(store, key) do 86 | store.__next__ 87 | |> FileStore.delete(key) 88 | |> wrap(action: "delete key", key: key) 89 | end 90 | 91 | def delete_all(store, opts) do 92 | prefix = opts[:prefix] 93 | 94 | action = 95 | if prefix, 96 | do: "delete keys matching prefix", 97 | else: "delete all keys" 98 | 99 | store.__next__ 100 | |> FileStore.delete_all(opts) 101 | |> wrap(action: action, key: prefix) 102 | end 103 | 104 | def get_public_url(store, key, opts) do 105 | FileStore.get_public_url(store.__next__, key, opts) 106 | end 107 | 108 | def get_signed_url(store, key, opts) do 109 | store.__next__ 110 | |> FileStore.get_signed_url(key, opts) 111 | |> wrap(action: "generate signed URL for key", key: key) 112 | end 113 | 114 | def list!(store, opts) do 115 | FileStore.list!(store.__next__, opts) 116 | end 117 | 118 | defp wrap(result, error \\ Error, opts) 119 | 120 | defp wrap({:error, reason}, kind, opts) do 121 | {:error, struct(kind, Keyword.put(opts, :reason, reason))} 122 | end 123 | 124 | defp wrap(other, _, _), do: other 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /lib/file_store/middleware/logger.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.Logger do 2 | @moduledoc """ 3 | This middleware allows you to log all operations. 4 | 5 | See the documentation for `FileStore.Middleware` for more information. 6 | """ 7 | 8 | @enforce_keys [:__next__] 9 | defstruct [:__next__] 10 | 11 | def new(store) do 12 | %__MODULE__{__next__: store} 13 | end 14 | 15 | defimpl FileStore do 16 | require Logger 17 | 18 | def stat(store, key) do 19 | store.__next__ 20 | |> FileStore.stat(key) 21 | |> log("STAT", key: key) 22 | end 23 | 24 | def write(store, key, content, opts) do 25 | store.__next__ 26 | |> FileStore.write(key, content, opts) 27 | |> log("WRITE", key: key) 28 | end 29 | 30 | def read(store, key) do 31 | store.__next__ 32 | |> FileStore.read(key) 33 | |> log("READ", key: key) 34 | end 35 | 36 | def copy(store, src, dest) do 37 | store.__next__ 38 | |> FileStore.copy(src, dest) 39 | |> log("COPY", src: src, dest: dest) 40 | end 41 | 42 | def rename(store, src, dest) do 43 | store.__next__ 44 | |> FileStore.rename(src, dest) 45 | |> log("RENAME", src: src, dest: dest) 46 | end 47 | 48 | def upload(store, source, key) do 49 | store.__next__ 50 | |> FileStore.upload(source, key) 51 | |> log("UPLOAD", key: key) 52 | end 53 | 54 | def download(store, key, dest) do 55 | store.__next__ 56 | |> FileStore.download(key, dest) 57 | |> log("DOWNLOAD", key: key) 58 | end 59 | 60 | def delete(store, key) do 61 | store.__next__ 62 | |> FileStore.delete(key) 63 | |> log("DELETE", key: key) 64 | end 65 | 66 | def delete_all(store, opts) do 67 | store.__next__ 68 | |> FileStore.delete_all(opts) 69 | |> log("DELETE ALL", opts) 70 | end 71 | 72 | def get_public_url(store, key, opts) do 73 | FileStore.get_public_url(store.__next__, key, opts) 74 | end 75 | 76 | def get_signed_url(store, key, opts) do 77 | FileStore.get_signed_url(store.__next__, key, opts) 78 | end 79 | 80 | def list!(store, opts) do 81 | FileStore.list!(store.__next__, opts) 82 | end 83 | 84 | defp log({:ok, value}, msg, meta) do 85 | log(:ok, msg, meta) 86 | {:ok, value} 87 | end 88 | 89 | defp log(:ok, msg, meta) do 90 | Logger.log(:debug, fn -> 91 | [msg, ?\s, "OK", ?\s, format_meta(meta)] 92 | end) 93 | 94 | :ok 95 | end 96 | 97 | defp log({:error, error}, msg, meta) do 98 | Logger.log(:error, fn -> 99 | [msg, ?\s, "ERROR", ?\s, format_meta(meta), format_error(error)] 100 | end) 101 | 102 | {:error, error} 103 | end 104 | 105 | defp format_meta(meta) do 106 | Enum.map(meta, fn {key, value} -> 107 | [Atom.to_string(key), ?\=, inspect(value)] 108 | end) 109 | end 110 | 111 | defp format_error(%{__exception__: true} = error) do 112 | [?\n, Exception.format(:error, error)] 113 | end 114 | 115 | defp format_error(error) do 116 | [" error=", inspect(error)] 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /lib/file_store/middleware/prefix.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.Prefix do 2 | @moduledoc """ 3 | This module adds a prefix to all operations. 4 | 5 | store 6 | |> FileStore.Middleware.Prefix.new(prefix: "companies/logos") 7 | |> FileStore.read("example.jpg") 8 | 9 | In the example above, the key `example.jpg` would become 10 | `companies/logos/example.jpg`. 11 | 12 | See the documentation for `FileStore.Middleware` for more information. 13 | """ 14 | 15 | @enforce_keys [:__next__, :prefix] 16 | defstruct [:__next__, :prefix] 17 | 18 | @doc "Add the prefix adapter to your store." 19 | def new(store, opts) do 20 | struct(__MODULE__, Keyword.put(opts, :__next__, store)) 21 | end 22 | 23 | defimpl FileStore do 24 | alias FileStore.Stat 25 | alias FileStore.Utils 26 | 27 | def stat(store, key) do 28 | with {:ok, stat} <- FileStore.stat(store.__next__, put_prefix(key, store)) do 29 | {:ok, %Stat{stat | key: remove_prefix(stat.key, store)}} 30 | end 31 | end 32 | 33 | def delete(store, key) do 34 | FileStore.delete(store.__next__, put_prefix(key, store)) 35 | end 36 | 37 | def delete_all(store, opts) do 38 | FileStore.delete_all(store.__next__, put_prefix(opts, store)) 39 | end 40 | 41 | def write(store, key, content, opts) do 42 | FileStore.write(store.__next__, put_prefix(key, store), content, opts) 43 | end 44 | 45 | def read(store, key) do 46 | FileStore.read(store.__next__, put_prefix(key, store)) 47 | end 48 | 49 | def copy(store, src, dest) do 50 | FileStore.copy(store.__next__, put_prefix(src, store), put_prefix(dest, store)) 51 | end 52 | 53 | def rename(store, src, dest) do 54 | FileStore.rename(store.__next__, put_prefix(src, store), put_prefix(dest, store)) 55 | end 56 | 57 | def upload(store, source, key) do 58 | FileStore.upload(store.__next__, source, put_prefix(key, store)) 59 | end 60 | 61 | def download(store, key, dest) do 62 | FileStore.download(store.__next__, put_prefix(key, store), dest) 63 | end 64 | 65 | def get_public_url(store, key, opts) do 66 | FileStore.get_public_url(store.__next__, put_prefix(key, store), opts) 67 | end 68 | 69 | def get_signed_url(store, key, opts) do 70 | FileStore.get_signed_url(store.__next__, put_prefix(key, store), opts) 71 | end 72 | 73 | def list!(store, opts) do 74 | store.__next__ 75 | |> FileStore.list!(put_prefix(opts, store)) 76 | |> Stream.map(&remove_prefix(&1, store)) 77 | end 78 | 79 | defp put_prefix(opts, store) when is_list(opts) do 80 | Keyword.update(opts, :prefix, store.prefix, &put_prefix(&1, store)) 81 | end 82 | 83 | defp put_prefix(key, store) do 84 | Utils.join(store.prefix, key) 85 | end 86 | 87 | defp remove_prefix(key, store) do 88 | key 89 | |> String.trim_leading(store.prefix) 90 | |> String.trim_leading("/") 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/file_store/stat.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Stat do 2 | @moduledoc """ 3 | A struct that holds file information. 4 | 5 | ### Fields 6 | 7 | * `key` - The unique identifier for the file. 8 | 9 | * `etag` - A fingerprint for the contents of a file. 10 | This is almost always an MD5 checksum. 11 | 12 | * `size` - The byte size of the file. 13 | 14 | * `type` - The content-type of the file. 15 | 16 | """ 17 | 18 | @enforce_keys [:key, :size, :etag, :type] 19 | defstruct [:key, :size, :etag, :type] 20 | 21 | @type t :: %__MODULE__{ 22 | key: binary, 23 | etag: binary, 24 | size: non_neg_integer, 25 | type: binary 26 | } 27 | 28 | @doc """ 29 | Compute an MD5 checksum. 30 | 31 | ### Example 32 | 33 | iex> FileStore.Stat.checksum("hello world") 34 | "5eb63bbbe01eeed093cb22bb8f5acdc3" 35 | 36 | """ 37 | @spec checksum(binary | Enumerable.t()) :: binary 38 | def checksum(data) when is_binary(data) do 39 | :md5 40 | |> :crypto.hash(data) 41 | |> Base.encode16() 42 | |> String.downcase() 43 | end 44 | 45 | def checksum(data) do 46 | data 47 | |> Enum.reduce(:crypto.hash_init(:md5), &:crypto.hash_update(&2, &1)) 48 | |> :crypto.hash_final() 49 | |> Base.encode16() 50 | |> String.downcase() 51 | end 52 | 53 | @doc """ 54 | Compute the MD5 checksum of a file on disk. 55 | 56 | ### Example 57 | 58 | iex> FileStore.Stat.checksum_file("test/fixtures/test.txt") 59 | {:ok, "0d599f0ec05c3bda8c3b8a68c32a1b47"} 60 | 61 | iex> FileStore.Stat.checksum_file("test/fixtures/missing.txt") 62 | {:error, :enoent} 63 | 64 | """ 65 | @spec checksum_file(Path.t()) :: {:ok, binary} | {:error, File.posix()} 66 | def checksum_file(path) do 67 | {:ok, path |> stream!() |> checksum()} 68 | rescue 69 | e in [File.Error] -> {:error, e.reason} 70 | end 71 | 72 | # In v1.16 `File.stream!/3` changed the ordering of its parameters. In order 73 | # to avoid any deprecation warnings going forward, we need to flip out the 74 | # implementation. 75 | if Version.compare(System.version(), "1.16.0") in [:gt, :eq] do 76 | defp stream!(path), do: File.stream!(path, 2048, []) 77 | else 78 | defp stream!(path), do: File.stream!(path, [], 2048) 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/file_store/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Utils do 2 | @moduledoc false 3 | 4 | def join(nil, nil), do: nil 5 | def join(a, nil), do: a 6 | def join(nil, b), do: b 7 | 8 | def join(a, b) do 9 | String.trim_trailing(a, "/") <> "/" <> String.trim_leading(b, "/") 10 | end 11 | 12 | def join_absolute(a, b) do 13 | if path = join(a, b) do 14 | "/" <> String.trim_leading(path, "/") 15 | end 16 | end 17 | 18 | def append_path(%URI{path: a} = uri, b) do 19 | %URI{uri | path: join_absolute(a, b)} 20 | end 21 | 22 | def put_query(%URI{query: nil} = uri, query) do 23 | %URI{uri | query: encode_query(query)} 24 | end 25 | 26 | def encode_query([]), do: nil 27 | def encode_query(query), do: URI.encode_query(query) 28 | 29 | @spec rename_key(Keyword.t(), term(), term()) :: Keyword.t() 30 | def rename_key(opts, key, new_key) do 31 | case Keyword.fetch(opts, key) do 32 | {:ok, value} -> 33 | opts 34 | |> Keyword.delete(key) 35 | |> Keyword.put(new_key, value) 36 | 37 | :error -> 38 | opts 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.MixProject do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/rzane/file_store" 5 | @version "0.0.0" 6 | 7 | def project do 8 | [ 9 | app: :file_store, 10 | version: @version, 11 | elixir: "~> 1.14", 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | start_permanent: Mix.env() == :prod, 14 | deps: deps(), 15 | package: package(), 16 | test_coverage: [tool: ExCoveralls], 17 | dialyzer: [ 18 | plt_add_apps: [:ex_aws, :ex_aws_s3], 19 | plt_file: {:no_warn, "priv/plts/dialyzer.plt"} 20 | ], 21 | docs: docs() 22 | ] 23 | end 24 | 25 | def application do 26 | [ 27 | extra_applications: [:logger] 28 | ] 29 | end 30 | 31 | defp elixirc_paths(:test), do: ["lib", "test/support"] 32 | defp elixirc_paths(_), do: ["lib"] 33 | 34 | defp package do 35 | [ 36 | description: "A unified interface for file storage backends.", 37 | maintainers: ["Ray Zane"], 38 | licenses: ["MIT"], 39 | links: %{ 40 | "GitHub" => @source_url, 41 | "Changelog" => @source_url <> "/releases" 42 | } 43 | ] 44 | end 45 | 46 | defp deps do 47 | [ 48 | {:ex_aws_s3, "~> 2.3", optional: true}, 49 | {:hackney, ">= 0.0.0", only: [:dev, :test]}, 50 | {:sweet_xml, ">= 0.0.0", only: [:dev, :test]}, 51 | {:jason, ">= 0.0.0", only: [:dev, :test]}, 52 | {:excoveralls, "~> 0.14", only: :test}, 53 | {:ex_doc, "~> 0.27", only: :dev, runtime: false}, 54 | {:dialyxir, "~> 1.1", only: [:dev, :test], runtime: false}, 55 | {:castore, "~> 1.0", only: [:test]}, 56 | {:credo, "~> 1.6", only: [:dev, :test], runtime: false} 57 | ] 58 | end 59 | 60 | defp docs do 61 | [ 62 | main: "readme", 63 | source_ref: "v#{@version}", 64 | source_url: @source_url, 65 | extras: ["README.md"] 66 | ] 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, 3 | "castore": {:hex, :castore, "1.0.12", "053f0e32700cbec356280c0e835df425a3be4bc1e0627b714330ad9d0f05497f", [:mix], [], "hexpm", "3dca286b2186055ba0c9449b4e95b97bf1b57b47c1f2644555879e659960c224"}, 4 | "certifi": {:hex, :certifi, "2.14.0", "ed3bef654e69cde5e6c022df8070a579a79e8ba2368a00acf3d75b82d9aceeed", [:rebar3], [], "hexpm", "ea59d87ef89da429b8e905264fdec3419f84f2215bb3d81e07a18aac919026c3"}, 5 | "credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"}, 6 | "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, 7 | "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, 8 | "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, 9 | "ex_aws": {:hex, :ex_aws, "2.5.8", "0393cfbc5e4a9e7017845451a015d836a670397100aa4c86901980e2a2c5f7d4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8f79777b7932168956c8cc3a6db41f5783aa816eb50de356aed3165a71e5f8c3"}, 10 | "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.6", "d135983bbd8b6df6350dfd83999437725527c1bea151e5055760bfc9b2d17c20", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "9874e12847e469ca2f13a5689be04e546c16f63caf6380870b7f25bf7cb98875"}, 11 | "ex_doc": {:hex, :ex_doc, "0.37.3", "f7816881a443cd77872b7d6118e8a55f547f49903aef8747dbcb345a75b462f9", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "e6aebca7156e7c29b5da4daa17f6361205b2ae5f26e5c7d8ca0d3f7e18972233"}, 12 | "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"}, 13 | "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, 14 | "hackney": {:hex, :hackney, "1.23.0", "55cc09077112bcb4a69e54be46ed9bc55537763a96cd4a80a221663a7eafd767", [:rebar3], [{:certifi, "~> 2.14.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "6cd1c04cd15c81e5a493f167b226a15f0938a84fc8f0736ebe4ddcab65c0b44e"}, 15 | "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, 16 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 17 | "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, 18 | "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, 19 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, 20 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 21 | "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, 22 | "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, 23 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, 24 | "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, 25 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, 26 | "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, 27 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 28 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, 29 | } 30 | -------------------------------------------------------------------------------- /test/file_store/adapters/disk_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.DiskTest do 2 | use FileStore.AdapterCase 3 | 4 | alias FileStore.Adapters.Disk 5 | 6 | @url "http://localhost:4000/foo" 7 | 8 | setup %{tmp: tmp} do 9 | {:ok, store: Disk.new(storage_path: tmp, base_url: "http://localhost:4000")} 10 | end 11 | 12 | test "get_public_url/3 with query params", %{store: store} do 13 | opts = [content_type: "text/plain", disposition: "attachment"] 14 | url = FileStore.get_public_url(store, "foo", opts) 15 | assert omit_query(url) == @url 16 | assert get_query(url, "content_type") == "text/plain" 17 | assert get_query(url, "disposition") == "attachment" 18 | end 19 | 20 | test "get_signed_url/3 with query params", %{store: store} do 21 | opts = [content_type: "text/plain", disposition: "attachment"] 22 | assert {:ok, url} = FileStore.get_signed_url(store, "foo", opts) 23 | assert omit_query(url) == @url 24 | assert get_query(url, "content_type") == "text/plain" 25 | assert get_query(url, "disposition") == "attachment" 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/file_store/adapters/memory_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.MemoryTest do 2 | use FileStore.AdapterCase 3 | 4 | alias FileStore.Adapters.Memory 5 | 6 | @url "http://localhost:4000/foo" 7 | 8 | setup do 9 | start_supervised!(Memory) 10 | {:ok, store: Memory.new(base_url: "http://localhost:4000")} 11 | end 12 | 13 | test "get_public_url/3 with query params", %{store: store} do 14 | opts = [content_type: "text/plain", disposition: "attachment"] 15 | url = FileStore.get_public_url(store, "foo", opts) 16 | assert omit_query(url) == @url 17 | assert get_query(url, "content_type") == "text/plain" 18 | assert get_query(url, "disposition") == "attachment" 19 | end 20 | 21 | test "get_signed_url/3 with query params", %{store: store} do 22 | opts = [content_type: "text/plain", disposition: "attachment"] 23 | assert {:ok, url} = FileStore.get_signed_url(store, "foo", opts) 24 | assert omit_query(url) == @url 25 | assert get_query(url, "content_type") == "text/plain" 26 | assert get_query(url, "disposition") == "attachment" 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /test/file_store/adapters/null_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.NullTest do 2 | use ExUnit.Case 3 | 4 | alias FileStore.Adapters.Null 5 | 6 | setup do 7 | {:ok, store: Null.new()} 8 | end 9 | 10 | test "get_public_url/2", %{store: store} do 11 | assert FileStore.get_public_url(store, "foo") == "foo" 12 | end 13 | 14 | test "get_signed_url/2", %{store: store} do 15 | assert FileStore.get_signed_url(store, "foo") == {:ok, "foo"} 16 | end 17 | 18 | test "write/3", %{store: store} do 19 | assert :ok = FileStore.write(store, "foo", "bar") 20 | end 21 | 22 | test "read/3", %{store: store} do 23 | assert :ok = FileStore.write(store, "foo", "bar") 24 | assert FileStore.read(store, "foo") == {:ok, ""} 25 | end 26 | 27 | test "download/3", %{store: store} do 28 | assert :ok = FileStore.download(store, "foo", "download.txt") 29 | end 30 | 31 | test "upload/3", %{store: store} do 32 | assert :ok = FileStore.upload(store, "upload.txt", "foo") 33 | end 34 | 35 | test "stat/2", %{store: store} do 36 | assert :ok = FileStore.write(store, "foo", "bar") 37 | assert {:ok, stat} = FileStore.stat(store, "foo") 38 | assert stat.key == "foo" 39 | assert stat.size == 0 40 | assert stat.etag == "d41d8cd98f00b204e9800998ecf8427e" 41 | end 42 | 43 | test "list/0", %{store: store} do 44 | assert Enum.to_list(FileStore.list!(store)) == [] 45 | end 46 | 47 | describe "copy/3" do 48 | test "copies a file", %{store: store} do 49 | :ok = FileStore.write(store, "foo", "test") 50 | assert :ok = FileStore.copy(store, "foo", "bar") 51 | end 52 | 53 | test "copies a non existing file", %{store: store} do 54 | assert :ok = FileStore.copy(store, "doesnotexist.txt", "shouldnotexist.txt") 55 | end 56 | end 57 | 58 | describe "rename/3" do 59 | test "renames a file", %{store: store} do 60 | :ok = FileStore.write(store, "foo", "test") 61 | assert :ok = FileStore.rename(store, "foo", "bar") 62 | end 63 | 64 | test "renames non existing file", %{store: store} do 65 | assert :ok = FileStore.rename(store, "doesnotexist.txt", "shouldnotexist.txt") 66 | end 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/file_store/adapters/s3_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.S3Test do 2 | use FileStore.AdapterCase 3 | alias FileStore.Adapters.S3 4 | alias FileStore.Stat 5 | 6 | @region "us-east-1" 7 | @bucket "filestore" 8 | @url "http://filestore.localhost:9000/foo" 9 | 10 | @config [ 11 | scheme: "http://", 12 | host: "localhost", 13 | port: 9000, 14 | region: @region, 15 | access_key_id: "development", 16 | secret_access_key: "development", 17 | json_codec: Jason, 18 | retries: [max_attempts: 1] 19 | ] 20 | 21 | setup do 22 | prepare_bucket!() 23 | {:ok, store: S3.new(bucket: @bucket, ex_aws: @config)} 24 | end 25 | 26 | test "get_public_url/3", %{store: store} do 27 | assert FileStore.get_public_url(store, "foo") == @url 28 | end 29 | 30 | test "get_public_url/3 with query params", %{store: store} do 31 | opts = [content_type: "text/plain", disposition: "attachment"] 32 | url = FileStore.get_public_url(store, "foo", opts) 33 | assert omit_query(url) == @url 34 | assert get_query(url, "response-content-type") == "text/plain" 35 | assert get_query(url, "response-content-disposition") == "attachment" 36 | end 37 | 38 | test "get_signed_url/3", %{store: store} do 39 | assert {:ok, url} = FileStore.get_signed_url(store, "foo") 40 | assert omit_query(url) == @url 41 | assert get_query(url, "X-Amz-Expires") == "3600" 42 | end 43 | 44 | test "get_signed_url/3 with query params", %{store: store} do 45 | opts = [content_type: "text/plain", disposition: "attachment"] 46 | assert {:ok, url} = FileStore.get_signed_url(store, "foo", opts) 47 | assert omit_query(url) == @url 48 | assert get_query(url, "X-Amz-Expires") == "3600" 49 | assert get_query(url, "response-content-type") == "text/plain" 50 | assert get_query(url, "response-content-disposition") == "attachment" 51 | end 52 | 53 | test "get_signed_url/3 with custom expiration", %{store: store} do 54 | assert {:ok, url} = FileStore.get_signed_url(store, "foo", expires_in: 4000) 55 | assert omit_query(url) == @url 56 | assert get_query(url, "X-Amz-Expires") == "4000" 57 | end 58 | 59 | describe "write/4" do 60 | test "sends the content-type with the data written", %{store: store} do 61 | :ok = FileStore.write(store, "foo", "{}", content_type: "application/json") 62 | 63 | assert {:ok, %Stat{type: "application/json"}} = FileStore.stat(store, "foo") 64 | end 65 | 66 | test "not sending content-type does not return on stat", %{store: store} do 67 | :ok = FileStore.write(store, "foo", "test") 68 | 69 | assert {:ok, %Stat{type: "application/octet-stream"}} = FileStore.stat(store, "foo") 70 | end 71 | end 72 | 73 | defp prepare_bucket! do 74 | @bucket 75 | |> ExAws.S3.put_bucket(@region) 76 | |> ExAws.request(@config) 77 | |> case do 78 | {:ok, _} -> :ok 79 | {:error, {:http_error, 409, _}} -> clean_bucket!() 80 | {:error, reason} -> raise "Failed to create bucket, error: #{inspect(reason)}" 81 | end 82 | end 83 | 84 | defp clean_bucket! do 85 | @bucket 86 | |> ExAws.S3.delete_all_objects(list_all_keys()) 87 | |> ExAws.request(@config) 88 | |> case do 89 | {:ok, _} -> :ok 90 | {:error, reason} -> raise "Failed to clean bucket, error: #{inspect(reason)}" 91 | end 92 | end 93 | 94 | defp list_all_keys do 95 | @bucket 96 | |> ExAws.S3.list_objects() 97 | |> ExAws.stream!(@config) 98 | |> Stream.map(& &1.key) 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /test/file_store/config_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.ConfigTest do 2 | use ExUnit.Case 3 | 4 | defmodule InlineConfig do 5 | use FileStore.Config, 6 | otp_app: :my_app, 7 | adapter: FileStore.Adapters.Memory, 8 | base_url: "http://example.com" 9 | end 10 | 11 | defmodule ApplicationConfig do 12 | use FileStore.Config, 13 | otp_app: :my_app, 14 | adapter: FileStore.Adapters.Memory 15 | end 16 | 17 | defmodule MiddlewareConfig do 18 | use FileStore.Config, 19 | otp_app: :my_app, 20 | adapter: FileStore.Adapters.Memory, 21 | base_url: "http://example.com", 22 | middleware: [{FileStore.Middleware.Prefix, prefix: "/foo"}] 23 | end 24 | 25 | test "new/0 with inline configuration" do 26 | assert InlineConfig.new() == %FileStore.Adapters.Memory{base_url: "http://example.com"} 27 | end 28 | 29 | test "new/0 with application config" do 30 | Application.put_env(:my_app, ApplicationConfig, base_url: "http://example.com") 31 | assert ApplicationConfig.new() == %FileStore.Adapters.Memory{base_url: "http://example.com"} 32 | end 33 | 34 | test "new/0 with a prefix" do 35 | assert MiddlewareConfig.new() == %FileStore.Middleware.Prefix{ 36 | __next__: %FileStore.Adapters.Memory{base_url: "http://example.com"}, 37 | prefix: "/foo" 38 | } 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/file_store/error_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.ErrorTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias FileStore.Error 5 | alias FileStore.UploadError 6 | alias FileStore.DownloadError 7 | 8 | describe "FileStore.Error" do 9 | test "generates an error message" do 10 | error = %Error{key: "key", action: "read key", reason: "blah"} 11 | assert Exception.message(error) == "could not read key \"key\": \"blah\"" 12 | end 13 | 14 | test "formats posix errors" do 15 | error = %Error{key: "key", action: "read key", reason: :enoent} 16 | 17 | assert Exception.message(error) == 18 | "could not read key \"key\": no such file or directory" 19 | end 20 | end 21 | 22 | describe "UploadError" do 23 | test "generates an error message" do 24 | error = %UploadError{key: "key", path: "path", reason: "blah"} 25 | assert Exception.message(error) == "could not upload file \"path\" to key \"key\": \"blah\"" 26 | end 27 | 28 | test "formats posix errors" do 29 | error = %UploadError{key: "key", path: "path", reason: :enoent} 30 | 31 | assert Exception.message(error) == 32 | "could not upload file \"path\" to key \"key\": no such file or directory" 33 | end 34 | end 35 | 36 | describe "DownloadError" do 37 | test "generates an error message" do 38 | error = %DownloadError{key: "key", path: "path", reason: "blah"} 39 | 40 | assert Exception.message(error) == 41 | "could not download key \"key\" to file \"path\": \"blah\"" 42 | end 43 | 44 | test "formats posix errors" do 45 | error = %DownloadError{key: "key", path: "path", reason: :enoent} 46 | 47 | assert Exception.message(error) == 48 | "could not download key \"key\" to file \"path\": no such file or directory" 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/file_store/middleware/errors_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.ErrorsTest do 2 | use FileStore.AdapterCase 3 | 4 | @config [base_url: "http://localhost:4000"] 5 | 6 | setup do 7 | start_supervised!(FileStore.Adapters.Memory) 8 | store = FileStore.Adapters.Memory.new(@config) 9 | store = FileStore.Middleware.Errors.new(store) 10 | {:ok, store: store} 11 | end 12 | 13 | describe "exceptions" do 14 | setup do 15 | store = FileStore.Adapters.Error.new() 16 | store = FileStore.Middleware.Errors.new(store) 17 | {:ok, store: store} 18 | end 19 | 20 | test "write/4", %{store: store} do 21 | error = %FileStore.Error{ 22 | action: "write to key", 23 | key: "key", 24 | reason: :boom 25 | } 26 | 27 | assert {:error, ^error} = FileStore.write(store, "key", "content") 28 | end 29 | 30 | test "read/2", %{store: store} do 31 | error = %FileStore.Error{ 32 | action: "read key", 33 | key: "key", 34 | reason: :boom 35 | } 36 | 37 | assert {:error, ^error} = FileStore.read(store, "key") 38 | end 39 | 40 | test "upload/3", %{store: store} do 41 | error = %FileStore.UploadError{ 42 | path: "path", 43 | key: "key", 44 | reason: :boom 45 | } 46 | 47 | assert {:error, ^error} = FileStore.upload(store, "path", "key") 48 | end 49 | 50 | test "download/3", %{store: store} do 51 | error = %FileStore.DownloadError{ 52 | path: "path", 53 | key: "key", 54 | reason: :boom 55 | } 56 | 57 | assert {:error, ^error} = FileStore.download(store, "key", "path") 58 | end 59 | 60 | test "stat/2", %{store: store} do 61 | error = %FileStore.Error{ 62 | action: "read stats for key", 63 | key: "key", 64 | reason: :boom 65 | } 66 | 67 | assert {:error, ^error} = FileStore.stat(store, "key") 68 | end 69 | 70 | test "delete/2", %{store: store} do 71 | error = %FileStore.Error{ 72 | action: "delete key", 73 | key: "key", 74 | reason: :boom 75 | } 76 | 77 | assert {:error, ^error} = FileStore.delete(store, "key") 78 | end 79 | 80 | test "delete_all/1", %{store: store} do 81 | error = %FileStore.Error{ 82 | action: "delete all keys", 83 | key: nil, 84 | reason: :boom 85 | } 86 | 87 | assert {:error, ^error} = FileStore.delete_all(store) 88 | end 89 | 90 | test "delete_all/2", %{store: store} do 91 | error = %FileStore.Error{ 92 | action: "delete keys matching prefix", 93 | key: "key", 94 | reason: :boom 95 | } 96 | 97 | assert {:error, ^error} = FileStore.delete_all(store, prefix: "key") 98 | end 99 | 100 | test "copy/3", %{store: store} do 101 | error = %FileStore.CopyError{ 102 | src: "src", 103 | dest: "dest", 104 | reason: :boom 105 | } 106 | 107 | assert {:error, ^error} = FileStore.copy(store, "src", "dest") 108 | end 109 | 110 | test "rename/3", %{store: store} do 111 | error = %FileStore.RenameError{ 112 | src: "src", 113 | dest: "dest", 114 | reason: :boom 115 | } 116 | 117 | assert {:error, ^error} = FileStore.rename(store, "src", "dest") 118 | end 119 | 120 | test "get_signed_url/3", %{store: store} do 121 | error = %FileStore.Error{ 122 | action: "generate signed URL for key", 123 | key: "key", 124 | reason: :boom 125 | } 126 | 127 | assert {:error, ^error} = FileStore.get_signed_url(store, "key") 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /test/file_store/middleware/logger_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.LoggerTest do 2 | use FileStore.AdapterCase 3 | import ExUnit.CaptureLog 4 | require Logger 5 | 6 | @config [base_url: "http://localhost:4000"] 7 | 8 | setup :silence_logger 9 | 10 | setup do 11 | start_supervised!(FileStore.Adapters.Memory) 12 | store = FileStore.Adapters.Memory.new(@config) 13 | store = FileStore.Middleware.Logger.new(store) 14 | {:ok, store: store} 15 | end 16 | 17 | test "logs a successful write", %{store: store} do 18 | Logger.configure(level: :debug) 19 | out = capture_log(fn -> FileStore.write(store, "foo", "bar") end) 20 | assert out =~ ~r/WRITE OK key="foo"/ 21 | end 22 | 23 | test "logs a failed read", %{store: store} do 24 | Logger.configure(level: :debug) 25 | out = capture_log(fn -> FileStore.read(store, "none") end) 26 | assert out =~ ~r/READ ERROR key="none" error=:enoent/ 27 | end 28 | 29 | defp silence_logger(_) do 30 | Logger.configure(level: :none) 31 | 32 | on_exit(fn -> 33 | Logger.configure(level: :debug) 34 | end) 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /test/file_store/middleware/prefix_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Middleware.PrefixTest do 2 | use FileStore.AdapterCase 3 | 4 | alias FileStore.Adapters.Memory 5 | alias FileStore.Middleware.Prefix 6 | 7 | @config [base_url: "http://localhost:4000"] 8 | @plain Memory.new(@config) 9 | 10 | setup do 11 | start_supervised!(Memory) 12 | store = Memory.new(@config) 13 | store = Prefix.new(store, prefix: "prefix") 14 | {:ok, store: store} 15 | end 16 | 17 | test "adds a prefix to keys", %{store: store} do 18 | assert :ok = FileStore.write(store, "foo", "prefixed") 19 | assert {:ok, "prefixed"} = FileStore.read(@plain, "prefix/foo") 20 | end 21 | 22 | test "stat/2 removes prefix from the key", %{store: store} do 23 | assert :ok = FileStore.write(store, "foo", "bar") 24 | 25 | assert {:ok, stat} = FileStore.stat(store, "foo") 26 | assert stat.key == "foo" 27 | 28 | assert {:ok, stat} = FileStore.stat(@plain, "prefix/foo") 29 | assert stat.key == "prefix/foo" 30 | end 31 | 32 | test "list!/2 removes prefix from the key", %{store: store} do 33 | assert :ok = FileStore.write(store, "foo", "bar") 34 | assert "foo" in Enum.to_list(FileStore.list!(store)) 35 | assert "prefix/foo" in Enum.to_list(FileStore.list!(@plain)) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/file_store/stat_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.StatTest do 2 | use ExUnit.Case 3 | doctest FileStore.Stat 4 | end 5 | -------------------------------------------------------------------------------- /test/file_store/utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStore.UtilsTest do 2 | use ExUnit.Case 3 | 4 | alias FileStore.Utils 5 | 6 | test "join/2" do 7 | assert Utils.join("a", "b") == "a/b" 8 | assert Utils.join("a", "b/") == "a/b/" 9 | assert Utils.join("a/", "b") == "a/b" 10 | assert Utils.join("a/", "b/") == "a/b/" 11 | assert Utils.join("a", nil) == "a" 12 | assert Utils.join(nil, "b") == "b" 13 | assert Utils.join(nil, nil) == nil 14 | end 15 | 16 | test "join_absolute/2" do 17 | assert Utils.join_absolute("a", "b") == "/a/b" 18 | assert Utils.join_absolute("a", "b/") == "/a/b/" 19 | assert Utils.join_absolute("/a", "/b/") == "/a/b/" 20 | assert Utils.join_absolute(nil, "b") == "/b" 21 | assert Utils.join_absolute("a", nil) == "/a" 22 | assert Utils.join_absolute(nil, nil) == nil 23 | end 24 | 25 | test "append_path/2" do 26 | assert %URI{path: "/bar"} = 27 | "http://example.com" 28 | |> URI.parse() 29 | |> Utils.append_path("bar") 30 | 31 | assert %URI{path: "/foo/bar"} = 32 | "http://example.com/foo" 33 | |> URI.parse() 34 | |> Utils.append_path("bar") 35 | 36 | assert %URI{path: "/foo/bar"} = 37 | "http://example.com/foo/" 38 | |> URI.parse() 39 | |> Utils.append_path("/bar") 40 | end 41 | 42 | test "put_query/2" do 43 | assert %URI{query: "foo=bar"} = 44 | "http://example.com" 45 | |> URI.parse() 46 | |> Utils.put_query(foo: "bar") 47 | end 48 | 49 | describe "rename_key/3" do 50 | test "renames existing key" do 51 | assert [changed: :foo] = Utils.rename_key([target: :foo], :target, :changed) 52 | end 53 | 54 | test "returns keywords unchanged when target does not exist" do 55 | assert [left: :foo] = Utils.rename_key([left: :foo], :target, :changed) 56 | end 57 | 58 | test "returns empty keywords" do 59 | assert [] = Utils.rename_key([], :target, :changed) 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /test/file_store_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FileStoreTest do 2 | use ExUnit.Case 3 | alias FileStore.Adapters.Null, as: Adapter 4 | 5 | @key "test" 6 | @path "test/fixtures/test.txt" 7 | @content "blah" 8 | @store Adapter.new() 9 | 10 | test "new/1" do 11 | assert @store == %Adapter{} 12 | end 13 | 14 | test "get_public_url/2" do 15 | assert FileStore.get_public_url(@store, @key) == @key 16 | end 17 | 18 | test "get_signed_url/2" do 19 | assert FileStore.get_signed_url(@store, @key) == {:ok, @key} 20 | end 21 | 22 | test "delete/2" do 23 | assert FileStore.delete(@store, @key) == :ok 24 | end 25 | 26 | test "upload/3" do 27 | assert :ok = FileStore.upload(@store, @path, @key) 28 | end 29 | 30 | test "write/3" do 31 | assert :ok = FileStore.write(@store, @key, @content) 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /test/fixtures/test.txt: -------------------------------------------------------------------------------- 1 | blah 2 | -------------------------------------------------------------------------------- /test/support/adapter_case.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.AdapterCase do 2 | @moduledoc false 3 | 4 | use ExUnit.CaseTemplate 5 | 6 | @tmp Path.join(System.tmp_dir!(), "file_store") 7 | 8 | setup do 9 | File.rm_rf!(@tmp) 10 | File.mkdir!(@tmp) 11 | {:ok, tmp: @tmp} 12 | end 13 | 14 | using do 15 | # credo:disable-for-this-file Credo.Check.Refactor.LongQuoteBlocks 16 | quote location: :keep do 17 | import FileStore.AdapterCase 18 | 19 | describe "write/4 conformance" do 20 | test "writes a file", %{store: store} do 21 | assert :ok = FileStore.write(store, "foo", "bar") 22 | assert {:ok, "bar"} = FileStore.read(store, "foo") 23 | end 24 | 25 | test "overwrites a file", %{store: store} do 26 | assert :ok = FileStore.write(store, "foo", "bar") 27 | assert {:ok, "bar"} = FileStore.read(store, "foo") 28 | 29 | assert :ok = FileStore.write(store, "foo", "baz") 30 | assert {:ok, "baz"} = FileStore.read(store, "foo") 31 | end 32 | end 33 | 34 | describe "read/2 conformance" do 35 | test "reads a file", %{store: store} do 36 | assert :ok = FileStore.write(store, "foo", "bar") 37 | assert {:ok, "bar"} = FileStore.read(store, "foo") 38 | end 39 | 40 | test "errors when file does not exist", %{store: store} do 41 | assert {:error, _} = FileStore.read(store, "does-not-exist") 42 | end 43 | end 44 | 45 | describe "upload/3 conformance" do 46 | test "uploads a file", %{store: store} do 47 | bar = write("bar.txt", "bar") 48 | 49 | assert :ok = FileStore.upload(store, bar, "foo") 50 | assert {:ok, "bar"} = FileStore.read(store, "foo") 51 | end 52 | 53 | test "overwrites a file", %{store: store} do 54 | bar = write("bar.txt", "bar") 55 | baz = write("baz.txt", "baz") 56 | 57 | assert :ok = FileStore.upload(store, bar, "foo") 58 | assert {:ok, "bar"} = FileStore.read(store, "foo") 59 | 60 | assert :ok = FileStore.upload(store, baz, "foo") 61 | assert {:ok, "baz"} = FileStore.read(store, "foo") 62 | end 63 | 64 | test "fails when the source file is missing", %{store: store} do 65 | assert {:error, _} = FileStore.upload(store, "doesnotexist.txt", "foo") 66 | end 67 | end 68 | 69 | describe "download/3 conformance" do 70 | test "downloads a file", %{store: store} do 71 | download = join("download.txt") 72 | 73 | assert :ok = FileStore.write(store, "foo", "bar") 74 | assert :ok = FileStore.download(store, "foo", download) 75 | assert File.read!(download) == "bar" 76 | end 77 | end 78 | 79 | describe "stat/2 conformance" do 80 | test "retrieves file info", %{store: store} do 81 | assert :ok = FileStore.write(store, "foo", "bar") 82 | assert {:ok, stat} = FileStore.stat(store, "foo") 83 | assert stat.key == "foo" 84 | assert stat.size == 3 85 | assert stat.etag == "37b51d194a7513e45b56f6524f2d51f2" 86 | end 87 | 88 | test "fails when the file is missing", %{store: store} do 89 | assert {:error, _} = FileStore.stat(store, "completegarbage") 90 | end 91 | end 92 | 93 | describe "delete/2 conformance" do 94 | test "deletes the file", %{store: store} do 95 | assert :ok = FileStore.write(store, "foo", "bar") 96 | assert :ok = FileStore.delete(store, "foo") 97 | end 98 | 99 | test "indicates success for non-existent keys", %{store: store} do 100 | assert :ok = FileStore.delete(store, "non-existent") 101 | assert :ok = FileStore.delete(store, "non/existent") 102 | end 103 | end 104 | 105 | describe "delete_all/2 conformance" do 106 | test "deletes all files", %{store: store} do 107 | assert :ok = FileStore.write(store, "foo", "") 108 | assert :ok = FileStore.write(store, "bar/buzz", "") 109 | assert :ok = FileStore.delete_all(store) 110 | assert {:error, _} = FileStore.stat(store, "foo") 111 | assert {:error, _} = FileStore.stat(store, "bar/buzz") 112 | end 113 | 114 | test "deletes files under prefix", %{store: store} do 115 | assert :ok = FileStore.write(store, "foo", "") 116 | assert :ok = FileStore.write(store, "bar/buzz", "") 117 | assert :ok = FileStore.write(store, "bar/baz", "") 118 | assert :ok = FileStore.delete_all(store, prefix: "bar") 119 | assert {:ok, _} = FileStore.stat(store, "foo") 120 | assert {:error, _} = FileStore.stat(store, "bar/buzz") 121 | assert {:error, _} = FileStore.stat(store, "bar/baz") 122 | end 123 | 124 | test "indicates success for non-existent keys", %{store: store} do 125 | assert :ok = FileStore.delete_all(store, prefix: "non-existent") 126 | end 127 | end 128 | 129 | describe "get_public_url/3 conformance" do 130 | test "returns a URL", %{store: store} do 131 | assert :ok = FileStore.write(store, "foo", "bar") 132 | assert url = FileStore.get_public_url(store, "foo") 133 | assert valid_url?(url) 134 | end 135 | end 136 | 137 | describe "get_signed_url/3 conformance" do 138 | test "returns a URL", %{store: store} do 139 | assert :ok = FileStore.write(store, "foo", "bar") 140 | assert {:ok, url} = FileStore.get_signed_url(store, "foo") 141 | assert valid_url?(url) 142 | end 143 | end 144 | 145 | describe "list!/2 conformance" do 146 | test "lists keys in the store", %{store: store} do 147 | assert :ok = FileStore.write(store, "foo", "") 148 | assert "foo" in Enum.to_list(FileStore.list!(store)) 149 | end 150 | 151 | test "lists nested keys in the store", %{store: store} do 152 | assert :ok = FileStore.write(store, "foo/bar", "") 153 | assert "foo/bar" in Enum.to_list(FileStore.list!(store)) 154 | end 155 | 156 | test "lists keys matching prefix", %{store: store} do 157 | assert :ok = FileStore.write(store, "bar", "") 158 | assert :ok = FileStore.write(store, "foo/bar", "") 159 | 160 | keys = Enum.to_list(FileStore.list!(store, prefix: "foo")) 161 | refute "bar" in keys 162 | assert "foo/bar" in keys 163 | end 164 | end 165 | 166 | describe "copy/3 conformance" do 167 | test "copies a file", %{store: store} do 168 | :ok = FileStore.write(store, "foo", "test") 169 | 170 | assert :ok = FileStore.copy(store, "foo", "bar") 171 | assert {:ok, "test"} = FileStore.read(store, "foo") 172 | end 173 | 174 | test "fails to copy a non existing file", %{store: store} do 175 | assert {:error, _} = FileStore.copy(store, "foo", "bar") 176 | assert {:error, _} = FileStore.stat(store, "bar") 177 | end 178 | 179 | test "copy replaces existing file", %{store: store} do 180 | :ok = FileStore.write(store, "foo", "test") 181 | :ok = FileStore.write(store, "bar", "i exist") 182 | 183 | assert :ok = FileStore.copy(store, "foo", "bar") 184 | assert {:ok, "test"} = FileStore.read(store, "bar") 185 | end 186 | end 187 | 188 | describe "rename/3 conformance" do 189 | test "renames a file", %{store: store} do 190 | :ok = FileStore.write(store, "foo", "test") 191 | 192 | assert :ok = FileStore.rename(store, "foo", "bar") 193 | assert {:error, _} = FileStore.stat(store, "foo") 194 | assert {:ok, _} = FileStore.stat(store, "bar") 195 | end 196 | 197 | test "fails to rename a non existing file", %{store: store} do 198 | assert {:error, _} = FileStore.rename(store, "foo", "bar") 199 | assert {:error, _} = FileStore.stat(store, "bar") 200 | end 201 | 202 | test "rename replaces existing file", %{store: store} do 203 | :ok = FileStore.write(store, "foo", "test") 204 | :ok = FileStore.write(store, "bar", "i exist") 205 | 206 | assert :ok = FileStore.rename(store, "foo", "bar") 207 | assert {:error, _} = FileStore.stat(store, "foo") 208 | assert {:ok, _} = FileStore.stat(store, "bar") 209 | end 210 | end 211 | end 212 | end 213 | 214 | def join(name) do 215 | Path.join(@tmp, name) 216 | end 217 | 218 | def write(name, data) do 219 | path = join(name) 220 | File.write!(path, data) 221 | path 222 | end 223 | 224 | def valid_url?(value) do 225 | case URI.parse(value) do 226 | %URI{scheme: nil} -> false 227 | %URI{host: nil} -> false 228 | %URI{scheme: scheme} -> scheme =~ ~r"^https?$" 229 | end 230 | end 231 | 232 | def get_query(url, param) do 233 | url 234 | |> URI.parse() 235 | |> Map.fetch!(:query) 236 | |> URI.decode_query() 237 | |> Map.fetch!(param) 238 | end 239 | 240 | def omit_query(url) do 241 | url 242 | |> URI.parse() 243 | |> Map.put(:query, nil) 244 | |> URI.to_string() 245 | end 246 | end 247 | -------------------------------------------------------------------------------- /test/support/error_adapter.ex: -------------------------------------------------------------------------------- 1 | defmodule FileStore.Adapters.Error do 2 | @moduledoc false 3 | 4 | defstruct [] 5 | 6 | def new do 7 | %__MODULE__{} 8 | end 9 | 10 | defimpl FileStore do 11 | def write(_store, _key, _content, _opts \\ []), do: {:error, :boom} 12 | def read(_store, _key), do: {:error, :boom} 13 | def upload(_store, _source, _key), do: {:error, :boom} 14 | def download(_store, _key, _destination), do: {:error, :boom} 15 | def stat(_store, _key), do: {:error, :boom} 16 | def delete(_store, _key), do: {:error, :boom} 17 | def delete_all(_store, _opts \\ []), do: {:error, :boom} 18 | def copy(_store, _src, _dest), do: {:error, :boom} 19 | def rename(_store, _src, _dest), do: {:error, :boom} 20 | def get_public_url(_store, key, _opts \\ []), do: key 21 | def get_signed_url(_store, _key, _opts \\ []), do: {:error, :boom} 22 | def list!(_store, _opts \\ []), do: [] 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | {:ok, _} = Application.ensure_all_started(:hackney) 2 | {:ok, _} = Application.ensure_all_started(:telemetry) 3 | 4 | ExUnit.start() 5 | --------------------------------------------------------------------------------