├── .envrc ├── .formatter.exs ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── TESTING.md ├── lib └── fly_rpc.ex ├── mix.exs ├── mix.lock └── test ├── fly_rpc_test.exs └── test_helper.exs /.envrc: -------------------------------------------------------------------------------- 1 | export MY_REGION=abc 2 | export PRIMARY_REGION=xyz -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | _build/ 3 | .elixir_ls/ 4 | 5 | # If you run "mix test --cover", coverage assets end up here. 6 | /cover/ 7 | 8 | # The directory Mix downloads your dependencies sources to. 9 | /deps/ 10 | 11 | # Where third-party dependencies like ExDoc output generated docs. 12 | /doc/ 13 | 14 | # Ignore .fetch files in case you like to edit your project deps locally. 15 | /.fetch 16 | 17 | # If the VM crashes, it generates a dump, let's ignore it too. 18 | erl_crash.dump 19 | 20 | # Also ignore archive artifacts (built via "mix archive.build"). 21 | *.ez 22 | 23 | # Ignore package tarball (built via "mix hex.build"). 24 | fly_rpc-*.tar 25 | 26 | # Temporary files, for example, from tests. 27 | /tmp/ 28 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.3.0 4 | 5 | Breaking changes: 6 | - Functions at the room `Fly` module were refactored into the `Fly.RPC` module. 7 | - It should be an easy update where a call to `Fly.is_primary?` becomes `Fly.RPC.is_primary?`. 8 | - This opens up the `Fly` namespace for other libraries as well. 9 | - `Fly.rpc_region/5` was changed to `Fly.RPC.rpc_region/3`. 10 | - The namespace changed. 11 | - The way the function to execute is passed in has changed. It now supports passing in an anonymous function like this: `Fly.RPC.rpc_region("hkg", fn -> 1 + 2 end)` 12 | - It still supports the MFA format (Module, Function, Arguments) but now uses a tuple like this: `Fly.RPC.rpc_region(:primary, {Kernel, :+, [1, 2]})` 13 | 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fly RPC 2 | 3 | Helps a clustered Elixir application know what [Fly.io](https:://fly.io) region it is deployed in, if that is the primary region, and provides features for executing a function through RPC (Remote Procedure Call) on another node by specifying the region to run it in. It is specifically designed to make it easier to execute code in the "primary" region. 4 | 5 | This library can be used outside of the [Fly.io](https://fly.io) platform as well. In order to work, the nodes need to be clustered and then set `PRIMARY_REGION` and `MY_REGION` ENV values. Everything else works the same. When running on [Fly.io](https://fly.io), the `FLY_REGION` ENV value provided by the platform is used for `MY_REGION`. 6 | 7 | The "primary" region refers to which region your primary, writeable database lives in. Writes to the primary database should ideally be performed from a server running in, or near, the primary region. 8 | 9 | [Online Documentation](https://hexdocs.pm/fly_rpc) 10 | 11 | ## Installation 12 | 13 | If [available in Hex](https://hex.pm/docs/publish), the package can be installed 14 | by adding `fly_rpc` to your list of dependencies in `mix.exs`: 15 | 16 | ```elixir 17 | def deps do 18 | [ 19 | {:fly_rpc, "~> 0.3.0"} 20 | ] 21 | end 22 | ``` 23 | 24 | Through ENV configuration, you can to tell the app which region is the "primary" region. 25 | 26 | `fly.toml` 27 | 28 | This example configuration says that the Sydney Australia region is the 29 | "primary" region. 30 | 31 | ```yaml 32 | [env] 33 | PRIMARY_REGION = "syd" 34 | ``` 35 | 36 | Add `Fly.RPC` to your application's supervision tree. 37 | 38 | ```elixir 39 | defmodule MyApp.Application do 40 | use Application 41 | 42 | def start(_type, _args) do 43 | # ... 44 | 45 | children = [ 46 | # Start the RPC server 47 | {Fly.RPC, []}, 48 | #... 49 | ] 50 | 51 | # ... 52 | end 53 | end 54 | ``` 55 | 56 | This starts a GenServer that reaches out to other nodes in the cluster to learn 57 | what Fly.io regions they are located in. The GenServer caches those results in a 58 | local ETS table for fast access. 59 | 60 | ## Usage 61 | 62 | The Fly.io platform already provides and ENV value of `FLY_REGION` which this library accesses and uses as the `MY_REGION`. When using this library on a platform other than [fly.io](https://fly.io), you can supply the ENV `MY_REGION` to identify what "region" the running instance is in. Think of the value as a text label of however you want to identify where it's running. 63 | 64 | ```elixir 65 | Fly.RPC.primary_region() 66 | #=> "syd" 67 | 68 | Fly.RPC.my_region() 69 | #=> "lax" 70 | 71 | Fly.RPC.is_primary?() 72 | #=> false 73 | ``` 74 | 75 | The real benefit comes in using the `Fly.RPC` module. 76 | 77 | ```elixir 78 | Fly.RPC.rpc_region("hkg", String, :upcase, ["fly"]) 79 | #=> "FLY" 80 | 81 | Fly.RPC.rpc_region(Fly.primary_region(), String, :upcase, ["fly"]) 82 | #=> "FLY" 83 | 84 | Fly.RPC.rpc_region(:primary, String, :upcase, ["fly"]) 85 | #=> "FLY" 86 | ``` 87 | 88 | Underneath the call, it's using `Node.spawn_link/4`. This spawns a new process on a node in the desired region. Normally, that spawn becomes an asynchronous process and what you get back is a `pid`. In this case, the call executes on the other node and the caller is blocked until the result is received or the request times out. 89 | 90 | By blocking the process, this makes it much easier to reason about your application code. 91 | 92 | 93 | The following is a convenience function for performing work on the primary. 94 | 95 | ```elixir 96 | Fly.RPC.rpc_primary(String, :upcase, ["fly"]) 97 | #=> "FLY" 98 | ``` 99 | 100 | ## Local Development 101 | 102 | When doing local development, the local and primary regions will be set to "local" by default. However, if we want to simulate running in a non-primary region locally, we can set the `MY_REGION` and `PRIMARY_REGION` environment variables explicitly: 103 | 104 | - `MY_REGION` - You tell the library what region it is running in. 105 | - `PRIMARY_REGION` - You tell the library which region is the "primary". 106 | 107 | By default, the value `"local"` is used for the regions. This works perfectly for local development as we are, effectively, the primary anyway. 108 | 109 | ## Explicitly Set the Region 110 | 111 | When running locally and we explicitly want to set the regions, the `MY_REGION` isn't set since the app isn't on Fly.io. Also, the `PRIMARY_REGION` specified in our `fly.toml` file isn't referenced. We just need a way to set those values when the application is running locally. 112 | 113 | I like using [direnv](https://direnv.net/) to automatically set and load ENV values when I enter specific directories. Using `direnv`, you can create a file named `.envrc` in your project directory. Add the following lines: 114 | 115 | ``` 116 | export MY_REGION=xyz 117 | export PRIMARY_REGION=xyz 118 | ``` 119 | 120 | This tells the app that it's running in the primary region called "xyz". It will connect to the database and perform writes directly. 121 | 122 | Another option is to start you application like this: 123 | 124 | ``` 125 | MY_REGION=xyz PRIMARY_REGION=xyz iex -S mix phx.server 126 | ``` 127 | 128 | You can also create a bash script file named `start` and have it perform the above command. 129 | 130 | ## Production Environment 131 | 132 | ### Prevent temporary outages during deployments 133 | 134 | When deploying on [Fly.io](https://fly.io), a new instance is rolled out before removing the old instance. This creates a period of time where both new and old instances are deployed together. By default, when deploying a Phoenix application, a new BEAM cookie is generated for each deployment. When the new instance rolls out with a new BEAM cookie, the old and new instances will not cluster together. BEAM instances must have the same cookie in order to connect. This is by design. 135 | 136 | This means a newly deployed application running in a secondary region using [fly_postgres](https://github.com/superfly/fly_postgres_elixir) is unable to perform writes to the older application running in the primary region. It is possible for writes to fail during that rollout window. 137 | 138 | To prevent this problem, the BEAM cookie can be explicitly set instead of using a randomly generated one for new builds. When explicitly set, the newly deployed application is still able to connect and cluster with the older application running in the primary region. 139 | 140 | Here is a guide to setting a static cookie for your project that is written into the code itself. This is fine to do because the cookie isn't considered a secret used for security. 141 | 142 | [fly.io/docs/app-guides/elixir-static-cookie/](https://fly.io/docs/app-guides/elixir-static-cookie/) 143 | 144 | When the cookie is static and unchanged from one deployment to the next, then applications can continue to cluster and access the applications running in primary region. 145 | 146 | ### Where Did My Function Go? 147 | 148 | When deploying on [Fly.io](https://fly.io), a new instance is rolled out before removing the old instance. This creates a period of time where both new and old instances are deployed together. 149 | 150 | In this scenario, let's assume: 151 | - Node A is an old node. 152 | - Node B is a new node. 153 | 154 | Node A attempts to execute a function on Node B. Node B contains new code and that function doesn't exist as called. Perhaps the function was renamed, the arity changed, a pattern match changed or it's a new function. 155 | 156 | Whatever the reason, we now have a situation where the function we want to call fails because it doesn't exist the way we expect it on the new node in the cluster. 157 | 158 | Be aware that this _can_ happen. It may cause a single request to fail and that may be fine. We can take steps in our applications to avoid any interruption if it's a critical function that needs to change. 159 | 160 | For critical functions, the following pattern can be used: 161 | - Create the new, changed function 162 | - Maintain backward compatibility with another function (if applicable) 163 | - Deploy the new code moving all instances to use the new, desired function. During the deploy, the backward compatible function prevent breakage. 164 | - A later deploy removes the backward compatible function. 165 | 166 | ## Features 167 | 168 | - [ ] Instrument with telemetry 169 | -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | Instructions for testing and developing multi-node RPC applications locally in a development environment. 4 | 5 | ## Start Nodes Locally 6 | 7 | Start multiple nodes locally on the same developer machine. Multiple nodes can be started in separate terminals. 8 | 9 | Node 1 - Primary Region: 10 | 11 | ```shell 12 | MY_REGION=xyz PRIMARY_REGION=xyz iex --name node1@127.0.0.1 -S mix 13 | ``` 14 | 15 | Node 2 - Non-Primary Region: 16 | 17 | ```shell 18 | MY_REGION=abc PRIMARY_REGION=xyz iex --name node2@127.0.0.1 -S mix 19 | ``` 20 | 21 | In the IEx shell, run this command to connect `node2` to `node1`. 22 | 23 | ```elixir 24 | Node.connect(:"node1@127.0.0.1") 25 | ``` 26 | 27 | The following command verifies the nodes are connected by showing the other connected node. 28 | 29 | ```elixir 30 | Node.list 31 | ``` 32 | 33 | The nodes are now verified as connected. Commands can be executed from either node to the other. 34 | 35 | NOTE: If running the test on the library itself, it won't work because the GenServer must be started first. Testing the library as part of another application is easier as the functions to execute on the other node are present and the GenServer will have started with the Application supervision tree. To start the GenServer manually in each node, execute: `Fly.RPC.start_link([])` 36 | 37 | Example: 38 | 39 | ```elixir 40 | Fly.rpc_primary(String, :upcase, ["fly"]) 41 | #=> "FLY" 42 | ``` 43 | 44 | Additional nodes can be started similarly with additional regions for testing. 45 | -------------------------------------------------------------------------------- /lib/fly_rpc.ex: -------------------------------------------------------------------------------- 1 | defmodule Fly.RPC do 2 | @moduledoc """ 3 | Performs RPC calls to nodes in Fly.io regions. 4 | 5 | Provides features to help Elixir applications more easily take advantage 6 | of the features that Fly.io provides. 7 | 8 | ## Configuration 9 | 10 | Assumes each node is running the `Fly.RPC` server in its supervision tree and 11 | exports `FLY_REGION` environment variable to identify the fly region. 12 | 13 | *Note*: anonymous function support only works when the release is identical 14 | across all nodes. This can be ensured by including the `FLY_IMAGE_REF` as part of 15 | the node name in your `rel/env.sh.eex` file: 16 | 17 | #!/bin/sh 18 | 19 | export ERL_AFLAGS="-proto_dist inet6_tcp" 20 | export RELEASE_DISTRIBUTION=name 21 | export RELEASE_NODE="${FLY_APP_NAME}-${FLY_IMAGE_REF##*-}@${FLY_PRIVATE_IP}" 22 | 23 | To run code on a specific region call `rpc_region/4`. A node found within the 24 | given region will be chosen at random. Raises if no nodes exist on the given 25 | region. 26 | 27 | The special `:primary` region may be passed to run the rpc against the region 28 | identified by the `PRIMARY_REGION` environment variable. 29 | 30 | ## Examples 31 | 32 | > rpc_region("hkg", fn -> String.upcase("fly") end) 33 | "FLY" 34 | 35 | > rpc_region("hkg", {String, :upcase, ["fly"]}) 36 | "FLY" 37 | 38 | > rpc_region(Fly.RPC.primary_region(), {String, :upcase, ["fly"]]) 39 | "FLY" 40 | 41 | > rpc_region(:primary, {String, :upcase, ["fly"]}) 42 | "FLY" 43 | 44 | ## Server 45 | 46 | The GenServer's responsibility is just to monitor other nodes as they enter 47 | and leave the cluster. It maintains a list of nodes and the Fly.io region 48 | where they are deployed in an ETS table that other processes can use to find 49 | and initiate their own RPC calls to. 50 | """ 51 | use GenServer 52 | require Logger 53 | 54 | @tab :fly_regions 55 | 56 | @doc """ 57 | Return the configured primary region. 58 | 59 | Reads and requires an ENV setting for `PRIMARY_REGION`. 60 | If not set, it returns `"local"`. 61 | """ 62 | def primary_region do 63 | case System.fetch_env("PRIMARY_REGION") do 64 | {:ok, region} -> region 65 | :error -> "local" 66 | end 67 | end 68 | 69 | @doc """ 70 | Return the configured current region. 71 | 72 | Reads the `FLY_REGION` ENV setting 73 | available when deployed on the Fly.io platform. When running on a different 74 | platform, that ENV value will not be set. Setting the `MY_REGION` ENV value 75 | instructs the node how to identify what "region" it is in. If not set, it 76 | returns `"local"`. 77 | 78 | The value itself is not important. If the value matches the value for the 79 | `PRIMARY_REGION` then it behaves as though it is the primary. 80 | """ 81 | def my_region do 82 | case System.get_env("FLY_REGION") || System.get_env("MY_REGION") do 83 | nil -> 84 | System.put_env("MY_REGION", "local") 85 | "local" 86 | 87 | region -> 88 | region 89 | end 90 | end 91 | 92 | @doc """ 93 | Return if the app instance is running in the primary region or not. 94 | """ 95 | def is_primary? do 96 | my_region() == primary_region() 97 | end 98 | 99 | def start_link(opts) do 100 | GenServer.start_link(__MODULE__, opts, name: __MODULE__) 101 | end 102 | 103 | @doc """ 104 | Returns the Elixir OTP nodes registered the region. Reads from a local cache. 105 | """ 106 | def region_nodes(tab \\ @tab, region) do 107 | case :ets.lookup(tab, region) do 108 | [{^region, nodes}] -> nodes 109 | [] -> [] 110 | end 111 | end 112 | 113 | @doc """ 114 | Asks a node what Fly region it's running in. 115 | 116 | Returns `:error` if RPC is not supported on remote node. 117 | """ 118 | def region(node) do 119 | if is_rpc_supported?(node) do 120 | {:ok, rpc(node, {__MODULE__, :my_region, []})} 121 | else 122 | Logger.info("Detected Fly.RPC support is not available on node #{inspect(node)}") 123 | :error 124 | end 125 | end 126 | 127 | @doc """ 128 | Executes the MFA on an available node in the desired region. 129 | 130 | If the region is the "primary" region or the "local" region then execute the 131 | function immediately. Supports the string name of the region or `:primary` for 132 | the current configured primary region. 133 | 134 | Otherwise find an available node and select one at random to execute the 135 | function. 136 | 137 | Raises `ArgumentError` when no available nodes. 138 | 139 | ## Example 140 | 141 | > RPC.rpc_region("hkg", fn -> 1 + 2 end) 142 | 3 143 | 144 | > RPC.rpc_region("hkg", {Kernel, :+, [1, 2]}) 145 | 3 146 | 147 | > RPC.rpc_region(:primary, {Kernel, :+, [1, 2]}) 148 | 3 149 | 150 | """ 151 | def rpc_region(region, func, opts \\ []) 152 | 153 | def rpc_region(region, func, opts) 154 | when (is_binary(region) or region == :primary) and (is_function(func, 0) or is_tuple(func)) and 155 | is_list(opts) do 156 | region = if region == :primary, do: primary_region(), else: region 157 | 158 | if region == my_region() do 159 | invoke(func) 160 | else 161 | timeout = Keyword.get(opts, :timeout, 5_000) 162 | available_nodes = region_nodes(region) 163 | 164 | if Enum.empty?(available_nodes), 165 | do: raise(ArgumentError, "no node found running in region #{inspect(region)}") 166 | 167 | node = Enum.random(available_nodes) 168 | 169 | rpc(node, func, timeout) 170 | end 171 | end 172 | 173 | def rpc_region(region, {mod, func, args}, opts) 174 | when is_binary(region) and is_atom(mod) and is_list(args) and is_list(opts) do 175 | rpc_region(region, fn -> apply(mod, func, args) end, opts) 176 | end 177 | 178 | @doc """ 179 | Execute the MFA on a node in the primary region. 180 | """ 181 | def rpc_primary(func, opts \\ []) 182 | 183 | def rpc_primary(func, opts) when is_function(func, 0) do 184 | rpc_region(:primary, func, opts) 185 | end 186 | 187 | def rpc_primary({module, func, args}, opts) do 188 | rpc_region(:primary, {module, func, args}, opts) 189 | end 190 | 191 | defp invoke(func) when is_function(func, 0), do: func.() 192 | defp invoke({mod, func, args}), do: apply(mod, func, args) 193 | 194 | @doc """ 195 | Executes the function on the remote node and waits for the response. 196 | 197 | Exits after `timeout` milliseconds. 198 | """ 199 | def rpc(node, func, timeout \\ 5000) do 200 | verbose_log(:info, func, "SEND") 201 | 202 | case erpc_call(node, func, timeout) do 203 | {:ok, result} -> 204 | verbose_log(:info, func, "RESP") 205 | 206 | result 207 | 208 | {:error, {:erpc, :timeout}} -> 209 | verbose_log(:error, func, "TIMEOUT") 210 | exit(:timeout) 211 | 212 | {:error, {:erpc, reason}} -> 213 | {:error, {:erpc, reason}} 214 | 215 | {:error, {:throw, value}} -> 216 | throw(value) 217 | 218 | {:error, {:exit, reason}} -> 219 | exit(reason) 220 | 221 | {:error, {_exception, reason, stack}} -> 222 | reraise(reason, stack) 223 | end 224 | end 225 | 226 | @doc """ 227 | Executes a function on the remote node to determine if the RPC API support is 228 | available. 229 | 230 | Support may not exist on the remote node in a "first roll out" scenario. 231 | """ 232 | def is_rpc_supported?(node) do 233 | case erpc_call(node, {Kernel, :function_exported?, [__MODULE__, :my_region, 0]}, 5000) do 234 | {:ok, result} when is_boolean(result) -> 235 | result 236 | 237 | {:error, reason} -> 238 | Logger.warning("Failed RPC supported test on #{inspect(node)}, got: #{inspect(reason)}") 239 | false 240 | end 241 | end 242 | 243 | defp erpc_call(node, {mod, func, args}, timeout) do 244 | try do 245 | {:ok, :erpc.call(node, mod, func, args, timeout)} 246 | catch 247 | :throw, value -> {:error, {:throw, value}} 248 | :exit, reason -> {:error, {:exit, reason}} 249 | :error, {:erpc, reason} -> {:error, {:erpc, reason}} 250 | :error, {exception, reason, stack} -> {:error, {exception, reason, stack}} 251 | end 252 | end 253 | 254 | defp erpc_call(node, func, timeout) when is_function(func, 0) do 255 | try do 256 | {:ok, :erpc.call(node, func, timeout)} 257 | catch 258 | :throw, value -> {:error, {:throw, value}} 259 | :exit, reason -> {:error, {:exit, reason}} 260 | :error, {:erpc, reason} -> {:error, {:erpc, reason}} 261 | :error, {exception, reason, stack} -> {:error, {exception, reason, stack}} 262 | end 263 | end 264 | 265 | ## RPC calls run on local node 266 | 267 | def init(_opts) do 268 | tab = :ets.new(@tab, [:named_table, :public, read_concurrency: true]) 269 | # monitor new node up/down activity 270 | :global_group.monitor_nodes(true) 271 | {:ok, %{nodes: MapSet.new(), tab: tab}, {:continue, :get_node_regions}} 272 | end 273 | 274 | def handle_continue(:get_node_regions, state) do 275 | new_state = 276 | Enum.reduce(Node.list(:visible), state, fn node_name, acc -> 277 | put_node(acc, node_name) 278 | end) 279 | 280 | {:noreply, new_state} 281 | end 282 | 283 | def handle_info({:nodeup, node_name}, state) do 284 | Logger.debug("nodeup #{node_name}") 285 | 286 | # Only react/track visible nodes (hidden ones are for IEx, etc) 287 | if node_name in Node.list(:visible) do 288 | {:noreply, put_node(state, node_name)} 289 | else 290 | {:noreply, state} 291 | end 292 | end 293 | 294 | def handle_info({:nodedown, node_name}, state) do 295 | Logger.debug("nodedown #{node_name}") 296 | {:noreply, drop_node(state, node_name)} 297 | end 298 | 299 | # Executed when a new node shows up in the cluster. Asks the node what region 300 | # it's running in. If the request isn't supported by the node, do nothing. 301 | # This happens when this node is the first node with this new code. It reaches 302 | # out to the other nodes (they show up as having just appeared) but they don't 303 | # yet have the new code. So this ignores that node until it gets new code, 304 | # restarts and will then again show up as a new node. 305 | @doc false 306 | def put_node(state, node_name) do 307 | case region(node_name) do 308 | {:ok, region} -> 309 | Logger.info("Discovered node #{inspect(node_name)} in region #{region}") 310 | region_nodes = region_nodes(state.tab, region) 311 | :ets.insert(state.tab, {region, [node_name | region_nodes]}) 312 | 313 | %{state | nodes: MapSet.put(state.nodes, {node_name, region})} 314 | 315 | :error -> 316 | state 317 | end 318 | end 319 | 320 | @doc false 321 | def drop_node(state, node_name) do 322 | # find the node information for the node going down. 323 | case get_node(state, node_name) do 324 | {^node_name, region} -> 325 | Logger.info("Dropping node #{inspect(node_name)} for region #{region}") 326 | # get the list of nodes currently registered in that region 327 | region_nodes = region_nodes(state.tab, region) 328 | # Remove the node from the known regions and update the local cache 329 | new_regions = Enum.reject(region_nodes, fn n -> n == node_name end) 330 | :ets.insert(state.tab, {region, new_regions}) 331 | 332 | # Remove the node entry from the GenServer's state 333 | new_nodes = 334 | Enum.reduce(state.nodes, state.nodes, fn 335 | {^node_name, ^region}, acc -> MapSet.delete(acc, {node_name, region}) 336 | {_node, _region}, acc -> acc 337 | end) 338 | 339 | # Return the new state 340 | %{state | nodes: new_nodes} 341 | 342 | # Node is not known to us. Ignore it. 343 | nil -> 344 | state 345 | end 346 | end 347 | 348 | defp get_node(state, name) do 349 | Enum.find(state.nodes, fn {n, _region} -> n == name end) 350 | end 351 | 352 | defp verbose_log(kind, func, subject) do 353 | if Application.get_env(:fly_rpc, :verbose_logging) do 354 | Logger.log(kind, fn -> "RPC #{subject} from #{my_region()} #{mfa_string(func)}" end) 355 | end 356 | end 357 | 358 | @doc false 359 | # Also used by fly_postgres 360 | def mfa_string(func) when is_function(func), do: inspect(func) 361 | 362 | def mfa_string({mod, func, args}) do 363 | "#{Atom.to_string(mod)}.#{Atom.to_string(func)}/#{length(args)}" 364 | end 365 | end 366 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Fly.MixProject do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :fly_rpc, 7 | version: "0.3.0", 8 | elixir: "~> 1.12", 9 | start_permanent: Mix.env() == :prod, 10 | name: "Fly RPC", 11 | source_url: "https://github.com/superfly/fly_rpc_elixir", 12 | description: description(), 13 | deps: deps(), 14 | package: package(), 15 | docs: docs() 16 | ] 17 | end 18 | 19 | # Run "mix help compile.app" to learn about applications. 20 | def application do 21 | [ 22 | extra_applications: [:logger] 23 | ] 24 | end 25 | 26 | # Run "mix help deps" to learn about dependencies. 27 | defp deps do 28 | [ 29 | {:ex_doc, "~> 0.25", only: :dev}, 30 | {:dialyxir, "~> 1.0", only: [:dev], runtime: false} 31 | ] 32 | end 33 | 34 | defp description do 35 | """ 36 | Library for making RPC calls to nodes in other fly.io regions. Specifically designed to make it easier to execute code in the "primary" region. 37 | """ 38 | end 39 | 40 | defp docs do 41 | [ 42 | main: "readme", 43 | # logo: "path/to/logo.png", 44 | extras: ["README.md"] 45 | ] 46 | end 47 | 48 | defp package do 49 | [ 50 | files: ["lib", "mix.exs", "README*", "LICENSE*"], 51 | maintainers: ["Mark Ericksen"], 52 | licenses: ["Apache-2.0"], 53 | links: %{"GitHub" => "https://github.com/superfly/fly_rpc_elixir"} 54 | ] 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, 3 | "earmark_parser": {:hex, :earmark_parser, "1.4.15", "b29e8e729f4aa4a00436580dcc2c9c5c51890613457c193cc8525c388ccb2f06", [:mix], [], "hexpm", "044523d6438ea19c1b8ec877ec221b008661d3c27e3b848f4c879f500421ca5c"}, 4 | "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, 5 | "ex_doc": {:hex, :ex_doc, "0.25.3", "3edf6a0d70a39d2eafde030b8895501b1c93692effcbd21347296c18e47618ce", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "9ebebc2169ec732a38e9e779fd0418c9189b3ca93f4a676c961be6c1527913f5"}, 6 | "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, 7 | "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, 8 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 9 | "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, 10 | } 11 | -------------------------------------------------------------------------------- /test/fly_rpc_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Fly.RPCTest do 2 | use ExUnit.Case, async: false 3 | 4 | doctest Fly.RPC, import: true 5 | 6 | describe "primary_region/0" do 7 | test "when no primary set, returns local" do 8 | delete_env_if_present("PRIMARY_REGION") 9 | assert "local" == Fly.RPC.primary_region() 10 | end 11 | 12 | test "returns ENV for PRIMARY_REGION when set" do 13 | System.put_env("PRIMARY_REGION", "abc") 14 | assert "abc" == Fly.RPC.primary_region() 15 | end 16 | end 17 | 18 | describe "my_region/0" do 19 | test "when no FLY_REGION set, use MY_REGION" do 20 | delete_env_if_present("FLY_REGION") 21 | System.put_env("MY_REGION", "custom") 22 | assert "custom" == Fly.RPC.my_region() 23 | end 24 | 25 | test "when no FLY_REGION and no MY_REGION set, sets to local" do 26 | delete_env_if_present("FLY_REGION") 27 | delete_env_if_present("MY_REGION") 28 | assert "local" == Fly.RPC.my_region() 29 | end 30 | 31 | test "returns ENV for FLY_REGION when set" do 32 | System.put_env("FLY_REGION", "abc") 33 | assert "abc" == Fly.RPC.my_region() 34 | end 35 | end 36 | 37 | defp delete_env_if_present(varname) do 38 | if System.get_env(varname) do 39 | System.delete_env(varname) 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.configure(capture_log: true) 2 | ExUnit.start() 3 | --------------------------------------------------------------------------------