├── .gitignore ├── Makefile ├── README.md ├── erlang.mk └── src ├── webdist.app.src ├── webdist_dist.erl ├── webdist_epmd.erl ├── webdist_epmd_compat.erl ├── webdist_epmd_router.erl ├── webdist_lib.erl └── webdist_server.erl /.gitignore: -------------------------------------------------------------------------------- 1 | /deps 2 | /ebin 3 | /.erlang.mk* 4 | /*.d 5 | .*.sw? 6 | erl_crash.dump 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = webdist 2 | ERLC_OPTS = +debug_info +warn_export_vars +warn_shadow_vars +warn_obsolete_guard +bin_opt_info +warn_export_all 3 | 4 | DEPS = cowboy 5 | dep_cowboy = git https://github.com/ninenines/cowboy.git 1.1.x 6 | 7 | SHELL_OPTS = -proto_dist webdist -sname webdist_test #-epmd_module webdist_epmd 8 | 9 | include erlang.mk 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Erlang distribution as HTTP/1.1 protocol upgrade 2 | ======= 3 | 4 | The idea 5 | ------ 6 | Erlang sometimes suffers from firewalls because it uses EPMD and dynamic port range for distribution. 7 | 8 | What if we could connect to other node using a single TCP connection? 9 | What if we could use the same TCP port also for a common task, e.g. web server? 10 | 11 | We already have ability to upgrade HTTP socket to any binary protocol (used in WebSockets). Why not to upgrade it to Erlang distribution? 12 | 13 | Implementation 14 | ------ 15 | Erlang has an option to provide custom distribution module. This project implements its own one. 16 | 17 | Distribution module makes an initial HTTP request, accepts some info and upgrade confirmation, 18 | then passes the open socket to ordinary Erlang distribution. 19 | 20 | Also we need a web server to accept a connection. Here Cowboy is used for that purpose. 21 | Cowboy handler gets host from header and node name for path, connects to that node, 22 | upgrades connection and then proxies all traffic between them. 23 | 24 | Future 25 | ------ 26 | Proxying is not very good. 27 | 28 | It would be cool to pass the open socket to target node using file descriptor passing via Unix sockets. 29 | We could borrow some libancillary bindings from [procket](https://github.com/msantos/procket) for this. 30 | 31 | Yes, it will not work in Windows and without privileges, so proxy will stay here as a fallback option. 32 | 33 | Starting the proxy 34 | -------- 35 | ```shell 36 | make 37 | ERL_LIBS=deps erl -pa ebin -s webdist_server 38 | ``` 39 | 40 | Starting self-hosted http-epmd (accepts connections to itself without proxying) 41 | ------- 42 | ```shell 43 | ERL_LIBS=deps erl -pa ebin -epmd_module webdist_epmd -webdist mode router -sname world -setcookie coocoo -s webdist_server -proto_dist inet6_tcp 44 | ``` 45 | (Here we start our own epmd in router mode, start server to accept sockets, use inet6_tcp as further dist proto) 46 | 47 | Connecting to other nodes 48 | ------ 49 | ```shell 50 | make 51 | ERL_LIBS=deps erl -pa ebin -proto_dist webdist -sname hello -setcookie coocoo 52 | ``` 53 | ```erlang 54 | (hello@stolen)1> net_adm:ping(world@stolen). 55 | {1448,727206,929391} hello@stolen:{webdist_dist,<0.41.0>,setup,world@stolen,normal,hello@stolen,shortnames,7000} 56 | {1448,727206,954859} hello@stolen:WebDist version header: 5 57 | pong 58 | ``` 59 | -------------------------------------------------------------------------------- /src/webdist.app.src: -------------------------------------------------------------------------------- 1 | {application, webdist, [ 2 | {description, ""}, 3 | {vsn, "0.1.0"}, 4 | {id, "git"}, 5 | {modules, []}, 6 | {registered, []}, 7 | {applications, [ 8 | kernel, 9 | stdlib 10 | ]}, 11 | {env, [ 12 | {port, 4380} 13 | ]} 14 | ]}. 15 | -------------------------------------------------------------------------------- /src/webdist_dist.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_dist). 2 | 3 | -define(dist_trace, true). 4 | 5 | -include_lib("kernel/include/net_address.hrl"). 6 | -include_lib("kernel/include/dist.hrl"). 7 | -include_lib("kernel/include/dist_util.hrl"). 8 | 9 | -export([listen/1, accept/1, close/1, select/1, setup/5]).% , accept_connection/5, 10 | %setup/5, is_node_name/1]). 11 | 12 | -export([perform_accept/1]). 13 | 14 | -export([accept_loop/2, do_setup/6, tick/1]). 15 | -export([setopts_pre_nodeup/1, setopts_post_nodeup/1]). 16 | 17 | listen(_Name) -> 18 | webdist_lib:load(), 19 | case inet6_tcp:listen(0, [{active, false}, {packet,2}]) of 20 | {ok, Socket} -> 21 | TcpAddress = get_tcp_address(Socket), 22 | Creation = 1, 23 | {ok, {Socket, TcpAddress, Creation}}; 24 | Error -> 25 | Error 26 | end. 27 | 28 | get_tcp_address(Socket) -> 29 | {ok, Address} = inet:sockname(Socket), 30 | {ok, Host} = inet:gethostname(), 31 | #net_address { 32 | address = Address, 33 | host = Host, 34 | protocol = tcp, 35 | family = family(Socket) 36 | }. 37 | 38 | accept(Listen) -> 39 | spawn_opt(?MODULE, accept_loop, [self(), Listen], [link, {priority, max}]). 40 | 41 | accept_loop(Kernel, Listen) -> 42 | case inet6_tcp:accept(Listen) of 43 | {ok, Socket} -> 44 | error_logger:info("Dist: Accepted ~w~n", [Socket]), 45 | %Kernel ! {accept,self(),Socket,inet6,tcp}, 46 | accept_loop(Kernel, Listen); 47 | Error -> 48 | exit(Error) 49 | end. 50 | 51 | perform_accept(Socket) when is_port(Socket) -> 52 | Kernel = whereis(net_kernel), 53 | perform_accept(Kernel, Socket). 54 | 55 | perform_accept(Kernel, Socket) when is_pid(Kernel), is_port(Socket) -> 56 | Family = family(Socket), 57 | ok = inet:setopts(Socket, [{packet, 2}, {mode, list}, {active, false}, {nodelay, true}]), 58 | Kernel ! {accept,self(),Socket,Family,tcp}, 59 | _ = controller(Kernel, Socket), 60 | ok. 61 | 62 | controller(Kernel, Socket) -> 63 | receive 64 | {Kernel, controller, Pid} -> 65 | flush_controller(Pid, Socket), 66 | inet_tcp:controlling_process(Socket, Pid), 67 | flush_controller(Pid, Socket), 68 | Pid ! {self(), controller}; 69 | {Kernel, unsupported_protocol} -> 70 | exit(unsupported_protocol) 71 | end. 72 | 73 | flush_controller(Pid, Socket) -> 74 | receive 75 | {tcp, Socket, Data} -> 76 | Pid ! {tcp, Socket, Data}, 77 | flush_controller(Pid, Socket); 78 | {tcp_closed, Socket} -> 79 | Pid ! {tcp_closed, Socket}, 80 | flush_controller(Pid, Socket) 81 | after 0 -> 82 | ok 83 | end. 84 | 85 | close(Socket) -> 86 | inet6_tcp:close(Socket). 87 | 88 | 89 | select(_) -> true. 90 | 91 | 92 | setup(Node, Type, MyNode, LongOrShortNames, SetupTime) -> 93 | spawn_link(?MODULE, do_setup, [self(), Node, Type, MyNode, LongOrShortNames, SetupTime]). 94 | 95 | 96 | do_setup(Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) -> 97 | ?trace("~p~n",[{?MODULE, self(), setup, Node, Type, MyNode, LongOrShortNames,SetupTime}]), 98 | [Name, Address] = splitnode(Node), 99 | Port = webdist_lib:conf(port), 100 | Timer = dist_util:start_timer(SetupTime), 101 | case gen_tcp:connect(Address, Port, [binary, {active, false}], SetupTime) of 102 | {ok, Socket} -> 103 | http_setup(Kernel, Socket, Node, Name, Address, Type, MyNode, Timer); 104 | __Other -> 105 | ?trace("connect to ~120p failed: ~120p~n", [Node, __Other]), 106 | ?shutdown(Node) 107 | end. 108 | 109 | 110 | http_setup(Kernel, Socket, Node, Name, Address, Type, MyNode, Timer) -> 111 | % Send request 112 | gen_tcp:send(Socket, http_req(Name, Address)), 113 | 114 | % Receive expected response line... 115 | ok = inet:setopts(Socket, [{packet, http_bin}]), 116 | {ok, {http_response, {1,1}, 101, _}} = gen_tcp:recv(Socket, 0, 5000), 117 | % ... and headers 118 | {ok, Version} = recv_headers(Socket, undefined), 119 | ok = inet:setopts(Socket, [{active, false}, {packet, 2}, {mode, list}]), 120 | HSData = make_hsdata(Kernel, Node, Type, MyNode, Socket, make_netaddr(Address, Socket), Version, Timer), 121 | dist_util:handshake_we_started(HSData). 122 | 123 | http_req(Name, Address) -> 124 | [<<"CONNECT /erl/">>, Name, <<" HTTP/1.1\r\n">>, 125 | <<"Host: ">>, Address, <<"\r\n">>, 126 | <<"Connection: Upgrade\r\n">>, 127 | <<"Upgrade: erlang-distribution\r\n">>, 128 | <<"\r\n">>]. 129 | 130 | recv_headers(Socket, Version) -> 131 | ok = inet:setopts(Socket, [{packet, httph_bin}]), 132 | {ok, Resp} = gen_tcp:recv(Socket, 0, 500), 133 | case Resp of 134 | {http_header, _, <<"Version">>, _, BinVersion} -> 135 | ?trace("WebDist version header: ~s~n", [BinVersion]), 136 | recv_headers(Socket, binary_to_integer(BinVersion)); 137 | {http_header, _, _, _, _} -> 138 | recv_headers(Socket, Version); 139 | http_eoh -> 140 | {ok, Version} 141 | end. 142 | 143 | splitnode(Node) -> 144 | case string:tokens(atom_to_list(Node), "@") of 145 | [Name, Address] -> 146 | [Name, Address]; 147 | _ -> 148 | error_logger:error_msg("** Nodename ~p illegal **~n", [Node]), 149 | ?shutdown(Node) 150 | end. 151 | 152 | 153 | make_netaddr(Address, Socket) -> 154 | {ok, {Ip, TcpPort}} = inet:peername(Socket), 155 | #net_address { 156 | address = {Ip,TcpPort}, 157 | host = Address, 158 | protocol = tcp, 159 | family = family(Ip)}. 160 | 161 | family({_,_,_,_}) -> inet; 162 | family({_,_,_,_,_,_,_,_}) -> inet6; 163 | family(Socket) when is_port(Socket) -> 164 | case erlang:port_get_data(Socket) of 165 | inet_tcp -> inet; 166 | inet6_tcp -> inet6 167 | end. 168 | 169 | 170 | make_hsdata(Kernel, Node, Type, MyNode, Socket, NetAddr, Version, Timer) -> 171 | #hs_data{ 172 | kernel_pid = Kernel, 173 | other_node = Node, 174 | this_node = MyNode, 175 | socket = Socket, 176 | timer = Timer, 177 | this_flags = 0, 178 | other_version = Version, 179 | f_send = fun inet6_tcp:send/2, 180 | f_recv = fun inet6_tcp:recv/3, 181 | f_setopts_pre_nodeup = fun ?MODULE:setopts_pre_nodeup/1, 182 | f_setopts_post_nodeup = fun ?MODULE:setopts_post_nodeup/1, 183 | f_getll = fun inet:getll/1, 184 | f_address = fun(_,_) -> NetAddr end, 185 | mf_tick = fun ?MODULE:tick/1, 186 | mf_getstat = fun inet_tcp_dist:getstat/1, 187 | request_type = Type 188 | }. 189 | 190 | 191 | tick(Socket) -> 192 | case prim_inet:send(Socket, [], [force]) of 193 | {error, closed} -> 194 | self() ! {tcp_closed, Socket}, 195 | {error, closed}; 196 | R -> 197 | R 198 | end. 199 | 200 | %% we may not always want the nodelay behaviour 201 | %% for performance reasons 202 | 203 | nodelay() -> 204 | case application:get_env(kernel, dist_nodelay) of 205 | undefined -> 206 | {nodelay, true}; 207 | {ok, true} -> 208 | {nodelay, true}; 209 | {ok, false} -> 210 | {nodelay, false}; 211 | _ -> 212 | {nodelay, true} 213 | end. 214 | 215 | setopts_pre_nodeup(S) -> 216 | inet:setopts(S, [{active, false}, {packet, 4}, nodelay()]). 217 | 218 | setopts_post_nodeup(S) -> 219 | inet:setopts(S, [{active, true}, {deliver, port}, {packet, 4}, nodelay()]). 220 | -------------------------------------------------------------------------------- /src/webdist_epmd.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_epmd). 2 | 3 | -export([mode/0, start_link/0]). 4 | 5 | -export([port_please/2, port_please/3]). 6 | 7 | 8 | mode() -> 9 | application:get_env(webdist, mode, compat). 10 | 11 | module() -> 12 | module(mode()). 13 | 14 | module(compat) -> webdist_epmd_compat; 15 | module(router) -> webdist_epmd_router. 16 | 17 | start_link() -> 18 | application:load(webdist), 19 | Mode = mode(), 20 | error_logger:info_msg("EPMD start_link mode: ~w~n", [Mode]), 21 | (module()):start_link(). 22 | 23 | 24 | 25 | port_please(Node, HostName) -> 26 | port_please(Node, HostName, infinity). 27 | 28 | port_please(Node, HostName, Timeout) -> 29 | error_logger:info_msg("EPMD port_please: ~120p @ ~120p~n", [Node, HostName]), 30 | (module()):port_please(Node, HostName, Timeout). 31 | -------------------------------------------------------------------------------- /src/webdist_epmd_compat.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_epmd_compat). 2 | 3 | -export([start_link/0]). 4 | -export([port_please/3]). 5 | 6 | -behaviour(gen_server). 7 | -export([init/1, handle_call/3, handle_info/2, handle_cast/2, terminate/2, code_change/3]). 8 | 9 | 10 | start_link() -> 11 | % We have to use 'erl_epmd' name because all calls go through that module 12 | % which relies on process with this name 13 | gen_server:start_link({local, erl_epmd}, ?MODULE, compat, []). 14 | 15 | port_please(Node, HostName, Timeout) -> 16 | erl_epmd:port_please(Node, HostName, Timeout). 17 | 18 | init(compat) -> 19 | %% Compatibility mode: proxy all calls to vanilla erl_empd module 20 | error_logger:info_msg("EPMD compat init~n", []), 21 | erl_epmd:init([]). 22 | 23 | handle_call(Call, From, State) -> 24 | error_logger:info_msg("EPMD Call ~120p~n", [Call]), 25 | erl_epmd:handle_call(Call, From, State). 26 | 27 | handle_info(Info, State) -> 28 | error_logger:info_msg("EPMD Info ~120p~n", [Info]), 29 | erl_epmd:handle_info(Info, State). 30 | 31 | handle_cast(Cast, State) -> 32 | error_logger:info_msg("EPMD Cast ~120p~n", [Cast]), 33 | erl_epmd:handle_cast(Cast, State). 34 | 35 | 36 | terminate(Reason, State) -> 37 | erl_epmd:terminate(Reason, State). 38 | 39 | code_change(OldVsn, State, Extra) -> 40 | erl_epmd:code_change(OldVsn, State, Extra). 41 | -------------------------------------------------------------------------------- /src/webdist_epmd_router.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_epmd_router). 2 | 3 | -export([start_link/0]). 4 | -export([port_please/3]). 5 | 6 | -behaviour(gen_server). 7 | -export([init/1, handle_call/3, handle_info/2, handle_cast/2, terminate/2, code_change/3]). 8 | 9 | 10 | start_link() -> 11 | % We have to use 'erl_epmd' name because all calls go through that module 12 | % which relies on process with this name 13 | gen_server:start_link({local, erl_epmd}, ?MODULE, router, []). 14 | 15 | port_please(Node, HostName, Timeout) -> 16 | gen_server:call(erl_epmd, {port_please, Node, HostName, Timeout}, Timeout). 17 | 18 | init(router) -> 19 | State = #{ 20 | my_nodehost => undefined, 21 | my_name => undefined, 22 | my_port => undefined 23 | }, 24 | {ok, {router, State}}. 25 | 26 | handle_info(_, {router, State}) -> 27 | {noreply, {router, State}}. 28 | 29 | handle_cast(_, {router, State}) -> 30 | {noreply, {router, State}}. 31 | 32 | 33 | handle_call({register, Name, PortNo}, _From, {router, #{my_name := undefined} = State}) -> 34 | NewState = State#{my_name := Name, my_port := PortNo}, 35 | {reply, {ok, 1}, {router, NewState}}; 36 | 37 | handle_call({register, _, _}, _From, {router, State}) -> 38 | {reply, {error, already_registered}, {router, State}}; 39 | 40 | %% We don't know own name. Try to get it, then continue lookup 41 | handle_call({port_please, Node, Host, Timeout}, From, {router, #{my_nodehost := undefined} = State}) -> 42 | case node() of 43 | nonode@nohost -> 44 | {reply, get_port(Node, Host, Timeout, State), {router, State}}; 45 | RealNode -> 46 | NewState = State#{my_nodehost := nodehost(RealNode)}, 47 | handle_call({port_please, Node, Host, Timeout}, From, {router, NewState}) 48 | end; 49 | 50 | %% port_please for our own node. Return 'self' shortcut 51 | handle_call({port_please, Node, Host, _}, _From, {router, #{my_nodehost := {Node, Host}} = State}) -> 52 | %% TODO: find a way to get our ?epmd_dist_high and ?epmd_dist_low (supported dist version range) 53 | {reply, {self, 5}, {router, State}}; 54 | 55 | handle_call({port_please, Node, Host, Timeout}, _From, {router, State}) -> 56 | {reply, get_port(Node, Host, Timeout, State), {router, State}}; 57 | 58 | handle_call(_, _From, {router, State}) -> 59 | {reply, {error, not_implemented}, {router, State}}. 60 | 61 | 62 | code_change(_OldVsn, {router, State}, _Extra) -> 63 | {ok, {router, State}}. 64 | 65 | terminate(_, _) -> 66 | ok. 67 | 68 | 69 | nodehost(NodeName) when is_atom(NodeName) -> 70 | [Node, Host] = string:tokens(atom_to_list(NodeName), "@"), 71 | {Node, Host}. 72 | 73 | get_port(_, _, _, _) -> 74 | noport. 75 | -------------------------------------------------------------------------------- /src/webdist_lib.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_lib). 2 | 3 | -export([load/0]). 4 | -export([conf/1]). 5 | 6 | load() -> 7 | application:load(webdist). 8 | 9 | conf(Key) -> 10 | case application:get_env(webdist, Key) of 11 | undefined -> error({no_key, Key}); 12 | {ok, Value} -> Value 13 | end. 14 | -------------------------------------------------------------------------------- /src/webdist_server.erl: -------------------------------------------------------------------------------- 1 | -module(webdist_server). 2 | 3 | -export([start/0]). 4 | 5 | % cowboy callbacks 6 | -export([init/3, handle/2, terminate/3]). 7 | -export([upgrade/4]). 8 | 9 | start() -> 10 | webdist_lib:load(), 11 | application:ensure_all_started(cowboy), 12 | cowboy:start_http(?MODULE, 5, [{ip, {0,0,0,0,0,0,0,0}}, {port, webdist_lib:conf(port)}], [{env, [{dispatch, cowboy_routes()}]}]). 13 | 14 | 15 | cowboy_routes() -> 16 | NodePath = {"/erl/:name", ?MODULE, node}, 17 | DiscPath = {"/erl", ?MODULE, discovery}, 18 | cowboy_router:compile([{'_', [NodePath, DiscPath]}]). 19 | 20 | 21 | 22 | 23 | init(_Type, _Req0, node) -> 24 | % {Host, Req1} = cowboy_req:host(Req0), 25 | % {Name, Req2} = cowboy_req:binding(name, Req1), 26 | {upgrade, protocol, ?MODULE}. 27 | 28 | handle(Req, State) -> 29 | {ok, Req2} = cowboy_req:reply(500, [], <<>>, Req), 30 | {ok, Req2, State}. 31 | 32 | terminate(_Reason, _Req, _State) -> 33 | ok. 34 | 35 | 36 | 37 | upgrade(Req0, _Env, ?MODULE, node) -> 38 | {HostBin, Req1} = cowboy_req:host(Req0), 39 | Host = binary_to_list(HostBin), 40 | {Name, Req2} = cowboy_req:binding(name, Req1), 41 | % Discover target node port 42 | case webdist_epmd:port_please(binary_to_list(Name), Host) of 43 | {port, Port, Version} -> 44 | {ok, ErlSocket} = gen_tcp:connect(Host, Port, [{packet, raw}, binary, {active, once}, {nodelay, true}]), 45 | confirm_and_loop(Req2, Version, ErlSocket); 46 | {self, Version} -> 47 | confirm_and_accept(Req2, Version); 48 | noport -> 49 | {ok, Req3} = cowboy_req:reply(404, [], <<>>, Req2), 50 | {halt, Req3}; 51 | {error, Error} -> 52 | {ok, Req3} = cowboy_req:reply(500, [], io_lib:format("Error: ~120p~n", [Error]), Req2), 53 | {halt, Req3} 54 | end. 55 | 56 | confirm_and_loop(Req2, Version, ErlSocket) -> 57 | % Confirm upgrade 58 | {ok, Req3} = cowboy_req:upgrade_reply(101, [{<<"upgrade">>, <<"erlang-distribution">>}, {<<"version">>, integer_to_binary(Version)}], Req2), 59 | % Ensure upgrade is sent and do accept expected message 60 | receive {cowboy_req, resp_sent} -> ok after 1000 -> erlang:error(no_response_ack) end, 61 | % Downgrade to low-level 62 | [Socket, Transport] = cowboy_req:get([socket, transport], Req2), 63 | ok = Transport:setopts(Socket, [{packet, raw}, binary, {active, once}, {nodelay, true}]), 64 | proxy_loop(Transport, Socket, ErlSocket), 65 | {halt, Req3}. 66 | 67 | proxy_loop(TransportA, SockA, SockB) -> 68 | receive 69 | {_, SockA, DataA} when is_binary(DataA) -> 70 | ok = gen_tcp:send(SockB, DataA), 71 | TransportA:setopts(SockA, [{active, once}]), 72 | proxy_loop(TransportA, SockA, SockB); 73 | {_, SockB, DataB} when is_binary(DataB) -> 74 | TransportA:send(SockA, DataB), 75 | inet:setopts(SockB, [{active, once}]), 76 | proxy_loop(TransportA, SockA, SockB); 77 | {tcp_closed, SockA} -> 78 | gen_tcp:close(SockB); 79 | {tcp_closed, SockB} -> 80 | TransportA:close(SockA); 81 | Other -> 82 | error({unexpected_message, Other}) 83 | end. 84 | 85 | 86 | confirm_and_accept(Req2, Version) -> 87 | % Confirm upgrade 88 | {ok, _Req3} = cowboy_req:upgrade_reply(101, [{<<"upgrade">>, <<"erlang-distribution">>}, {<<"version">>, integer_to_binary(Version)}], Req2), 89 | % Ensure upgrade is sent and do accept expected message 90 | receive {cowboy_req, resp_sent} -> ok after 1000 -> erlang:error(no_response_ack) end, 91 | % Downgrade to low-level 92 | [Socket] = cowboy_req:get([socket], Req2), 93 | webdist_dist:perform_accept(Socket), 94 | exit({shutdown, passed_to_dist}). 95 | --------------------------------------------------------------------------------