├── .gitignore ├── Makefile ├── README.md ├── app.config ├── rebar ├── rebar.config ├── src ├── http_proxy.app.src ├── http_proxy_app.erl ├── http_proxy_sup.erl └── toppage_handler.erl ├── start.sh └── test ├── app.config └── http_proxy_SUITE.erl /.gitignore: -------------------------------------------------------------------------------- 1 | doc 2 | log 3 | logs 4 | deps 5 | ebin 6 | test/*.beam 7 | erl_crash.dump 8 | *.swp 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: deps 2 | ./rebar compile 3 | deps: 4 | ./rebar get-deps 5 | run: build 6 | ./start.sh 7 | test: build 8 | ./rebar ct skip_deps=true 9 | test_full: build 10 | ./rebar eunit 11 | ./rebar ct 12 | clean: 13 | ./rebar clean 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # erlang-http-proxy 2 | 3 | Nontrivial HTTP proxy server in Erlang 4 | -------------------------------------------------------------------------------- /app.config: -------------------------------------------------------------------------------- 1 | [ 2 | {lager, [ 3 | {handlers, [ 4 | {lager_file_backend, [ 5 | {file, "log/error.log"}, {level, error} 6 | ]}, 7 | {lager_file_backend, [ 8 | {file, "log/console.log"}, {level, info} 9 | ]} 10 | ]} 11 | ]}, 12 | 13 | {http_proxy, [ 14 | { port, 8080 }, 15 | { workers, 256 }, 16 | { timeout, 10000 }, 17 | { sync_stream, true }, 18 | % { stream_chunk_size, 4096 }, 19 | { enable_gzip, true }, 20 | { rewrite_rules, [ 21 | % { "^http://[^/]+/mail/(.*)$", "http://mail.ru/\\1" } 22 | ]} 23 | ]} 24 | ]. 25 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/afiskon/erlang-http-proxy/081b641c914c822c4b77ba685f55a576686ca1fd/rebar -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {lib_dirs, ["deps"]}. 2 | {erl_opts, [{parse_transform, lager_transform}]}. 3 | 4 | {deps, [ 5 | {cowboy, ".*", 6 | {git, "git://github.com/extend/cowboy.git", 7 | {tag, "a07d063fd8fc8a1c"}}}, 8 | {lager, ".*", 9 | {git, "git://github.com/basho/lager.git", 10 | {tag, "2.0.0"}}}, 11 | {ibrowse, ".*", 12 | {git, "git://github.com/cmullaparthi/ibrowse.git", 13 | {tag, "v4.0.1"}}} 14 | ]}. 15 | -------------------------------------------------------------------------------- /src/http_proxy.app.src: -------------------------------------------------------------------------------- 1 | {application, http_proxy, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | cowboy 10 | ]}, 11 | {mod, { http_proxy_app, []}}, 12 | {env, []} 13 | ]}. 14 | -------------------------------------------------------------------------------- /src/http_proxy_app.erl: -------------------------------------------------------------------------------- 1 | -module(http_proxy_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | ibrowse:start(), 14 | lager:start(), 15 | Dispatch = [ 16 | {'_', [ { '_', toppage_handler, [] } ] } 17 | ], 18 | {ok, Port} = application:get_env(port), 19 | {ok, Timeout} = application:get_env(timeout), 20 | {ok, Workers} = application:get_env(workers), 21 | {ok, _} = cowboy:start_http(http, Workers, 22 | [{port, Port}], 23 | [{dispatch, Dispatch},{timeout, Timeout}] 24 | ), 25 | http_proxy_sup:start_link(). 26 | 27 | stop(_State) -> 28 | ok. 29 | -------------------------------------------------------------------------------- /src/http_proxy_sup.erl: -------------------------------------------------------------------------------- 1 | 2 | -module(http_proxy_sup). 3 | 4 | -behaviour(supervisor). 5 | 6 | %% API 7 | -export([start_link/0]). 8 | 9 | %% Supervisor callbacks 10 | -export([init/1]). 11 | 12 | %% Helper macro for declaring children of supervisor 13 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 14 | 15 | %% =================================================================== 16 | %% API functions 17 | %% =================================================================== 18 | 19 | start_link() -> 20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 21 | 22 | %% =================================================================== 23 | %% Supervisor callbacks 24 | %% =================================================================== 25 | 26 | init([]) -> 27 | {ok, { 28 | {one_for_one, 5, 10}, 29 | [] 30 | }}. 31 | 32 | -------------------------------------------------------------------------------- /src/toppage_handler.erl: -------------------------------------------------------------------------------- 1 | -module(toppage_handler). 2 | 3 | -export([init/3]). 4 | -export([handle/2]). 5 | -export([terminate/2]). 6 | 7 | -type request() :: cowboy_req:req(). 8 | -type headers() :: cowboy_http:headers(). 9 | -type processor() :: fun((binary()) -> binary()). 10 | -type finalizer() :: fun(() -> binary()). 11 | -type streamfunc() :: fun((any()) -> ok | { error, atom() }). 12 | -type ibrowse_options() :: [{atom(), any()}]. 13 | -type rewrite_rules() :: [{any(), string()}]. 14 | 15 | -record(callbacks, { 16 | processor :: processor(), 17 | finalizer :: finalizer(), 18 | stream_next :: streamfunc(), 19 | stream_close :: streamfunc() 20 | }). 21 | 22 | -record(state, { 23 | this_node :: binary(), 24 | enable_gzip :: boolean(), 25 | rewrite_rules :: rewrite_rules(), 26 | ibrowse_options :: ibrowse_options(), 27 | callbacks :: #callbacks{} 28 | }). 29 | 30 | -define(SECRET_PROXY_HEADER, <<"x-erlang-http-proxy">>). 31 | -define(HACKER_REDIRECT_PAGE, <<"http://www.fbi.gov/">>). 32 | 33 | % copy-pasted from /usr/lib/erlang/lib/erts-5.9.1/src/zlib.erl 34 | -define(MAX_WBITS, 15). 35 | 36 | init(_Transport, Req, []) -> 37 | lager:debug("~p Initializing...", [self()]), 38 | { ok, EnableGzip } = application:get_env(http_proxy, enable_gzip), 39 | {Headers, _} = cowboy_req:headers(Req), 40 | AcceptGzip = 41 | case proplists:lookup(<<"accept-encoding">>, Headers) of 42 | none -> false; 43 | {_Key, AcceptEncoding} -> 44 | lists:any( 45 | fun(X) -> X == "gzip" end, 46 | string:tokens( 47 | lists:filter( 48 | fun(X) -> X =/= 16#20 end, 49 | binary_to_list(AcceptEncoding) 50 | ), "," 51 | ) 52 | ) 53 | end, 54 | State = #state { 55 | enable_gzip = EnableGzip andalso AcceptGzip, 56 | this_node = this_node(), 57 | rewrite_rules = init_rewrite_rules(), 58 | ibrowse_options = init_ibrowse_options(), 59 | callbacks = init_default_callbacks() 60 | }, 61 | {ok, Req, State}. 62 | 63 | handle( 64 | Req, 65 | #state { 66 | ibrowse_options = IBrowseOptions, 67 | rewrite_rules = RewriteRules, 68 | this_node = ThisNode 69 | } = State) -> 70 | {Headers, _} = cowboy_req:headers(Req), 71 | {Method, _} = cowboy_req:method(Req), 72 | {ok, Body, _} = cowboy_req:body(Req), 73 | Url = case proplists:lookup(?SECRET_PROXY_HEADER, Headers) of 74 | {?SECRET_PROXY_HEADER, ThisNode} -> 75 | {Peer, _} = cowboy_req:peer(Req), 76 | lager:warning("~p Recursive request from ~p!", [self(), Peer]), 77 | ?HACKER_REDIRECT_PAGE; 78 | none -> 79 | { ReqUrl, _ } = cowboy_req:url(Req), 80 | RewriteResult = apply_rewrite_rules(ReqUrl, RewriteRules), 81 | case ReqUrl == RewriteResult of 82 | true -> ok; 83 | false -> 84 | lager:debug("~p Request URL: ~p", [self(), ReqUrl]) 85 | end, 86 | RewriteResult 87 | end, 88 | lager:debug("~p Fetching ~s", [self(), Url]), 89 | ModifiedHeaders = modify_req_headers(Headers, ThisNode), 90 | {ibrowse_req_id, _RequestId} = ibrowse:send_req( 91 | binary_to_list(Url), 92 | headers_cowboy_to_ibrowse(ModifiedHeaders), 93 | req_type_cowboy_to_ibrowse(Method), 94 | Body, 95 | IBrowseOptions, 96 | infinity 97 | ), 98 | 99 | FinalReq = receive_loop(State, Req), 100 | lager:debug("~p Done", [self()]), 101 | {ok, FinalReq, State}. 102 | 103 | terminate(_Req, _State) -> 104 | ok. 105 | 106 | %%%=================================================================== 107 | %%% Internal functions 108 | %%%=================================================================== 109 | 110 | -spec init_rewrite_rules() -> rewrite_rules(). 111 | init_rewrite_rules() -> 112 | { ok, RewriteRules } = application:get_env(http_proxy, rewrite_rules), 113 | lists:map( 114 | fun({ReString,ReplaceString}) -> 115 | {ok, CompiledRe} = re:compile(ReString), 116 | {CompiledRe, ReplaceString} 117 | end, 118 | RewriteRules 119 | ). 120 | 121 | -spec init_ibrowse_options() -> ibrowse_options(). 122 | init_ibrowse_options() -> 123 | { ok, SyncStream } = application:get_env(http_proxy, sync_stream), 124 | OptionsTemplate = [{ response_format, binary }], 125 | case SyncStream of 126 | true -> 127 | [ {stream_to, {self(), once}} | OptionsTemplate ]; 128 | false -> 129 | { ok, ChunkSize } = application:get_env( 130 | http_proxy, stream_chunk_size), 131 | OptionsTemplate ++ [ 132 | { stream_to, self() }, 133 | { stream_chunk_size, ChunkSize } 134 | ] 135 | end. 136 | 137 | -spec init_default_callbacks() -> #callbacks{}. 138 | init_default_callbacks() -> 139 | { ok, SyncStream } = application:get_env(http_proxy, sync_stream), 140 | CallbacksTemplate = #callbacks { 141 | processor = fun(X) -> X end, 142 | finalizer = fun() -> <<>> end, 143 | stream_next = fun(_ReqId) -> ok end, 144 | stream_close = fun(_ReqId) -> ok end 145 | }, 146 | case SyncStream of 147 | true -> 148 | CallbacksTemplate#callbacks { 149 | stream_next = fun(ReqId) -> ibrowse:stream_next(ReqId) end, 150 | stream_close = fun(ReqId) -> ibrowse:stream_close(ReqId) end 151 | }; 152 | false -> 153 | CallbacksTemplate 154 | end. 155 | 156 | receive_loop( 157 | #state { 158 | enable_gzip = EnableGzip, 159 | callbacks = #callbacks { 160 | processor = Processor, 161 | finalizer = Finalizer, 162 | stream_next = StreamNext, 163 | stream_close = StreamClose 164 | } = Callbacks 165 | } = State, 166 | Req) -> 167 | receive 168 | { ibrowse_async_headers, RequestId, Code, IBrowseHeaders } -> 169 | ok = StreamNext(RequestId), 170 | Headers = headers_ibrowse_to_cowboy(IBrowseHeaders), 171 | ModifiedHeaders = modify_res_headers(Headers), 172 | 173 | { NewHeaders, NewCallbacks} = 174 | case EnableGzip of 175 | true -> 176 | optional_add_gzip_compression( 177 | ModifiedHeaders, Callbacks 178 | ); 179 | false -> 180 | { ModifiedHeaders, Callbacks } 181 | end, 182 | { ok, NewReq } = send_headers(Req, Code, NewHeaders), 183 | receive_loop(State#state { callbacks = NewCallbacks }, NewReq); 184 | { ibrowse_async_response, RequestId, Data } -> 185 | ok = StreamNext(RequestId), 186 | ok = send_chunk(Req, Processor(Data)), 187 | receive_loop(State, Req); 188 | 189 | { ibrowse_async_response_end, RequestId } -> 190 | ok = StreamClose(RequestId), 191 | ok = send_chunk(Req, Finalizer()), 192 | Req 193 | end. 194 | 195 | -spec send_chunk(request(), binary()) -> ok | {error, atom()}. 196 | send_chunk(Req, Data) -> 197 | case Data of 198 | <<>> -> ok; 199 | _ -> 200 | cowboy_req:chunk(Data, Req) 201 | end. 202 | 203 | -spec apply_rewrite_rules(binary(), rewrite_rules()) -> binary(). 204 | apply_rewrite_rules(Url, []) -> 205 | Url; 206 | apply_rewrite_rules(Url, [{CompiledRe,ReplaceString}|OtherRules]) -> 207 | ApplyResult = re:replace(Url, CompiledRe, ReplaceString), 208 | case is_list(ApplyResult) of 209 | true -> iolist_to_binary(ApplyResult); 210 | false -> apply_rewrite_rules(Url, OtherRules) 211 | end. 212 | 213 | -spec optional_add_gzip_compression(headers(), #callbacks{}) -> { headers(), #callbacks{} }. 214 | optional_add_gzip_compression(Headers, Callbacks) -> 215 | case proplists:get_value(<<"content-encoding">>, Headers) of 216 | undefined -> 217 | lager:debug("~p Using gzip compression", [self()]), 218 | ZlibStream = zlib:open(), 219 | ok = zlib:deflateInit(ZlibStream, default, deflated, 16+?MAX_WBITS, 8, default), 220 | { 221 | [ {<<"content-encoding">>, <<"gzip">>} | Headers ], 222 | Callbacks#callbacks { 223 | processor = 224 | fun(<<>>) -> <<>>; 225 | (Data) -> 226 | iolist_to_binary( 227 | zlib:deflate(ZlibStream, Data, sync) 228 | ) 229 | end, 230 | finalizer = fun() -> 231 | Data = iolist_to_binary( 232 | zlib:deflate(ZlibStream, <<>>, finish) 233 | ), 234 | ok = zlib:deflateEnd(ZlibStream), 235 | ok = zlib:close(ZlibStream), 236 | Data 237 | end 238 | } 239 | }; 240 | _Other -> 241 | { Headers, Callbacks } 242 | end. 243 | 244 | -spec send_headers(request(), string(), headers()) -> { ok, request() }. 245 | send_headers(Req, Code, Headers) -> 246 | NewReq = lists:foldl( 247 | fun({HeaderName, HeaderValue}, Acc) -> 248 | cowboy_req:set_resp_header( 249 | HeaderName, 250 | HeaderValue, 251 | Acc) 252 | end, 253 | Req, 254 | Headers 255 | ), 256 | cowboy_req:chunked_reply(list_to_integer(Code), NewReq). 257 | 258 | -spec modify_req_headers(headers(), binary()) -> headers(). 259 | modify_req_headers(Headers, ThisNode) -> 260 | FilteredHeaders = lists:filter( 261 | fun({<<"proxy-connection">>, _}) -> false; 262 | ({?SECRET_PROXY_HEADER, _}) -> false; 263 | ({<<"host">>, _}) -> false; 264 | ({_, _}) -> true 265 | end, 266 | Headers 267 | ), 268 | [ {?SECRET_PROXY_HEADER, ThisNode} 269 | | FilteredHeaders]. 270 | 271 | -spec modify_res_headers(headers()) -> headers(). 272 | modify_res_headers(Headers) -> 273 | lists:filter( 274 | fun({<<"date">>, _}) -> false; 275 | ({<<"transfer-encoding">>, _}) -> false; 276 | ({<<"connection">>, _}) -> false; 277 | ({<<"server">>, _}) -> false; 278 | ({<<"content-length">>, _}) -> false; 279 | ({_, _}) -> true 280 | end, 281 | Headers 282 | ). 283 | 284 | -spec req_type_cowboy_to_ibrowse(binary()) -> get | head | post. 285 | req_type_cowboy_to_ibrowse(RequestBinary) -> 286 | case string:to_lower(binary_to_list(RequestBinary)) of 287 | "post" -> post; 288 | "head" -> head; 289 | _Other -> get 290 | end. 291 | 292 | -spec headers_ibrowse_to_cowboy([{string(),string()}]) -> headers(). 293 | headers_ibrowse_to_cowboy(Headers) -> 294 | lists:map( 295 | fun({K,V}) -> 296 | { list_to_binary(string:to_lower(K)), list_to_binary(V) } 297 | end, 298 | Headers 299 | ). 300 | 301 | -spec headers_cowboy_to_ibrowse(headers()) -> [{string(),string()}]. 302 | headers_cowboy_to_ibrowse(Headers) -> 303 | lists:map( 304 | fun({K,V}) -> 305 | { binary_to_list(K), binary_to_list(V) } 306 | end, 307 | Headers 308 | ). 309 | 310 | -spec this_node() -> binary(). 311 | this_node() -> 312 | [Node, Host] = string:tokens(atom_to_list(node()), "@"), 313 | iolist_to_binary([Node, "/", Host]). 314 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | export ERL_CRASH_DUMP_SECONDS=1 4 | 5 | erl \ 6 | -name http_proxy@`hostname` \ 7 | -pa ebin deps/*/ebin \ 8 | -config app.config \ 9 | -eval 'lists:foreach(fun(App) -> ok = application:start(App) end, [ ranch, crypto, cowboy, http_proxy ])' 10 | -------------------------------------------------------------------------------- /test/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | {lager, [ 3 | {handlers, [ 4 | {lager_console_backend, [ 5 | info 6 | ]}, 7 | {lager_file_backend, [ 8 | {"log/error.log", error, 10485760, "$D0", 5}, 9 | {"log/console.log", info, 10485760, "$D0", 5}, 10 | {"log/debug.log", debug, 10485760, "$D0", 5} 11 | ]} 12 | ]} 13 | ]}, 14 | 15 | {http_proxy, [ 16 | { port, 8080 }, 17 | { workers, 10 }, 18 | { timeout, 10000 }, 19 | { sync_stream, false }, 20 | { stream_chunk_size, 4096 }, 21 | { enable_gzip, true }, 22 | { rewrite_rules, []} 23 | ]} 24 | ]. 25 | -------------------------------------------------------------------------------- /test/http_proxy_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(http_proxy_SUITE). 2 | 3 | -compile(export_all). 4 | 5 | suite() -> 6 | [{timetrap,{seconds,300}}]. 7 | 8 | init_per_suite(Config) -> 9 | lists:foreach( 10 | fun(App) -> application:start(App) end, 11 | [ ranch, crypto, cowboy, http_proxy ] 12 | ), 13 | ibrowse:start(), 14 | Config. 15 | 16 | end_per_suite(Config) -> 17 | lists:foreach( 18 | fun(App) -> application:start(App) end, 19 | lists:reverse([ ranch, crypto, cowboy, http_proxy ]) 20 | ), 21 | ibrowse:stop(), 22 | Config. 23 | 24 | init_per_group(_GroupName, Config) -> 25 | Config. 26 | 27 | end_per_group(_GroupName, _Config) -> 28 | ok. 29 | 30 | init_per_testcase(_TestCase, Config) -> 31 | Config. 32 | 33 | end_per_testcase(_TestCase, _Config) -> 34 | ok. 35 | 36 | groups() -> 37 | []. 38 | 39 | all() -> 40 | [ 41 | basic_case 42 | ]. 43 | 44 | basic_case() -> 45 | []. 46 | 47 | basic_case(_Config) -> 48 | {ok, "200", _Headers, _Data} = ibrowse:send_req( 49 | "http://ya.ru/", [], get, [], 50 | [{proxy_host, "localhost"}, { proxy_port, 8080 }] 51 | ), 52 | ok. 53 | --------------------------------------------------------------------------------