├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── c_src ├── errcat.c └── stdin_forcer.c ├── rebar.config ├── src ├── stdinout.erl ├── stdinout_pool.app.src └── stdinout_pool_server.erl └── test ├── errors.sh └── stdinout_tests.erl /.gitignore: -------------------------------------------------------------------------------- 1 | ebin/ 2 | priv/ 3 | deps/ 4 | *.o 5 | .eunit/ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | notifications: 3 | email: false 4 | otp_release: 5 | - 17.4 6 | - 17.1 7 | - 17.0 8 | - R16B03-1 9 | - R15B03 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2011-2015 Matt Stancliff 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | stdinout_pool: send to stdin and read stdout on external processes 2 | ================================================================== 3 | 4 | [![Build Status](https://secure.travis-ci.org/mattsta/erlang-stdinout-pool.png)](http://travis-ci.org/mattsta/erlang-stdinout-pool) 5 | 6 | What is it? 7 | ----------- 8 | stdinout_pool maintains a pool of idle processes waiting for input. 9 | 10 | You send input to a pool with `stdinout:send(Pool, Data)` and get back whatever 11 | the process returns on stdout or stderr (as of version 2.0+, stdout and stderr 12 | are identified independently). 13 | 14 | You can also send input to a TCP port and get back the stdout output from a 15 | process in your pool. 16 | 17 | Why is this special? 18 | -------------------- 19 | Erlang ports are unable to send EOF or close stdin on ports. To get around 20 | that limitation, stdinout_pool includes a C port to intercept stdin and 21 | forward it to a spawned process. When the C port sees a null byte 22 | (automatically appended to your input when you call `stdinout:send/2`), it 23 | closes stdin and sends stdout from the spawned process to normal stdout. 24 | 25 | Erlang ports are also unable to differentiate between stdout and stderr. 26 | To get round that limitation, starting with version 2.0.0, 27 | stdinout_pool automatically tags stdout/stderr 28 | for you and returns your output as an iolist wrapped 29 | in an appropriate tuple: `{stdout, iolist()}` or `{stderr, iolist()}`. 30 | 31 | Note: Your input must not contain null bytes or your input will terminate early. 32 | There are no other restrictions on the input. stdinout_pool was designed for 33 | sending text between processes, so if you have more complex binary protocol needs, 34 | you may need to modify the library to add a prefix length port protocol instead of 35 | automatically relying on null byte markers to terminate input. 36 | 37 | API 38 | --- 39 | **BREAKING CHANGE**: Version 2.0+ returns output wrapped in `stdout` and `stderr` 40 | tuples instead of returning a bare result. If you don't care about stdout-vs-stderr, 41 | you can use `stdinoutpool:unwrap/1` or just strip off the first element of the 42 | result tuple yourself. If the new output breaks your existing usage, you can 43 | always the older verson by pulling the [v1.3.7](https://github.com/mattsta/erlang-stdinout-pool/releases/tag/v1.3.7) tag 44 | 45 | --- 46 | 47 | All you need for `stdinout_pool` is a command to run and sit idle waiting for 48 | stdin to close or get an eof. 49 | 50 | `stdinout:start_link(Name, CommandWithArgs, [IP, Port, [NumberOfProcesses]])` 51 | IP is a string IP (e.g. "127.0.0.1") or a tuple IP (e.g. {127, 0, 0, 1}). 52 | Port is an integer. 53 | 54 | `stdinout:send(ServerNameOrPid, Data)` returns an iolist of stdout from the pool 55 | wrapped in a tuple of `{stdout, iolist()}` or `{stderr, iolist()}`. 56 | 57 | The number of default idle processes is the number of cores the erlang VM sees 58 | as determined by one of: `erlang:system_info(cpu_topology)`, `erlang:system_info(logical_processors_available)`, 59 | or `erlang:system_info(schedulers_online)`. 60 | 61 | Note: there is no upper limit on the number of spawned processes. 62 | The process count is how 63 | many already-spun-up-and-waiting processes to keep lingering. If you have 4 64 | waiting processes but suddenly get 30 requests, extra processes will be spawned 65 | to accomidate your workload (and your throughput will take a dive because you 66 | will be limited by process spawn time). When everything settles down again, you will be back to 4 idle processes. 67 | 68 | You can optionally pass in an IP and Port combination to create a network 69 | server to respond to requests. If IP and Port are each set to the atom `none`, 70 | then no network server is created. 71 | 72 | Usage 73 | ----- 74 | ### Erlang API Basic STDIN/STDOUT Usage (updated for 2.0+) 75 | 76 | ```erlang 77 | Eshell V6.0 (abort with ^G) 78 | 1> stdinout:start_link(uglify, "/home/matt/bin/cl-uglify-js"). 79 | {ok,<0.34.0>} 80 | 2> stdinout:send(uglify, "(function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})();"). 81 | {stdout,[<<"(function(){function b(){return a()}function a(){return 3}b()})();">>]} 82 | 3> stdinout:start_link(closure, "/usr/java/latest/bin/java -jar /home/matt/bin/closure/compiler.jar"). 83 | {ok,<0.38.0>} 84 | 4> stdinout:send(closure, "(function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})();"). 85 | {stdout,[<<"(function(){function a(){return 3}function b(){return a()}b()})();\n">>]} 86 | 5> stdinout:start_link(yui_js, "/usr/java/latest/bin/java -jar /home/matt/./repos/yuicompressor/build/yuicompressor-2.4.8pre.jar --type js"). 87 | {ok,<0.42.0>} 88 | 6> stdinout:send(yui_js, "(function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})();"). 89 | {stdout,[<<"(function(){function b(){return 3}function a(){return b()}a()})();">>]} 90 | ``` 91 | 92 | Note: `cl-uglify-js` returned the result in an average of 20ms. 93 | `closure.jar` returned the result in an average of 800ms. 94 | `yuicompressor.jar` returned the result in an average of 107ms. 95 | 96 | ### Erlang API STDIN->STDOUT->STDIN->...->STDOUT Pipes 97 | 98 | 99 | ### Network API Usage 100 | Start a stdinout server with an explicit IP/Port to bind: 101 | 102 | ```erlang 103 | 93> stdinout:start_link(bob_network, "/home/matt/bin/cl-uglify-js", "127.0.0.1", 6641). 104 | {ok,<0.10209.0>} 105 | ``` 106 | 107 | Now from a shell, send some data: 108 | 109 | ```bash 110 | matt@vorash:~% echo "(function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})();" | nc localhost 6641 111 | STDINOUT_POOL_ERROR: Length line too long: [(function(] (first ten bytes). 112 | ``` 113 | 114 | Uh oh, what went wrong? We have to tell the server how big our input is going 115 | to be first. The protocol for the network server is simple: the first line 116 | must contain the number of bytes of content you send. The line is terminated 117 | by a unix newline (i.e. "\n"). 118 | 119 | Trying again, we store the JS in a variable, use wc to get the length, then 120 | send the length on the first line and the content after it: 121 | 122 | ```bash 123 | matt@vorash:~% SEND_JS="(function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})();" 124 | matt@vorash:~% echo $SEND_JS |wc -c 125 | 142 126 | matt@vorash:~% echo $SEND_JS |(wc -c && echo $SEND_JS) | nc localhost 6641 127 | (function(){function b(){return a()}function a(){return 3}b()})(); 128 | ``` 129 | 130 | Success! Here is the same example with wc/cat/nc goodness, but from a file: 131 | 132 | ```bash 133 | matt@vorash:~% cat send_example.js 134 | (function() { function YoLetsReturnThree(){ return 3 ; } function LOLCallingThree() { return YoLetsReturnThree() ; }; LOLCallingThree();})(); 135 | matt@vorash:~% wc -c send_example.js | (awk '{print $1}' && cat send_example.js )| nc localhost 6641 136 | (function(){function b(){return a()}function a(){return 3}b()})(); 137 | ``` 138 | 139 | Now you have a fully functioning stdin/stdout server accessible from erlang 140 | or from the network. 141 | 142 | 143 | Building 144 | -------- 145 | Dependencies: 146 | 147 | rebar get-deps 148 | 149 | Build: 150 | 151 | rebar compile 152 | 153 | 154 | Testing 155 | ------- 156 | Automated: 157 | 158 | rebar eunit skip_deps=true 159 | 160 | Automated with timing details: 161 | 162 | rebar eunit skip_deps=true -v 163 | 164 | In the `test/` directory there is a short script to verify error conditions. 165 | You can load test error conditions with: 166 | 167 | time seq 0 300 |xargs -n 1 -P 16 ./errors.sh TARGET-IP TARGET-PORT 168 | 169 | `P` is the number of time to run the script in parallel. Increase or decrease 170 | as necessary. 171 | 172 | You can load test sending other data to stdinout over the network too, but 173 | those tests can be target-specific depending on what you are spawning, your 174 | memory usage of spawned processes, and overall workload expectations. 175 | 176 | Tests work properly under Linux, but two tests fail under OS X due to spacing 177 | differences in output. You can visually spot check those to make sure they 178 | are essentially the same. 179 | 180 | History 181 | ------- 182 | I wanted to get on-the-fly javascript minification done as fast as possible. 183 | For small to medium-size JS snippets, starting up a Lisp or Java VM was 184 | the dominating factor for throughput (too many `os:cmd/1` calls). 185 | 186 | The obvious next step was to start up some processes, let them sit idle, 187 | send stdin when needed, swallow output, then re-spawn dead processes. 188 | It seemed so simple, 189 | except erlang is unable to close stdin or send an EOF to stdin on spawned 190 | processes. Dammit. The time came to dig out pipe/dup2 man pages and 191 | make a stdin/stdout child process forwarder. Thus, a stdin/stdout 192 | proxy named stdin_forcer.c was born. 193 | 194 | Tying everything back together, there is now a way to close stdin on spawned 195 | programs. But, what good is tying a general 196 | concept like "data in, data out" to an erlang-only API? Not very good. Why 197 | not glue on a network server so anybody can send stdin and get stdout? There 198 | is no good reason it should not exist, so now the erlang stdin/stdout forwarder 199 | has a network API. Potential usage: cluster of machines to on-the-fly minify 200 | JS (use cl-uglify-js), CSS (yuicompressor.jar --type css), or transform 201 | anything else you can shove data into and get something useful back out. 202 | -------------------------------------------------------------------------------- /c_src/errcat.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | int isOpen(int fd) { 8 | errno = 0; 9 | fcntl(fd, F_GETFD); 10 | return errno != EBADF; 11 | } 12 | 13 | int main(int argc, char **argv) { 14 | char buf[BUFSIZ]; 15 | ssize_t count; 16 | 17 | do { 18 | count = read(STDIN_FILENO, buf, BUFSIZ); 19 | } while (count == -1 && errno == EINTR); 20 | 21 | if (count == -1) { 22 | perror("read"); 23 | exit(1); 24 | } 25 | 26 | do { 27 | buf[count] = 0; 28 | fputs(buf, stderr); 29 | count = read(STDIN_FILENO, buf, BUFSIZ); 30 | } while (count > 0 && isOpen(STDIN_FILENO)); 31 | 32 | exit(EXIT_SUCCESS); 33 | } 34 | -------------------------------------------------------------------------------- /c_src/stdin_forcer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #define PARENT_READ readpipe[0] 12 | #define CHILD_WRITE readpipe[1] 13 | #define CHILD_READ writepipe[0] 14 | #define PARENT_WRITE writepipe[1] 15 | #define PARENT_ERROR errorpipe[0] 16 | #define CHILD_ERROR errorpipe[1] 17 | 18 | /* Status bytes */ 19 | enum { 20 | /* ASCII & UTF-8 control character: 145 | 0x91 | PU1 | for private use. */ 21 | SUCCESS_BYTE = 0x91, 22 | /* ASCII & UTF-8 control character: 146 | 0x92 | PU2 | for private use. */ 23 | ERROR_BYTE = 0x92, 24 | }; 25 | 26 | int dup2close(int oldfd, int newfd) { 27 | int dupResult; 28 | do { 29 | dupResult = dup2(oldfd, newfd); 30 | } while ((dupResult == -1) && (errno == EINTR)); 31 | 32 | return close(oldfd); 33 | } 34 | 35 | void toSTDOUT(int fd, const char firstByte) { 36 | char buf[BUFSIZ]; 37 | ssize_t count; 38 | 39 | do { 40 | count = read(fd, buf, BUFSIZ); 41 | } while (count == -1 && errno == EINTR); 42 | 43 | if (count == -1) { 44 | perror("read"); 45 | exit(1); 46 | } else if (count > 0) { 47 | if (firstByte) 48 | write(STDOUT_FILENO, &firstByte, 1); /* Write first byte */ 49 | 50 | do { 51 | write(STDOUT_FILENO, buf, count); /* write buffer to STDOUT */ 52 | count = read(fd, buf, BUFSIZ); 53 | } while (count > 0); 54 | } 55 | } 56 | 57 | int main(int argc, char *argv[]) { 58 | int readpipe[2], writepipe[2], errorpipe[2]; 59 | int unused __attribute__((unused)); 60 | pid_t cpid; 61 | 62 | assert(1 < argc && argc < 64); 63 | 64 | if (pipe(readpipe) == -1 || pipe(writepipe) == -1 || 65 | pipe(errorpipe) == -1) { 66 | perror("pipe"); 67 | exit(EXIT_FAILURE); 68 | } 69 | 70 | cpid = fork(); 71 | if (cpid == -1) { 72 | perror("fork"); 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | if (cpid == 0) { 77 | /* Forked Child with STDIN forwarding */ 78 | char *cmd = argv[1]; 79 | char *exec_args[64] = {0}; 80 | int i; 81 | 82 | for (i = 0; i < argc - 1; i++) { 83 | /* args to stdin_forcer are the program and optional args to exec. 84 | Here we copy pointers pointing to strings of cmd/args. 85 | exec_args is indexed one lower than argv. */ 86 | exec_args[i] = argv[i + 1]; 87 | } 88 | 89 | close(PARENT_READ); /* We aren't the parent. Decrement fd refcounts. */ 90 | close(PARENT_WRITE); 91 | close(PARENT_ERROR); 92 | 93 | /* CHILD_READ = STDIN to the exec'd process. 94 | CHILD_WRITE = STDOUT to the exec'd process. 95 | CHILD_ERROR = STDERR to the exec'd process. */ 96 | if (dup2close(CHILD_READ, STDIN_FILENO) || 97 | dup2close(CHILD_WRITE, STDOUT_FILENO) || 98 | dup2close(CHILD_ERROR, STDERR_FILENO)) { 99 | perror("dup2 or close"); 100 | _exit(EXIT_FAILURE); 101 | } 102 | 103 | /* At this point, the execv'd program's STDIN and STDOUT are the pipe */ 104 | if (execv(cmd, exec_args) == -1) { 105 | perror("execve"); 106 | } 107 | _exit(EXIT_FAILURE); /* Silence a warning */ 108 | } else { 109 | /* Original Parent Process */ 110 | char buf; 111 | 112 | close(CHILD_READ); /* We aren't the child. Close its read/write. */ 113 | close(CHILD_WRITE); 114 | close(CHILD_ERROR); 115 | 116 | /* Read until 0 byte */ 117 | /* TODO: Create a better length-prefixed protocol so we don't 118 | * rely on a single end-of-stream byte markers. */ 119 | while (read(STDIN_FILENO, &buf, 1) > 0 && buf != 0x0) { 120 | unused = write(PARENT_WRITE, &buf, 1); 121 | } 122 | close(PARENT_WRITE); /* closing PARENT_WRITE sends EOF to CHILD_READ */ 123 | 124 | toSTDOUT(PARENT_READ, (uint8_t)SUCCESS_BYTE); 125 | toSTDOUT(PARENT_ERROR, (uint8_t)ERROR_BYTE); 126 | 127 | close(PARENT_READ); /* done reading from writepipe */ 128 | close(PARENT_ERROR); /* done reading from errorpipe */ 129 | close(STDOUT_FILENO); /* done writing to stdout */ 130 | 131 | wait(NULL); /* Wait for child to exit */ 132 | 133 | exit(EXIT_SUCCESS); /* This was a triumph */ 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps, [ 2 | {oneshot, "1.7.0", 3 | {git, "https://github.com/mattsta/oneshot.git", {tag, "v1.7.0"}}} 4 | ]}. 5 | 6 | {port_specs, [{"priv/stdin_forcer", ["c_src/stdin_forcer.c"]}, 7 | {"priv/errcat", ["c_src/errcat.c"]}]}. -------------------------------------------------------------------------------- /src/stdinout.erl: -------------------------------------------------------------------------------- 1 | -module(stdinout). 2 | 3 | -export([start_link/2, start_link/4, start_link/5]). 4 | 5 | -export([send/2, send/3]). 6 | -export([send_raw/2]). 7 | -export([reload/1]). 8 | -export([pipe/2]). 9 | -export([shutdown/1]). 10 | 11 | -export([unwrap/1]). 12 | 13 | -define(TIMEOUT, 60000). 14 | 15 | %%==================================================================== 16 | %% Starting 17 | %%==================================================================== 18 | start_link(GenServerName, Cmd) -> 19 | stdinout_pool_server:start_link(GenServerName, Cmd). 20 | 21 | start_link(GenServerName, Cmd, IP, Port) -> 22 | stdinout_pool_server:start_link(GenServerName, Cmd, IP, Port). 23 | 24 | start_link(GenServerName, Cmd, IP, Port, SocketCount) -> 25 | stdinout_pool_server:start_link(GenServerName, Cmd, IP, Port, SocketCount). 26 | 27 | %%==================================================================== 28 | %% respawn all running processes 29 | %%==================================================================== 30 | reload(Server) -> 31 | gen_server:call(Server, reload, ?TIMEOUT). 32 | 33 | %%==================================================================== 34 | %% stdin->stdout through pool without consuming error byte 35 | %%==================================================================== 36 | send_raw(Server, Content) -> 37 | gen_server:call(Server, {stdin, Content}, ?TIMEOUT). 38 | 39 | %%==================================================================== 40 | %% stdin->stdout through pool or network 41 | %%==================================================================== 42 | send({Host, Port}, Content) -> 43 | check_err(send(Host, Port, Content)); 44 | send(Server, Content) -> 45 | check_err(send_raw(Server, Content)). 46 | 47 | unwrap({_Type, Result}) -> Result. 48 | 49 | % Extract SUCCESS_BYTE marking stdout result 50 | check_err([<<145>> | Rest]) -> {stdout, Rest}; 51 | check_err([<<145, Tail/binary>> | Rest]) -> {stdout, [<> | Rest]}; 52 | % Extract ERROR_BYTE marking stderr result 53 | check_err([<<146>> | Rest]) -> {stderr, Rest}; 54 | check_err([<<146, Tail/binary>> | Rest]) -> {stderr, [<> | Rest]}; 55 | check_err([]) -> {stdout, []}; % empty response 56 | check_err(Data) -> {error, invalid_status_byte, Data}. 57 | 58 | %%==================================================================== 59 | %% stdin->stdout through network 60 | %%==================================================================== 61 | send(Host, Port, Content) -> 62 | case integer_to_list(iolist_size(Content)) of 63 | "0" -> []; % if we aren't sending anything, don't send anything 64 | Length -> Formatted = [Length, "\n", Content], 65 | {ok, Sock} = 66 | gen_tcp:connect(Host, Port, [binary, {active, false}]), 67 | gen_tcp:send(Sock, Formatted), 68 | recv_loop(Sock, []) 69 | end. 70 | 71 | recv_loop(Sock, Accum) -> 72 | case gen_tcp:recv(Sock, 0) of 73 | {ok, Bin} -> recv_loop(Sock, [Bin | Accum]); 74 | {error, closed} -> lists:reverse(Accum) 75 | end. 76 | 77 | %%==================================================================== 78 | %% stdin->stdout through a series of pipes using pool or network 79 | %%==================================================================== 80 | pipe(Content, []) -> 81 | {stdout, Content}; 82 | % If ErrorRegex is an integer, we have a {Host, Port} tuple, not a regex. 83 | pipe(Content, [{Server, ErrorRegex} | T]) when not is_integer(ErrorRegex) -> 84 | case send(Server, Content) of 85 | {stdout, Stdout} -> 86 | case re:run(Stdout, ErrorRegex) of 87 | nomatch -> pipe(Stdout, T); 88 | {match, _} -> {error, Server, Stdout} 89 | end; 90 | {stderr, Errout} -> {stderr, Server, Errout} 91 | end; 92 | pipe(Content, [Server | T]) -> 93 | case send(Server, Content) of 94 | {stdout, Stdout} -> pipe(Stdout, T); 95 | {stderr, Errout} -> {stderr, Server, Errout} 96 | end. 97 | 98 | %%=================================================================== 99 | %% Stopping 100 | %%==================================================================== 101 | shutdown(Server) when is_pid(Server) -> 102 | exit(Server, shutdown); 103 | shutdown(Server) when is_atom(Server) -> 104 | shutdown(whereis(Server)). 105 | -------------------------------------------------------------------------------- /src/stdinout_pool.app.src: -------------------------------------------------------------------------------- 1 | {application, stdinout_pool, 2 | [ 3 | {description, "manage a pool of processes used for stdin/stdout"}, 4 | {vsn, "2.0.0"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {env, []} 11 | ]}. 12 | -------------------------------------------------------------------------------- /src/stdinout_pool_server.erl: -------------------------------------------------------------------------------- 1 | -module(stdinout_pool_server). 2 | -behaviour(gen_server). 3 | 4 | %% gen_server callbacks 5 | -export([init/1, 6 | handle_call/3, handle_cast/2, 7 | handle_info/2, terminate/2, code_change/3]). 8 | 9 | %% api callbacks 10 | -export([start_link/1, start_link/2, start_link/4, start_link/5]). 11 | 12 | % oneshot callback 13 | -export([handle_oneshot/1]). 14 | 15 | -record(state, {cmd, ip, port, available, reserved, count, forcer}). 16 | 17 | %%==================================================================== 18 | %% api callbacks 19 | %%==================================================================== 20 | start_link(Cmd) -> 21 | start_link(?MODULE, Cmd). 22 | 23 | start_link(GenServerName, Cmd) -> 24 | start_link(GenServerName, Cmd, none, none). 25 | 26 | start_link(GenServerName, Cmd, IP, Port) -> 27 | start_link(GenServerName, Cmd, IP, Port, count_cpus()). 28 | 29 | start_link(GenServerName, Cmd, IP, Port, SocketCount) -> 30 | gen_server:start_link({local, GenServerName}, ?MODULE, 31 | [Cmd, IP, Port, SocketCount], []). 32 | 33 | count_cpus() -> 34 | count_cpus(erlang:system_info(cpu_topology), 0). 35 | count_cpus(undefined, 0) -> 36 | case erlang:system_info(logical_processors_available) of 37 | unknown -> erlang:system_info(schedulers_online); 38 | Count -> Count 39 | end; 40 | count_cpus(undefined, Count) -> 41 | Count; 42 | count_cpus([], Count) -> 43 | Count; 44 | count_cpus([{node, [{processor, Cores}]} | T], Count) when is_list (Cores) -> 45 | count_cpus(T, Count + length(Cores)); 46 | count_cpus([{node, [{processor, _}]} | T], Count) -> 47 | count_cpus(T, Count + 1); 48 | count_cpus([{processor, Cores} | T], Count) when is_list (Cores) -> 49 | count_cpus(T, Count + length(Cores)); 50 | count_cpus([{processor, _} | T], Count) -> 51 | count_cpus(T, Count + 1). 52 | 53 | %%==================================================================== 54 | %% gen_server callbacks 55 | %%==================================================================== 56 | 57 | %%-------------------------------------------------------------------- 58 | %% Function: init(Args) -> {ok, State} | 59 | %% {ok, State, Timeout} | 60 | %% ignore | 61 | %% {stop, Reason} 62 | %% Description: Initiates the server 63 | %%-------------------------------------------------------------------- 64 | init([Cmd, IP, Port, SocketCount]) -> 65 | process_flag(trap_exit, true), 66 | Forcer = get_base_dir(?MODULE) ++ "/priv/stdin_forcer", 67 | initial_setup(#state{cmd = Cmd, forcer = Forcer, 68 | ip = IP, port = Port, count = SocketCount}). 69 | 70 | %%-------------------------------------------------------------------- 71 | %% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | 72 | %% {reply, Reply, State, Timeout} | 73 | %% {noreply, State} | 74 | %% {noreply, State, Timeout} | 75 | %% {stop, Reason, Reply, State} | 76 | %% {stop, Reason, State} 77 | %% Description: Handling call messages 78 | %%-------------------------------------------------------------------- 79 | strip_ok({ok, Data}) -> Data; 80 | strip_ok(Data) -> Data. 81 | 82 | % TODO: Use poolboy instead of this hack of a pool. 83 | % If we run out of available processes, just make another one. 84 | % There should be a limit here. Maybe track used ones and make max 2xCount ports 85 | % Or, we could make a waiting_clients list and service them on respawn 86 | handle_call({stdin, Content}, From, #state{available = []} = State) -> 87 | NewAvail = [setup(State)], 88 | handle_call({stdin, Content}, From, State#state{available = NewAvail}); 89 | 90 | handle_call({stdin, Content}, From, #state{available = [H|T]} = State) -> 91 | % quickly spawn so we can be a non-blocking gen_server: 92 | spawn(fun() -> 93 | port_connect(H, self()), % attach port to this spawned process 94 | port_command(H, strip_ok(Content)), % send our stdin content to the wrapper 95 | port_command(H, <<0>>), % tell the wrapper we're done 96 | gen_server:reply(From, gather_response(H)), 97 | port_close(H) 98 | end), 99 | {noreply, State#state{available = T}}; 100 | 101 | handle_call(reload, _From, #state{available = Running} = State) -> 102 | [port_close(R) || R <- Running], 103 | {reply, ok, State#state{available = []}}. 104 | 105 | gather_response(Port) -> 106 | gather_response(Port, []). 107 | gather_response(Port, Accum) -> 108 | receive 109 | {Port, {data, Bin}} -> gather_response(Port, [Bin | Accum]); 110 | {Port, eof} -> lists:reverse(Accum) 111 | after % max 30 seconds of time for the process to send EOF (close stdout) 112 | 30000 -> {died, lists:reverse(Accum)} 113 | end. 114 | 115 | %%-------------------------------------------------------------------- 116 | %% Function: handle_cast(Msg, State) -> {noreply, State} | 117 | %% {noreply, State, Timeout} | 118 | %% {stop, Reason, State} 119 | %% Description: Handling cast messages 120 | %%-------------------------------------------------------------------- 121 | handle_cast(_Msg, State) -> 122 | {noreply, State}. 123 | 124 | %%-------------------------------------------------------------------- 125 | %% Function: handle_info(Info, State) -> {noreply, State} | 126 | %% {noreply, State, Timeout} | 127 | %% {stop, Reason, State} 128 | %% Description: Handling all non call/cast messages 129 | %%-------------------------------------------------------------------- 130 | 131 | % A cmd exited. Let's remove it from the proper list of servers. 132 | handle_info({'EXIT', _Pid, _Reason}, 133 | #state{available=Available, count = Count} = State) 134 | when length(Available) >= Count -> 135 | {noreply, State}; % don't respawn if we have more than Count live processes 136 | handle_info({'EXIT', Pid, _Reason}, 137 | #state{available=Available, reserved=Reserved} = State) -> 138 | case lists:member(Pid, Available) of 139 | true -> RemovedOld = Available -- [Pid], 140 | NewAvail = [setup(State) | RemovedOld], 141 | {noreply, State#state{available=NewAvail}}; 142 | false -> RemovedOld = Reserved -- [Pid], 143 | NewAvail = [setup(State) | Available], 144 | {noreply, State#state{available = NewAvail, reserved = RemovedOld}} 145 | end; 146 | 147 | handle_info(Info, State) -> 148 | error_logger:error_msg("Other info: ~p with state ~p~n", [Info, State]), 149 | {noreply, State}. 150 | 151 | %%-------------------------------------------------------------------- 152 | %% Function: terminate(Reason, State) -> void() 153 | %% Description: This function is called by a gen_server when it is about to 154 | %% terminate. It should be the opposite of Module:init/1 and do any necessary 155 | %% cleaning up. When it returns, the gen_server terminates with Reason. 156 | %% The return value is ignored. 157 | %%-------------------------------------------------------------------- 158 | terminate(_Reason, #state{available = A, reserved = R}) -> 159 | [port_close(P) || P <- A], 160 | [port_close(P) || P <- R], 161 | ok. 162 | 163 | %%-------------------------------------------------------------------- 164 | %% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} 165 | %% Description: Convert process state when code is changed 166 | %%-------------------------------------------------------------------- 167 | code_change(_OldVsn, State, _Extra) -> 168 | {ok, State}. 169 | 170 | %%-------------------------------------------------------------------- 171 | %%% Internal functions 172 | %%-------------------------------------------------------------------- 173 | initial_setup(#state{count = Count} = State) -> 174 | OpenedPorts = [setup(State) || _ <- lists:seq(1, Count)], 175 | setup_external_server(State), 176 | {ok, State#state{ 177 | available = OpenedPorts, 178 | reserved = []}}. 179 | 180 | setup(#state{cmd = Cmd, forcer = Forcer}) -> 181 | % io:format("Opening ~s~n", [Cmd]), % Uncomment to see re-spawns happen live 182 | open_port({spawn_executable, Forcer}, 183 | [stream, use_stdio, binary, eof, 184 | {args, string:tokens(Cmd, " ")}]). 185 | 186 | get_base_dir(Module) -> 187 | {file, Here} = code:is_loaded(Module), 188 | filename:dirname(filename:dirname(Here)). 189 | 190 | %%-------------------------------------------------------------------- 191 | %%% oneshot TCP server callbacks/functions/executors 192 | %%% The exit calls below exit the connected TCP process, not pool_server 193 | %%-------------------------------------------------------------------- 194 | 195 | % Break if the first line received (the Length header) is > than 9 characters 196 | oneshot_length(Sock, Acc) when length(Acc) > 9 -> 197 | gen_tcp:send(Sock, oneshot_error({length, lists:reverse(Acc)})), 198 | exit(normal); 199 | oneshot_length(Sock, Acc) -> 200 | % Receive one byte at a time so we can stop at \n 201 | % (also protection against naughty clients who don't send length headers) 202 | case gen_tcp:recv(Sock, 1) of 203 | {ok, <<"\n">>} -> RAcc = lists:reverse(Acc), 204 | try 205 | list_to_integer(binary_to_list(iolist_to_binary(RAcc))) 206 | catch 207 | error:badarg -> 208 | gen_tcp:send(Sock, oneshot_error({format, RAcc})), 209 | exit(normal) 210 | end; 211 | {ok, Bin} -> oneshot_length(Sock, [Bin | Acc]); 212 | {error, _} -> exit(self(), normal) 213 | end. 214 | 215 | handle_oneshot(Pool) -> 216 | handle_oneshot(Pool, -1, []). 217 | handle_oneshot(Pool, TotalSz, Acc) -> 218 | receive 219 | {socket_ready, Socket} -> % oneshot_length reads byte-by-byte. no active. 220 | inet:setopts(Socket, [{active, false}]), 221 | TotalSize = oneshot_length(Socket, []), 222 | % re-enable active so we receive messages here 223 | inet:setopts(Socket, [{active, once}]), 224 | handle_oneshot(Pool, TotalSize, Acc); 225 | {tcp, Socket, Bin} -> NewAcc = [Bin | Acc], 226 | NewSz = iolist_size(NewAcc), 227 | if 228 | NewSz >= TotalSz -> 229 | RAcc = lists:reverse(NewAcc), 230 | % If we read more than TotalSz, only send up to 231 | % TotalSz to stdinout:send/2. 232 | UseAcc = case NewSz of 233 | TotalSz -> RAcc; 234 | _ -> UR = iolist_to_binary(RAcc), 235 | <> = UR, 237 | Usable 238 | end, 239 | Stdout = case stdinout:send_raw(Pool, UseAcc) of 240 | {died, StdAccum} -> 241 | oneshot_error({died, StdAccum}); 242 | Value -> Value 243 | end, 244 | ok = gen_tcp:send(Socket, Stdout); 245 | true -> 246 | inet:setopts(Socket, [{active, once}]), 247 | handle_oneshot(Pool, TotalSz, NewAcc) 248 | end 249 | after % max 30 seconds of socket wait time 250 | 30000 -> died 251 | end. 252 | 253 | oneshot_error(Error) -> 254 | [<<"STDINOUT_POOL_ERROR: ">>, 255 | oneshot_error_pretty(Error)]. 256 | 257 | oneshot_error_pretty({died, Accum}) -> 258 | [<<"Timeout reached when running command or no EOF received. ">>, 259 | <<"Data so far: ">>, Accum]; 260 | oneshot_error_pretty({format, Accum}) -> 261 | [<<"Non-number found in Length line: [">>, Accum, <<"].\n">>]; 262 | oneshot_error_pretty({length, Accum}) -> 263 | [<<"Length line too long: [">>, Accum, <<"] (first ten bytes).\n">>]. 264 | 265 | 266 | setup_external_server(#state{ip = none, port = none}) -> 267 | ok; 268 | setup_external_server(#state{ip = IP, port = Port}) -> 269 | ThisGenServer = self(), 270 | {ok, _Pid} = 271 | oneshot_server:start_link(IP, Port, fun() -> 272 | ?MODULE:handle_oneshot(ThisGenServer) 273 | end). 274 | -------------------------------------------------------------------------------- /test/errors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | HOST=$1 4 | PORT=$2 5 | echo cat | nc $HOST $PORT 6 | echo cateeeeeeeeeeeeeeeeeeeeeeeeeeeeee | nc $HOST $PORT 7 | 8 | -------------------------------------------------------------------------------- /test/stdinout_tests.erl: -------------------------------------------------------------------------------- 1 | -module(stdinout_tests). 2 | -include_lib("eunit/include/eunit.hrl"). 3 | 4 | -define(E(A, B), ?assertEqual(A, B)). 5 | -define(_E(A, B), ?_assertEqual(A, B)). 6 | -define(B_OUT(X), validate(stdout, X)). 7 | -define(B_ERR(X), validate(stderr, X)). 8 | validate(Out, {Out, Data}) -> iolist_to_binary(Data). 9 | 10 | get_base_dir(Module) -> 11 | {file, Here} = code:is_loaded(Module), 12 | filename:dirname(filename:dirname(Here)). 13 | 14 | add_oneshot_dep() -> 15 | code:add_path(get_base_dir(?MODULE) ++ "deps/oneshot/ebin"). 16 | 17 | setup_ports() -> 18 | add_oneshot_dep(), 19 | 20 | Echoerr = get_base_dir(?MODULE) ++ "/priv/errcat", 21 | 22 | % Obviously this test only works if you have /bin/cat and /usr/bin/wc 23 | CAT1 = stdinout:start_link(cat1, "/bin/cat"), 24 | CAT2 = stdinout:start_link(cat2, "/bin/cat"), 25 | CAT3 = stdinout:start_link(cat3, "/bin/cat"), 26 | CAT4 = stdinout:start_link(cat4, "/bin/cat"), 27 | WC1 = stdinout:start_link(wc, "/usr/bin/wc"), 28 | 29 | CAT6 = stdinout:start_link(catn1, "/bin/cat", "127.0.0.1", 6651), 30 | CAT7 = stdinout:start_link(catn2, "/bin/cat", "127.0.0.1", 6652), 31 | WC2 = stdinout:start_link(wcn, "/usr/bin/wc", "127.0.0.1", 6653), 32 | 33 | ERR1 = stdinout:start_link(errcat, Echoerr), 34 | ERR2 = stdinout:start_link(errcat_net, Echoerr, "127.0.0.1", 6654), 35 | 36 | [unlink(P) || {ok, P} <- [CAT1, CAT2, CAT3, CAT4, WC1, CAT6, CAT7, WC2, ERR1, ERR2]], 37 | [P || {ok, P} <- [CAT1, CAT2, CAT3, CAT4, WC1, CAT6, CAT7, WC2, ERR1, ERR2]]. 38 | 39 | cleanup_ports(Ps) -> 40 | [stdinout:shutdown(P) || P <- Ps]. 41 | 42 | % Test responses on STDOUT 43 | everything_erlang_API_in_parallel_test_() -> 44 | {setup, 45 | fun setup_ports/0, 46 | fun cleanup_ports/1, 47 | fun(_) -> 48 | {inparallel, 49 | [ 50 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(cat1, "hello"))), 51 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(cat1, <<"hello">>))), 52 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(cat1, [<<"hello">>]))), 53 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(cat1, [<<"he">>, <<"llo">>]))), 54 | ?_E(ok, stdinout:reload(cat1)), 55 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(cat1, ["he", "llo"]))), 56 | ?_E(<<>>, ?B_OUT(stdinout:send(cat1, ""))), 57 | ?_E(ok, stdinout:reload(cat2)), 58 | ?_E(<<"hello">>, ?B_OUT(stdinout:pipe("hello",[cat1, cat2, cat3, cat4]))), 59 | ?_E({error, cat1, <<"hello">>}, 60 | % this wacky fun exists to erlang:iolist_to_binary/1 the ErrorIoList 61 | fun() -> 62 | {error, cat1, ErrorIoList} = 63 | stdinout:pipe("hello", [{cat1, "he"}, cat2, cat3, cat4]), 64 | {error, cat1, iolist_to_binary(ErrorIoList)} 65 | end()), 66 | ?_E(ok, stdinout:reload(cat1)), 67 | ?_E({error, wc, <<" 0 1 5\n">>}, 68 | fun() -> 69 | {error, wc, ErrorIoList} = 70 | stdinout:pipe("hello", [{cat1, "bob"}, {wc, "5"}, cat3, cat4]), 71 | {error, wc, iolist_to_binary(ErrorIoList)} 72 | end()), 73 | 74 | % Assert that binary responses on stdout can start with status byte 145 or 146 75 | ?_E(<<145,23,88,97>>, ?B_OUT(stdinout:send(cat1, <<145,23,88,97>>))), 76 | ?_E(<<146,23,88,97>>, ?B_OUT(stdinout:send(cat1, <<146,23,88,97>>))), 77 | ?_E(<<146,23,88,97>>, ?B_OUT(stdinout:pipe(<<146,23,88,97>>,[cat1, cat2, cat3, cat4]))) 78 | ] 79 | } 80 | end 81 | }. 82 | 83 | % Test responses on STDERR 84 | everything_erlang_API_in_parallel_error_test_() -> 85 | {setup, 86 | fun setup_ports/0, 87 | fun cleanup_ports/1, 88 | fun(_) -> 89 | {inparallel, 90 | [ 91 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(errcat, "hello"))), 92 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(errcat, <<"hello">>))), 93 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(errcat, [<<"hello">>]))), 94 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(errcat, [<<"he">>, <<"llo">>]))), 95 | ?_E(ok, stdinout:reload(errcat)), 96 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(errcat, ["he", "llo"]))), 97 | ?_E({stderr, errcat, [<<"hello">>]}, stdinout:pipe("hello",[errcat, cat2, cat3, cat4])), 98 | ?_E({stderr, errcat, [<<"hello">>]}, stdinout:pipe("hello",[cat1, cat2, errcat, cat4])), 99 | 100 | % Assert that binary responses on stderr can start with status byte 145 or 146 101 | ?_E(<<145,23,88,97>>, ?B_ERR(stdinout:send(errcat, <<145,23,88,97>>))), 102 | ?_E(<<146,23,88,97>>, ?B_ERR(stdinout:send(errcat, <<146,23,88,97>>))), 103 | ?_E({stderr, errcat, [<<146,23,88,97>>]}, stdinout:pipe(<<146,23,88,97>>,[errcat, cat2, cat3, cat4])), 104 | ?_E({stderr, errcat, [<<146,23,88,97>>]}, stdinout:pipe(<<146,23,88,97>>,[cat1, cat2, errcat, cat4])) 105 | ] 106 | } 107 | end 108 | }. 109 | 110 | -define(C1, {"localhost", 6651}). 111 | -define(C2, {"localhost", 6652}). 112 | -define(W1, {"localhost", 6653}). 113 | network_API_test_() -> 114 | {setup, 115 | fun setup_ports/0, 116 | fun cleanup_ports/1, 117 | fun(_) -> 118 | {inparallel, 119 | [ 120 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(?C1, "hello"))), 121 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(?C2, <<"hello">>))), 122 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(?C1, [<<"hello">>]))), 123 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(?C1, [<<"he">>, <<"llo">>]))), 124 | ?_E(<<"hello">>, ?B_OUT(stdinout:send(?C2, ["he", "llo"]))), 125 | ?_E(<<>>, ?B_OUT(stdinout:send(?C1, ""))), 126 | ?_E(<<"hello">>, ?B_OUT(stdinout:pipe("hello", [?C1, ?C2, ?C1, ?C2]))), 127 | ?_E({error, ?C1, <<"hello">>}, 128 | % this wacky fun exists to erlang:iolist_to_binary/1 the ErrorIoList 129 | fun() -> 130 | {error, ?C1, ErrorIoList} = 131 | stdinout:pipe("hello", [{?C1, "he"}, ?C2, ?C2, ?C1]), 132 | {error, ?C1, iolist_to_binary(ErrorIoList)} 133 | end()), 134 | ?_E({error, ?W1, <<" 0 1 5\n">>}, 135 | fun() -> 136 | {error, ?W1, ErrorIoList} = 137 | stdinout:pipe("hello", [{?C2, "bob"}, {?W1, "5"}, ?C2, ?C1]), 138 | {error, ?W1, iolist_to_binary(ErrorIoList)} 139 | end()), 140 | 141 | % Assert that network binary responses on stdout can start with status byte 145 or 146 142 | ?_E(<<145,23,88,97>>, ?B_OUT(stdinout:send(?C1, <<145,23,88,97>>))), 143 | ?_E(<<146,23,88,97>>, ?B_OUT(stdinout:send(?C2, <<146,23,88,97>>))), 144 | ?_E(<<146,23,88,97>>, ?B_OUT(stdinout:pipe(<<146,23,88,97>>, [?C1, ?C2, ?C1, ?C2]))) 145 | ] 146 | } 147 | end 148 | }. 149 | 150 | -define(E2, {"localhost", 6654}). 151 | network_API_error_test_() -> 152 | {setup, 153 | fun setup_ports/0, 154 | fun cleanup_ports/1, 155 | fun(_) -> 156 | {inparallel, 157 | [ 158 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(?E2, "hello"))), 159 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(?E2, <<"hello">>))), 160 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(?E2, [<<"hello">>]))), 161 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(?E2, [<<"he">>, <<"llo">>]))), 162 | ?_E(<<"hello">>, ?B_ERR(stdinout:send(?E2, ["he", "llo"]))), 163 | 164 | ?_E({stderr,{"localhost",6654},[<<"hello">>]}, 165 | stdinout:pipe("hello", [?C1, ?C2, ?E2, ?C1])), 166 | 167 | % Assert that network binary responses on stderr can start with status byte 145 or 146 168 | ?_E({stderr, [<<145,23,88,97>>]}, stdinout:send(errcat, <<145,23,88,97>>)), 169 | ?_E({stderr, [<<146,23,88,97>>]}, stdinout:send(errcat, <<146,23,88,97>>)), 170 | ?_E({stderr, {"localhost",6654}, [<<146,23,88,97>>]}, stdinout:pipe(<<146,23,88,97>>,[?C1, ?C2, ?E2, ?C1])) 171 | ] 172 | } 173 | end 174 | }. 175 | --------------------------------------------------------------------------------