├── README
├── .gitignore
├── Makefile
├── AUTHORS
├── c.iml
├── src
├── c
│ ├── Makefile
│ ├── jlinuxfork.h
│ ├── binrunner.c
│ └── jlinuxfork.c
└── java
│ └── net
│ └── axiak
│ └── runtime
│ ├── SpawnRuntime.java
│ └── SpawnedProcess.java
├── test
├── Makefile
└── TestClass.java
├── java.iml
├── LICENSE
├── README.rst
└── java_posix_spawn.ipr
/README:
--------------------------------------------------------------------------------
1 | ./README.rst
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | docs/
2 | *.iws
3 | *.class
4 | *.so
5 | binrunner
6 | .idea
7 | target/*
8 | build/*
9 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | all:
2 | make -C src/c all
3 |
4 | install:
5 | make -C src/c install
6 |
7 | clean:
8 | make -C src/c clean
9 |
--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | The main author is Michael Axiak, you can contact him at mike@axiak.net.
2 |
3 | This project would not exist without the support of CrunchTime Information Systems (http://www.crunchtime.com/)
4 |
5 |
--------------------------------------------------------------------------------
/c.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/src/c/Makefile:
--------------------------------------------------------------------------------
1 | includes = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux
2 | cc = cc
3 |
4 | all: binrunner library
5 |
6 | library: jlinuxfork.c jlinuxfork.h
7 | $(cc) -fPIC -Wno-long-long -pedantic -O3 -Wall -g -o libjlinuxfork.so -shared -Wl,-soname,libspawn.so $(includes) jlinuxfork.c -lc
8 |
9 | clean:
10 | rm -fv *.so binrunner
11 |
12 | binrunner:
13 | $(cc) -O3 -Wall -pedantic -o binrunner binrunner.c
14 | chmod -v +x binrunner
15 |
16 | install: binrunner libjlinuxfork.so
17 | sudo mv binrunner /usr/bin
18 | sudo cp libjlinuxfork.so /usr/lib/
19 |
--------------------------------------------------------------------------------
/test/Makefile:
--------------------------------------------------------------------------------
1 | target_dir = ../target
2 | classpath = $(target_dir)/jlinuxfork.jar
3 | javac = $(JAVA_HOME)/bin/javac
4 | java = $(JAVA_HOME)/bin/java
5 |
6 | # TUNE THESE
7 | XMS = 100m
8 | XMX = 200m
9 |
10 |
11 | runtest: compile
12 | $(java) -classpath $(classpath):. -Xms$(XMS) -Xmx$(XMX) -Djava.library.path=$(target_dir) -Dposixspawn.binrunner=$(target_dir)/binrunner TestClass
13 |
14 | runfallback: compile
15 | $(java) -classpath $(classpath):. -Xms$(XMS) -Xmx$(XMX) TestClass
16 |
17 |
18 | compile: TestClass.class
19 | $(javac) -classpath $(classpath):. TestClass.java
20 |
21 |
22 |
--------------------------------------------------------------------------------
/java.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
--------------------------------------------------------------------------------
/src/c/jlinuxfork.h:
--------------------------------------------------------------------------------
1 | /* DO NOT EDIT THIS FILE - it is machine generated */
2 | #include
3 | /* Header for class com_crunchtime_utils_runtime_SpawnedProcess */
4 |
5 | #ifndef _Included_net_axiak_runtime_SpawnedProcess
6 | #define _Included_net_axiak_runtime_SpawnedProcess
7 | #ifdef __cplusplus
8 | extern "C" {
9 | #endif
10 | /*
11 | * Class: net_axiak_runtime_SpawnedProcess
12 | * Method: execProcess
13 | * Signature: ([Ljava/lang/String;[Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;)I
14 | */
15 | JNIEXPORT jint JNICALL Java_net_axiak_runtime_SpawnedProcess_execProcess
16 | (JNIEnv *, jobject, jobjectArray, jobjectArray, jstring, jstring, jobject, jobject, jobject);
17 |
18 | /*
19 | * Class: net_axiak_runtime_SpawnedProcess
20 | * Method: waitForProcess
21 | * Signature: (I)I
22 | */
23 | JNIEXPORT jint JNICALL Java_net_axiak_runtime_SpawnedProcess_waitForProcess
24 | (JNIEnv *, jobject, jint);
25 |
26 | /*
27 | * Class: net_axiak_runtime_SpawnedProcess
28 | * Method: killProcess
29 | * Signature: (I)V
30 | */
31 | JNIEXPORT void JNICALL Java_net_axiak_runtime_SpawnedProcess_killProcess
32 | (JNIEnv *, jobject, jint);
33 |
34 |
35 | #ifdef __cplusplus
36 | }
37 | #endif
38 | #endif
39 |
--------------------------------------------------------------------------------
/test/TestClass.java:
--------------------------------------------------------------------------------
1 | import net.axiak.runtime.*;
2 | import java.io.*;
3 |
4 | class TestClass {
5 | @SuppressWarnings("unused")
6 | public static void main(String [] args) {
7 | String[] memory_soaker = new String[1000000];
8 | String[] cmd = {"ls", "-l"};
9 |
10 | for (int i = 0; i < memory_soaker.length; i++) {
11 | memory_soaker[i] = String.valueOf(i);
12 | }
13 |
14 | System.out.println("");
15 | System.out.println("");
16 | System.out.println("");
17 |
18 | try {
19 | SpawnRuntime runtime = new SpawnRuntime(Runtime.getRuntime());
20 | if (runtime.isLinuxSpawnLoaded()) {
21 | System.out.println(" >> java_posix_spawn libraries are found and being tested.");
22 | } else {
23 | System.out.println(" >> java_posix_spawn is not built and/or there was an error. Running with fallback.");
24 | }
25 |
26 | Process result = runtime.exec(cmd);
27 | int lastChar = 0;
28 | do {
29 | lastChar = result.getInputStream().read();
30 | if (lastChar != -1)
31 | System.out.print((char)lastChar);
32 | } while(lastChar != -1);
33 | } catch (IOException ignored) {
34 | System.out.println(ignored);
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2010-2011, Michael C. Axiak and CrunchTime Information Systems
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 | * Redistributions of source code must retain the above copyright
7 | notice, this list of conditions and the following disclaimer.
8 | * Redistributions in binary form must reproduce the above copyright
9 | notice, this list of conditions and the following disclaimer in the
10 | documentation and/or other materials provided with the distribution.
11 | * Neither the name of the nor the
12 | names of its contributors may be used to endorse or promote products
13 | derived from this software without specific prior written permission.
14 |
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 | DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY
19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
--------------------------------------------------------------------------------
/src/c/binrunner.c:
--------------------------------------------------------------------------------
1 | #include
2 | #include
3 | #include
4 | #include
5 | #include
6 | #include
7 | #include
8 | #include
9 | #include
10 |
11 |
12 | int main(int argc, char ** argv)
13 | {
14 | int fds_map[3] = {-1, -1, -1};
15 | int i;
16 | char ** new_argv;
17 | int open_max;
18 | int open_index;
19 |
20 | if (argc < 6) {
21 | fprintf(stderr, "Usage: %s stdin# stdout# stderr# chdir program [argv0 ... ]\n",
22 | argv[0]);
23 | return -1;
24 | }
25 |
26 | fds_map[0] = atoi(argv[1]);
27 | fds_map[1] = atoi(argv[2]);
28 | fds_map[2] = atoi(argv[3]);
29 |
30 | for (i = 0; i < 3; i++) {
31 | if (dup2(fds_map[i], i) == -1) {
32 | fprintf(stderr, "Error in dup2\n");
33 | return -1;
34 | }
35 | }
36 |
37 | if (!(strlen(argv[4]) == 1 && strncmp(argv[4], ".", 1) == 0)) {
38 | if (chdir(argv[4]) != 0) {
39 | fprintf(stderr, "Error in chdir()\n");
40 | return -1;
41 | }
42 | }
43 |
44 | new_argv = (char **)malloc(sizeof(char *) * (argc - 4));
45 | for (i = 5; i < argc; i++) {
46 | new_argv[i - 5] = argv[i];
47 | }
48 | new_argv[argc - 5] = NULL;
49 |
50 | open_max = sysconf(_SC_OPEN_MAX);
51 | for(open_index=3;open_index < open_max; open_index++) {
52 |
53 | fcntl(open_index, F_SETFD, FD_CLOEXEC);
54 |
55 | }
56 |
57 | if (execvp(argv[5], new_argv) == -1) {
58 | fprintf(stderr, "Error: %s\n", strerror(errno));
59 | return -1;
60 | }
61 | return -1;
62 | }
63 |
--------------------------------------------------------------------------------
/src/java/net/axiak/runtime/SpawnRuntime.java:
--------------------------------------------------------------------------------
1 | package net.axiak.runtime;
2 |
3 | import java.io.File;
4 | import java.io.IOException;
5 | import java.util.StringTokenizer;
6 |
7 | public class SpawnRuntime {
8 | private Runtime runtime;
9 | private Boolean linuxSpawnLoaded;
10 | private static SpawnRuntime instance;
11 |
12 | public SpawnRuntime(Runtime runtime) {
13 | this.runtime = runtime;
14 | linuxSpawnLoaded = SpawnedProcess.isLibraryLoaded();
15 | }
16 |
17 | public Process exec(String [] cmdarray, String [] envp, File chdir) throws IOException {
18 | if (linuxSpawnLoaded) {
19 | return new SpawnedProcess(cmdarray, envp, chdir);
20 | } else {
21 | return runtime.exec(cmdarray, envp, chdir);
22 | }
23 | }
24 |
25 | public Process exec(String [] cmdarray, String [] envp) throws IOException {
26 | return exec(cmdarray, envp, new File("."));
27 | }
28 |
29 | public Process exec(String [] cmdarray) throws IOException {
30 | return exec(cmdarray, null);
31 | }
32 |
33 | public Process exec(String command) throws IOException {
34 | String[] cmdarray = {command};
35 | return exec(command, null, new File("."));
36 | }
37 |
38 | public Process exec(String command, String[] envp, File dir) throws IOException {
39 | if (command.length() == 0) {
40 | throw new IllegalArgumentException("Empty command");
41 | }
42 | StringTokenizer st = new StringTokenizer(command);
43 | String[] cmdarray = new String[st.countTokens()];
44 | for (int i = 0; st.hasMoreTokens(); i++) {
45 | cmdarray[i] = st.nextToken();
46 | }
47 | return exec(cmdarray, envp, dir);
48 | }
49 |
50 | public Process exec(String command, String[] envp) throws IOException {
51 | return exec(command, envp, new File("."));
52 | }
53 |
54 | public static SpawnRuntime getInstance() {
55 | if (instance == null) {
56 | instance = new SpawnRuntime(Runtime.getRuntime());
57 | }
58 | return instance;
59 | }
60 |
61 | public Boolean isLinuxSpawnLoaded() {
62 | return linuxSpawnLoaded;
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | Synopsis
2 | ========
3 |
4 | This code attempts to implement some of the ideas mentioned at http://bugs.sun.com/view_bug.do?bug_id=5049299.
5 |
6 | Basically, the goal is to avoid a fork() call whenever we need to spawn a process so that a large project
7 | does not have to unnecessarily duplicate its virtual memory usage.
8 |
9 | Install
10 | =======
11 |
12 | If you read the Theory of Operation, you'll learn the trick. The upshot is that this library requires
13 | three discrete components to work correctly:
14 |
15 | 1. The JAR (``jlinuxfork.jar``)
16 | 2. The dynamic-linked library (``libjlinuxfork-{arch}.so``)
17 | 3. The binary redirect program (``binrunner``)
18 |
19 | While the JAR file is standard Java and can be dropped in a dependency management via pom or other systems, the other two components make the usage a of this library a little bit trickier.
20 |
21 | Simple Install
22 | --------------
23 |
24 | The simplest install is to just install the components to the operating system.
25 | This requires no configuration at run time and can be built thus::
26 |
27 | $ git clone clone https://axiak@github.com/axiak/java_posix_spawn.git
28 | $ cd java_posix_spawn
29 | $ ant
30 | $ cd target
31 | $ sudo cp libjlinuxfork*so /usr/lib/
32 | $ sudo cp binrunner /usr/bin
33 | $ cp jlinuxfork.jar YOUR_PROJECT_LIB
34 |
35 |
36 | Running the Test
37 | -----------------
38 |
39 | To run the test to see that it works, do the following::
40 |
41 | $ git clone clone https://axiak@github.com/axiak/java_posix_spawn.git
42 | $ cd java_posix_spawn
43 | $ ant
44 | $ cd test
45 | $ make
46 |
47 | This should then print out the output of running "ls -l /" without any of the duplication of virtual address space.
48 |
49 | The question now is, is it behaving better than the standard java library? To be sure, follow these steps:
50 |
51 | 1. Start a VM that does not have much memory (e.g. 512MB)
52 | 2. Edit test/``Makefile`` to adjust ``Xms`` and ``Xmx`` to make the heap large enough.
53 | 3. In the ``test`` directory, run ``make`` to see this library, and run ``make runfallback`` to see the fallback action.
54 |
55 |
56 | Custom Paths
57 | -----------------
58 |
59 | If you don't want to place the libraries in system-level directories, then there
60 | are two java settings you can define when running your java application:
61 |
62 | - ``java.system.path`` - Set to the directory containing the ``.so`` file.
63 | - ``posixspawn.binrunner`` - Set to the complete path of your ``binrunner`` file.
64 |
65 | For example, to run the ``TestClass`` in the repository, the bundled ``Makefile``, runs the following::
66 |
67 | java -classpath ../target/jlinuxfork.jar:. -Djava.library.path=../target -Dposixspawn.binrunner=../target/binrunner TestClass
68 |
69 | Description of Operation
70 | ==============================
71 |
72 | Problem
73 | ---------
74 | In linux, a ``fork`` causes linux to duplicate the allocation of virtual memory
75 | in the virtual memory address space. This works well for lots of small applications spawning each other, but in the java world large applications can spawn small applications in rapid succession and leave the operating system unable to manage the virtual address space.
76 |
77 | Solution
78 | ----------
79 |
80 | The workaround is to use ``vfork`` rather than ``fork`` before spawning another process. With fork, this is how java spawns a process:
81 |
82 | 1. Create 3 pipes (for stdin, stdout, and stderr of the child process)
83 | 2. Fork the process
84 | 3. Redirect stdin, stdout, and stderr to the appropriate pipes
85 | 4. Run ``execve`` or some equivalent to run a different application (and ``brk``)
86 |
87 | In order for ``vfork`` to work in a thread-safe manner, one cannot reference memory
88 | space in the parent process after the ``vfork`` call. (The only legal calls are ``_exit`` and ``execve``. It is not legal to redirect file descriptors after a call to vfork, since we haven't created the new memory address space to redirect yet.) So all we can do in ``vfork`` is run an external program.
89 |
90 | The trick to get around this limitation of ``vfork`` is to create an intermediary program, called ``binrunner``. binrunner is a really simple C program that expects the following::
91 |
92 | Usage: binrunner stdin# stdout# stderr# chdir program [argv0 ... ]
93 |
94 | So essentially you tell it what file descriptors you want redirected, what directory you want to change into, what program you want to run, and you complete running arguments at the end to pass to the next program. Binrunner will then perform the pipe redirection, call chdir() and then call execvp.
95 |
96 | So this is how this library will spawn a new process:
97 |
98 | 1. Create 3 pipes
99 | 2. Call vfork
100 | 3. Run binrunner with the pipe information, the new directory, and the program information
101 | 4. binrunner redirects the pipes
102 | 5. binrunner calls ``chdir()``
103 | 6. binrunner calls ``execvp`` to run the application (and ``brk``)
104 |
105 |
106 |
107 | vfork rather than posix_spawn
108 | -----------------------------
109 |
110 | The library is actually slightly misnamed -- rather than using ``posix_spawn`` blindly, it actually calls ``vfork`` manually. This is due to that fact that posix_spawn does not produce a useful error in the case where the ``execve``
111 | fails. By calling ``vfork`` manually, the application can safely write an error
112 | to ``stderr`` rather than keeping completely silent.
113 |
114 | Documentation
115 | =============
116 |
117 | Beyond this README, I put up a shell of the API at http://axiak.github.com/java_posix_spawn/
118 | The API is supposed to mirror that of Java's Runtime, so it shouldn't be anything new. The only
119 | added method is ``SpawnRuntime.isLinuxSpawnLoaded()``, which indicates if the java library was
120 | successfully able to load the dynamic library. On Windows you would expect that method to always
121 | return false.
122 |
123 | Dependencies
124 | =============
125 |
126 | - Java >= 1.4
127 | - A C compiler
128 | - make
129 | - ant (>=1.7?)
130 |
131 | Supported Platforms
132 | =====================
133 |
134 | The API is written such that it will fall back to the standard java Runtime API if it cannot load the dynamic libraries. This means that windows can just run the java code without any support for compilation (since its Runtime exec doesn't suffer from the but, it's safe).
135 |
136 | As for non-windows systems, this library was tested on linux 32- and 64-bit. No testing has been done on other posix-compliant systems, but the code strictly adheres to posix standards.
137 |
138 | License
139 | =======
140 |
141 | The library is released in the Modified BSD License. See LICENSE for more detail.
142 |
143 | Known Bugs
144 | ==========
145 |
146 | None at the moment. Please file an issue if you find any.
147 |
148 |
--------------------------------------------------------------------------------
/src/java/net/axiak/runtime/SpawnedProcess.java:
--------------------------------------------------------------------------------
1 | package net.axiak.runtime;
2 |
3 | import java.io.*;
4 | import java.security.AccessController;
5 | import java.security.PrivilegedAction;
6 |
7 | public class SpawnedProcess extends Process {
8 | private String name;
9 | private int exitCode;
10 | private int pid;
11 | private boolean hasExited;
12 |
13 | private OutputStream stdin;
14 | private InputStream stdout;
15 | private InputStream stderr;
16 | private FileDescriptor stdin_fd;
17 | public FileDescriptor stdout_fd;
18 | private FileDescriptor stderr_fd;
19 |
20 | private static String binrunner;
21 | private static boolean libLoaded;
22 | private static Throwable libLoadError;
23 |
24 | private native int execProcess(String [] cmdarray, String [] env, String chdir, String binrunner,
25 | FileDescriptor stdin_fd, FileDescriptor stdout_fd,
26 | FileDescriptor stderr_fd) throws IndexOutOfBoundsException, IOException;
27 | private native int waitForProcess(int pid);
28 | private native void killProcess(int pid);
29 |
30 | static {
31 | binrunner = findBinRunner(System.getProperty("posixspawn.binrunner", "binrunner"));
32 | try {
33 | String arch = System.getProperty("os.arch", "i386");
34 | try {
35 | System.loadLibrary("jlinuxfork-" + arch);
36 | } catch (Throwable t) {
37 | System.loadLibrary("jlinuxfork");
38 | }
39 | libLoaded = true;
40 | }
41 | catch (Throwable t) {
42 | libLoaded = false;
43 | libLoadError = t;
44 | }
45 | }
46 |
47 | private static String findBinRunner(String originalPath) {
48 | if (new File(originalPath).exists()) {
49 | return originalPath;
50 | }
51 | String Path = System.getenv("PATH");
52 | String [] paths = ((Path == null) ? "/usr/bin:/bin" : Path).split(":");
53 | for (String path: paths) {
54 | if (path == null) {
55 | continue;
56 | }
57 | File currentFile = new File(path, originalPath);
58 | if (currentFile.exists()) {
59 | return currentFile.getAbsolutePath();
60 | }
61 | }
62 | return null;
63 | }
64 |
65 | private static class Gate {
66 |
67 | private boolean exited = false;
68 | private IOException savedException;
69 |
70 | synchronized void exit() { /* Opens the gate */
71 | exited = true;
72 | this.notify();
73 | }
74 |
75 | synchronized void waitForExit() { /* wait until the gate is open */
76 | boolean interrupted = false;
77 | while (!exited) {
78 | try {
79 | this.wait();
80 | } catch (InterruptedException e) {
81 | interrupted = true;
82 | }
83 | }
84 | if (interrupted) {
85 | Thread.currentThread().interrupt();
86 | }
87 | }
88 |
89 | void setException (IOException e) {
90 | savedException = e;
91 | }
92 |
93 | IOException getException() {
94 | return savedException;
95 | }
96 | }
97 |
98 |
99 | public SpawnedProcess(final String [] cmdarray, final String [] envp, final File chdir) throws IOException {
100 |
101 | if (binrunner == null) {
102 | throw new RuntimeException("Couldn't find binrunner program. Tried: " + System.getProperty("linuxfork.binrunner", "binrunner"));
103 | }
104 |
105 | for (String arg : cmdarray) {
106 | if (arg == null) {
107 | throw new NullPointerException();
108 | }
109 | }
110 | String prog = cmdarray[0];
111 | SecurityManager security = System.getSecurityManager();
112 | if (security != null) {
113 | security.checkExec(prog);
114 | }
115 |
116 | stdin_fd = new FileDescriptor();
117 | stdout_fd = new FileDescriptor();
118 | stderr_fd = new FileDescriptor();
119 |
120 | final Gate gate = new Gate();
121 |
122 | AccessController.doPrivileged(
123 | new PrivilegedAction