├── .cargo ├── cc ├── config.toml ├── linker ├── linker.so └── rustc ├── .gitignore ├── .gitmodules ├── Cargo.toml ├── README.md ├── benches ├── baseline.rs ├── bench.c └── inger.rs ├── compiler ├── .gitignore ├── Makefile ├── ingerc ├── ingerc.cpp └── ingerc.ts ├── configure ├── internal └── libsignal │ ├── Cargo.toml │ └── src │ ├── lib.rs │ ├── libgotcha.rs │ └── pthread.rs ├── ld ├── ldd ├── profile ├── src ├── compile_assert.rs ├── ffi.rs ├── force.rs ├── future.rs ├── groups.rs ├── lib.rs ├── lifetime.rs ├── linger.rs ├── localstores.rs ├── main.rs ├── preemption.rs ├── profiler.rs ├── reusable.rs ├── signals.rs ├── stacks.rs ├── tcb.rs ├── tcbstub.rs ├── timer.rs └── unfurl.rs ├── tests └── inger │ ├── lock.rs │ └── main.rs └── testsuite ├── .gitignore ├── build ├── custom-glibc.patch ├── test └── testinger.c /.cargo/cc: -------------------------------------------------------------------------------- 1 | ../external/libgotcha/cc -------------------------------------------------------------------------------- /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | ../external/libgotcha/libgotcha.cargo -------------------------------------------------------------------------------- /.cargo/linker: -------------------------------------------------------------------------------- 1 | ../ld -------------------------------------------------------------------------------- /.cargo/linker.so: -------------------------------------------------------------------------------- 1 | ../ld.so -------------------------------------------------------------------------------- /.cargo/rustc: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LIBGOTCHA_SYMBOLS="--globalize-symbol libgotcha_arch_prctl --globalize-symbol libgotcha_pthread_kill" exec rustc "$@" 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.cargo/config 2 | /target/ 3 | 4 | *.a 5 | *.lock 6 | *.log 7 | *.rs.bk 8 | *.so 9 | 10 | !/.cargo/cc.so 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libgotcha"] 2 | path = external/libgotcha 3 | url = ../libinger 4 | branch = gotcha 5 | [submodule "libtimetravel"] 6 | path = external/libtimetravel 7 | url = ../libinger 8 | branch = timetravel 9 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "inger" 3 | version = "0.0.0" 4 | edition = "2018" 5 | 6 | [lib] 7 | crate-type = ["dylib"] 8 | harness = false 9 | 10 | [features] 11 | notls = [] 12 | 13 | [dependencies] 14 | libc = "*" 15 | 16 | [dependencies.gotcha] 17 | path = "external/libgotcha" 18 | 19 | [dependencies.signal] 20 | path = "internal/libsignal" 21 | features = ["libgotcha"] 22 | 23 | [dependencies.timetravel] 24 | path = "external/libtimetravel" 25 | features = ["libgotcha"] 26 | 27 | [[bench]] 28 | name = "inger" 29 | harness = false 30 | 31 | [[bench]] 32 | name = "baseline" 33 | harness = false 34 | 35 | [dev-dependencies] 36 | bencher = "*" 37 | 38 | [workspace] 39 | exclude = ["external/libgotcha/examples/cargo"] 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | _libinger_ 2 | ========== 3 | This is the source code of the _libinger_ library for making function calls with timeouts from Rust 4 | and C, described in our ATC '20 paper, "Lightweight Preemptible Functions." _As the proverb goes, 5 | "If you don't want your function calls to linger, link with `-linger`."_ 6 | 7 | Also present are a few supporting libraries: 8 | * `external/libgotcha`: runtime providing the libset abstraction that makes all of this possible 9 | * `external/libtimetravel`: makes it easier to safely perform unstructured jumps from Rust code 10 | * `internal/libsignal`: unsafe wrappers around C library functions (don't use at home!) 11 | 12 | 13 | License 14 | ------- 15 | The entire contents and history of this repository are distributed under the following license: 16 | ``` 17 | Copyright 2020 Carnegie Mellon University 18 | 19 | Licensed under the Apache License, Version 2.0 (the "License"); 20 | you may not use this file except in compliance with the License. 21 | You may obtain a copy of the License at 22 | 23 | http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | Unless required by applicable law or agreed to in writing, software 26 | distributed under the License is distributed on an "AS IS" BASIS, 27 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 28 | See the License for the specific language governing permissions and 29 | limitations under the License. 30 | ``` 31 | 32 | 33 | System requirements 34 | ------------------- 35 | Presently, x86-64 GNU/Linux is the only supported platform. The build requirements are as follows: 36 | * `rustc` ≥1.36.0 (versions starting with 1.37.0 include a breaking change to dylib symbol exports— 37 | https://github.com/rust-lang/rust/issues/64340 —but the build system now implements a workaround) 38 | * `cargo` ≤1.39.0/"0.40.0" or ≥1.55.0/"0.56.0" (a version between these introduced a breaking 39 | but since fixed regression in path resolution: https://github.com/rust-lang/cargo/issues/8202) 40 | * `bindgen` ≤0.52 (or a newer version with a wrapper script that passes the `--size_t-is-usize` 41 | flag, as per https://github.com/rust-lang/rust-bindgen/issues/1901) 42 | * `gcc` (tested with 9.2.1) 43 | * GNU `make` 44 | * GNU binutils (tested with 2.34) 45 | * GNU libc ≤2.33 (versions starting with 2.30 include a breaking change to `dlopen()` behavior on 46 | executables—https://sourceware.org/bugzilla/show_bug.cgi?id=24323 —but libgotcha now implements a 47 | workaround; versions starting with 2.34 combine libpthread into libc, which is unlikely to work 48 | out of the box) 49 | * `libebl` from elfutils 0.176 (newer versions eliminate the libebl shared library, which we 50 | currently depend on directly) 51 | * `git` 52 | On Debian-based systems, bindgen appears to require the libclang-dev package; if it is missing, you 53 | will get errors about missing headers. If you don't want to install this package, you may be able 54 | to work around it by symlinking /usr/local/include/asm to /usr/include/asm. 55 | 56 | 57 | A note on terminology 58 | --------------------- 59 | We use the term lightweight preemptible function (LPF) to refer to the timed _version_ of a 60 | function, as invoked via the `launch()` wrapper function in this library. It's not quite right to 61 | say that _libinger_ "provides preemptible functions"; rather, it provides a transformation from 62 | an ordinary function into a preemptible one. 63 | 64 | To provide the memory isolation necessary to introduce preemption and asynchronous cancellation at 65 | sub-thread granularity without breaking existing program dependencies, the _libgotcha_ runtime 66 | allocates a separate copy of all the program's loaded dynamic libraries for each preemptible 67 | function. While the paper refers to this isolation unit as a libset, that term was unfortunately 68 | coined late in development; as such, the source code and configuration variables refer to it as a 69 | "group" instead. 70 | 71 | 72 | A note on design 73 | ---------------- 74 | The paper describes the _libinger_ API, and the generated HTML documentation gives a few more usage 75 | details. Here are some of the guiding principles that inspired our interface choices: 76 | * **We do not assume users need asynchrony.** Hence, preemptible functions _run on the same kernel 77 | thread as their caller_. This is good for performance (and especially invocation latency), but 78 | it is also important to be aware of; for instance, it means that a preemptible function will 79 | deadlock if it attempts to acquire a lock held by its caller, or vice versa. If asynchrony is 80 | something you require, you can build it atop _libinger_, as we have demonstrated with our 81 | _libturquoise_ preemptive userland thread library. 82 | * **We assume that simply calling a function with a timeout is the common use case.** As such, the 83 | `launch()` wrapper both constructs and begins executing the preemptible function rather than 84 | asking the user to first employ a separate constructor. The latter behavior can be achieved by 85 | passing the sentinel `0` as the timeout, then later using `resume()` to invoke the preemptible 86 | function. 87 | * **We endeavor to keep argument and return value passing simple yet extensible.** Because Rust 88 | supports closures, the Rust version of `launch()` accepts only nullary functions: those seeking 89 | to pass arguments should just capture them from the environment. Because C supports neither 90 | closures nor generics, the C version of `launch()` accepts a single `void *` argument that can 91 | serve as an inout parameter; it occupies the last position in the parameter list to permit 92 | (possible) eventual support for variable argument lists. 93 | * **We choose defaults to favor flexibility and performance.** When a preemptible function times 94 | out, _libinger_ assumes the caller might later want to resume it from where it left off. As 95 | such, both `launch()` and `resume()` pause in this situation; this incurs some memory and time 96 | overhead to provide a separate execution stack and package the continuation object, but has much 97 | lower overall cost than asynchronous cancellation. If the program does require cancellation, it 98 | can request it explicitly by calling `cancel()` (C) or dropping the continuation object (Rust). 99 | * **We provide preemption out of the box, but the flexibility to cooperatively yield.** The 100 | `pause()` primitive allows a preemptible function to "yield" back to its caller by immediately 101 | "timing out." One can imagine building higher-level synchronization constructs atop this; for 102 | example, a custom mutex that paused instead of blocking would allow two or more preemptible 103 | functions to share state, even when some of them executed from the same kernel thread. 104 | * **We favor a simple, language-agnostic interface.** Because the interface is based on the 105 | foundational function call abstraction, it looks very similar in both C and Rust. Someday, it 106 | may look _equally_ similar in other languages as well, and in the meantime, it ought to enjoy 107 | compatibility with languages' C foreign-function interfaces. It's relatively simple to integrate 108 | higher-level abstractions on top, such as the Rust futures wrapper available in _libinger_'s 109 | `future` module. 110 | 111 | 112 | A note on implementation 113 | ------------------------ 114 | The timer signal handler in _libinger_ refuses to pause while the next libset is set to 0 (the 115 | starting libset). Because _libinger_ is statically linked with _libgotcha_, the latter enforces a 116 | transparent switch to this libset whenever a dynamic function call transfers control into the module 117 | in the process image that corresponds to `libinger.so`. This means that preemption is deferred on a 118 | given kernel thread while _libinger_'s own code is executing on that thread. 119 | 120 | Of course, things are not quite that simple. There are noteworthy exceptions to the rule: 121 | * The public _libinger_ Rust interface includes a number of generic functions. Because the Rust 122 | compiler monomorphizes such functions for the client code that uses them, _their implementations 123 | are_ not _in `libinger.so`!_ Rather, there is, roughly speaking, one or more copies of them 124 | (specialized for various type arguments) in each program module that calls them. The generic 125 | functions are therefore implemented such that they package everything that differs by type, then 126 | call into non-specialized functions such as `setup_stack()` and `switch_stack()` to do the scary 127 | stuff. 128 | * The `resume_preemption()` function is installed as a _libgotcha_ callback hook, and is implicitly 129 | invoked at the end of each deferred-preemption library call made by a preemptible function. 130 | _This happens in the preemptible function's libset rather than the starting one_; this is 131 | essential because the callback's main task is to force the timer signal handler to run 132 | _immediately_ and check for a timeout, and we don't want the libset to inhibit preemption! 133 | 134 | 135 | Building glibc from source 136 | -------------------------- 137 | _Note: If you seek only to test_ libinger _with a very small program that doesn't require many 138 | preemptible functions and has few dynamic library dependencies, and you are willing to build 139 | without debug symbols, you may be able to skip this section provided you restrict the number of 140 | libsets at runtime by exporting the `$LIBGOTCHA_NUMGROUPS` variable before invoking your program 141 | (e.g., `$ LIBGOTCHA_NUMGROUPS=1 ./i_only_use_one_lpf_at_a_time`)._ 142 | 143 | Although _libinger_ is compatible with an unmodified glibc in principle, in practice the build 144 | configuration used by most distributions is insufficient for two reasons: 145 | * By loading numerous copies of the application's libraries, we tend to exhaust the static storage 146 | pool provided by the dynamic linker. If your program hits this limit, it will crash at load time 147 | with an error like: `yourprogram: libgotcha error: Unable to load ancillary copies of library: 148 | somelibrary.so: cannot allocate memory in static TLS block`. 149 | * Stock glibc builds are limited to 16 linker namespaces, enough to support only 15 preemptible 150 | functions at any given time. If your program hits this limit, it will crash at runtime with an 151 | error like: `launch(): too many active timed functions`. 152 | 153 | Unfortunately, these configuration parameters are baked into the dynamic linker at build time. 154 | What's more, changing (at least) the latter alters the size of internal structures that are shared 155 | between `ld-linux.so`, `libc.so`, `libpthread.so`, and others, so making changes requires rebuilding 156 | all of glibc. Fortunately, provided you set the prefix properly when building glibc, the dynamic 157 | linker will know where to search (by absolute path) for the other libraries; as such, most 158 | applications that depend on `libinger.so` need only define a custom interpreter path pointing to the 159 | `ld-linux-x86-64.so.2` file in your build directory. 160 | 161 | Note that glibc 2.32 eliminated the below `TLS_STATIC_SURPLUS` compile-time constant and replaced it 162 | with a runtime tunable read from the environment. Debian backported this change to their glibc 2.31 163 | distribution. If you are on an affected version, disregard the related steps below; instead, export 164 | something like `GLIBC_TUNABLES=glibc.rtld.optional_static_tls=0x10000` when running your program. 165 | 166 | Follow these steps to build your own glibc, where all paths are relative to the root of this 167 | repository: 168 | 1. Clone the glibc source code: `$ git clone git://sourceware.org/git/glibc`. I'll assume you put 169 | it in `../glibc`, and have checked out whatever version you want to use. 170 | 1. Edit `TLS_STATIC_SURPLUS` in `../glibc/elf/dl-tls.c` to raise the multiplier on `DL_NNS` by at 171 | least two orders of magnitude*. Our current recommendation is a multiplier of 2000. 172 | 1. Change `DL_NNS` in `../glibc/sysdeps/generic/ldsodefs.h` to exceed by at least one the maximum 173 | number of simultaneous preemptible functions your program will need*. The default value of 16 174 | should be fine for use cases with low parallelism and where preemptible functions are usually 175 | allowed to run to completion before launching others; we have not tested values above 512. 176 | 1. Make a new empty directory to use for the build and `cd` into it. Let's say this is located at 177 | `../rtld`. 178 | 1. Decide where you want to place your new glibc "installation." I'll be using `../rtld/usr`. 179 | 1. Still working in `../rtld`, configure the build: 180 | `$ ../glibc/configure --prefix=$PWD/usr --disable-werror`. 181 | 1. Start the build: `$ make -j8`, assuming you have 8 cores. 182 | 1. When the build finishes successfully, run `$ make install` to populate the "installation" 183 | directory you chose earlier. Note that the `install` target appears to be finicky about running 184 | with multiple cores, so combining this step with the previous one can cause issues (at least) 185 | the first time you build glibc. 186 | 1. Add to the `../rtld/usr/lib` directory a symlink to _each_ of the following libraries on your 187 | system: `libasm.so.1` (from elfutils), `libbz2.so.1.0`, `libdw.so.1` (also from elfutils), 188 | `libgcc_s.so.1`, `liblzma.so.5`, `libstd-*.so` (from Rust), and `libz.so.1`. 189 | 1. Move back into the root of this repository and tell the build system where to find your custom 190 | dynamic linker build: `ln -s ../rtld/usr/lib/ld-linux-x86-64.so.2 ld.so`. 191 | 192 | \* The `inger-atc20-cfp` tag records the revision used to run the _libinger_ microbenchmarks for the 193 | ATC paper, and is annotated with the build customizations applied to glibc and _libgotcha_ for 194 | that use case. Similarly, the repository containing our full-system benchmarks contains 195 | annotated tags recording the configuration used to evaluate _hyper_ and _libpng_. To see the 196 | annotations, use `git show`. To run the benchmarks, use `cargo bench`. 197 | 198 | 199 | Building _libinger_ 200 | ------------------- 201 | The _libinger_ library is built with Cargo, although Make is also required because of _libgotcha_. 202 | 1. Follow the steps in the preceding section to build your own glibc root, or skip them at your own 203 | peril (in which case you will have to ignore the error in the following step). 204 | 1. Working in the root of this repository, configure the build: `$ ./configure`. 205 | 1. Build the library: `$ cargo build --release`. 206 | 1. If you intend to use preemptible functions from C, generate the header: 207 | `$ cargo run --release >libinger.h` 208 | 209 | The `libinger.so` library will be located in `target/release`; this and the header are the only 210 | files required to build C programs against _libinger_. For Rust programs, the `rlib` files under 211 | `target/release/deps` must also be present during the build phase. 212 | 213 | To build and view the HTML documentation for the Rust interface, simply do: `$ cargo doc --open`. 214 | There is no documentation for the C interface, but the header should be "self documenting," _as they 215 | say_. 216 | 217 | 218 | Building programs against _libinger_ 219 | ------------------------------------ 220 | _Note: This section describes how to build simple C and Rust programs against_ libinger _by invoking 221 | the compiler directly. If you want to use Cargo to build a Rust crate that depends on_ libinger _, 222 | skip to the next one._ 223 | 224 | Let's assume you want to write a small program that uses _libinger_, and (for simplicity) that you 225 | want to store the source file and executable in the root of this repository. 226 | 227 | For a C program, you'll want to begin your file with `#include "libinger.h"` and build with some 228 | variation of this command: 229 | ``` 230 | $ cc -fpic -Ltarget/release -Wl,-R\$ORIGIN/target/release -Wl,-I./ld.so -o prog prog.c -linger 231 | ``` 232 | 233 | For a Rust program, you should start your file with `extern crate inger;` and build like: 234 | ``` 235 | $ rustc -Ltarget/release/deps -Clink-arg=-Wl,-R\$ORIGIN/target/release -Clink-arg=-Wl,-I./ld.so prog.rs 236 | ``` 237 | 238 | Of course, you probably want to add other flags to your compiler invocation to request things like 239 | language edition, optimization, debugging, and warnings. If you are trying to use the system 240 | dynamic linker instead of one that you built from source, omit the [`-Clink-arg=`]`-Wl,-I./ld.so` 241 | switch, cross your fingers, and eat a bowl of lucky charms to minimize the probability that your 242 | program runs out of static storage during initialization. 243 | 244 | In the case of both languages, the resulting executable is distributable, and will continue to work 245 | as long as it is run from a directory containing both `ld.so` and `target/release/libinger.so`. 246 | Note however, that the dynamic linker pointed to by the former symlink is hard to distribute without 247 | an installer because its default library search path is based on the (absolute) prefix path used 248 | when configuring the build. 249 | 250 | 251 | Building crates against _libinger_ with Cargo 252 | --------------------------------------------- 253 | Cargo can build crates that depend on _libinger_, but this requires some extra configuration, not 254 | least because it very aggressively prefers to build dependencies as static libraries rather than 255 | shared ones. To set things up, after configuring the build system in this repository, do the 256 | following from the root of your other Cargo tree: 257 | ``` 258 | $ ln -s ../path/to/libinger/.cargo 259 | ``` 260 | 261 | If you have a nested tree of projects that depend on _libinger_, it suffices to do this once at the 262 | outermost level of the directory structure (but still within a Cargo project directory!). 263 | 264 | Now you'll just need to add an entry like this to your project's `Cargo.toml`: 265 | ``` 266 | [dependencies.inger] 267 | path = "../path/to/libinger" 268 | ``` 269 | 270 | 271 | Debugging tips 272 | -------------- 273 | If adding _libinger_ to your program causes it to segfault or otherwise crash, it's possible the 274 | culprit is _libgotcha_'s support for intercepting dynamic accesses to global variables, which makes 275 | use of heuristics. Ordinarily we notify the application of accesses we were unable to resolve by 276 | forwarding it a segmentation fault; this approach is intended to support applications and runtimes 277 | that respond to segfaults, but it can sometimes obscure the problem. See whether running your 278 | program with forwarding disabled gives a more informative _libgotcha_ error: 279 | ``` 280 | $ LIBGOTCHA_ABORTSEGV= ./yourprogram 281 | ``` 282 | 283 | If you have produced a release build and later need to switch to a debugging one, please note that 284 | **the build system does not support both types of builds simultaneously**: you must perform a full 285 | clean in between by running `$ ./configure`. If you are using the C interface, also note that 286 | **you must regenerate the header without the `--release` switch**; otherwise, the type declaration's 287 | size will not match the implementation! Unless _your_ application uses Cargo as a build system, 288 | you'll need to replace all instances of `release` with `debug` in the command you use to build it. 289 | If substituting a debug build of _libinger_ causes your program to crash on initialization with an 290 | error like `Unable to load ancillary copies of library`, either rebuild glibc with a higher 291 | `TLS_STATIC_SURPLUS` or launch with a reduced number of libsets, e.g.: 292 | ``` 293 | $ LIBGOTCHA_NUMGROUPS=2 ./yourprogram 294 | ``` 295 | 296 | The `valgrind` suite (at least Memcheck and Callgrind from version 3.16.1) is known to work, but 297 | currently conflicts with _libgotcha_'s global variable access interception. You can work around 298 | this by switching it off, at the risk of altering the semantics of your program: 299 | ``` 300 | $ LIBGOTCHA_NOGLOBALS= valgrind ./yourprogram #check it for memory errors 301 | $ LIBGOTCHA_NOGLOBALS= valgrind --tool=callgrind ./yourprogram #profile it 302 | ``` 303 | 304 | Sadly, LLVM's sanitizers rely heavily on dynamic linking tricks that are incompatible with 305 | _libgotcha_, so they are not available at this time. 306 | 307 | The `gdb` debugger works both with and without global variable access interception, although we 308 | recommend first trying without it for a much less confusing experience out of the box: 309 | ``` 310 | $ LIBGOTCHA_NOGLOBALS= gdb ./yourprogram 311 | ``` 312 | 313 | If you do need to preserve global variable semantics while debugging, you probably don't want to 314 | step into _libgotcha_'s segfault handler unless you're trying to debug the feature itself. You can 315 | instruct `gdb` to ignore segmentation faults like so: 316 | ``` 317 | $ gdb -ex handle\ SIGSEGV\ noprint ./yourprogram 318 | ``` 319 | 320 | The `rr` reverse-debugging backend (at least version 5.4.0) also works both with and without global 321 | variable access interception, but conflicts with _libgotcha_'s interception of certain calls into 322 | the runtime dynamic loader. Invoke it like so (omitting `LIBGOTCHA_NOGLOBALS=` if desired): 323 | ``` 324 | $ LIBGOTCHA_NODYNAMIC= LIBGOTCHA_NOGLOBALS= rr ./yourprogram #record execution for reverse debugging 325 | ``` 326 | 327 | If you step into code located outside the main libset, GDB will be missing symbol information, and 328 | therefore unable to display backtraces or source code, or apply your symbol breakpoints. Luckily, 329 | _libgotcha_ includes a debugger script to fix this problem. You must have the glibc sources 330 | corresponding to the `ld.so` build you are using. Add the following to your GDB arguments however 331 | you are invoking it (via `gdb`, `rr replay`, etc.): 332 | ``` 333 | -x .../path/to/libinger/external/libgotcha/libgotcha.gdb -ex dir\ .../path/to/glibc/dlfcn:.../path/to/glibc/elf 334 | ``` 335 | 336 | When debugging the program directly with GDB rather than from a recorded rr execution capture, the 337 | debugger and/or program may become overwhelmed by the extremely frequent preemption signals. If the 338 | debugger freezes or the program doesn't make progress when single stepping, try recompiling 339 | _libinger_ with a higher (say, by an order of magnitude or so) `QUANTUM_MICROSECONDS` value. If you 340 | still have trouble single stepping or the time spent stopped is causing _libinger_ to preempt the 341 | task you are trying to debug, you can disable preemption altogether by issuing a variation of the 342 | following GDB command to cover the preemption signal(s) affecting your task's execution: 343 | ``` 344 | (gdb) handle SIGALRM nopass 345 | ``` 346 | 347 | 348 | Troubleshooting 349 | --------------- 350 | **My distribution ships a version of glibc that is newer than 2.33. What should I do?** 351 | 352 | Such versions are untested. If you do not encounter any obvious issues, great! Otherwise, no need 353 | to modify your installation; instead, just follow the steps under _Building glibc from source_ above 354 | to build version 2.33. When linking your executable, you will want to use GCC's `-Wl,-I` switch 355 | (`-Clink-arg=-Wl,-I` in the case of rustc) to specify the newly-symlinked dynamic linker as the 356 | program's interpreter. If you are building the program using Cargo and its source is located in 357 | tree or you have symlinked its `.cargo` to this repository's folder of the same name, passing the 358 | flag should happen automatically. Note that, if you get any version errors when running the 359 | resulting binary, you may need to add a further `-L` and the path to the `lib` directory generated 360 | by the glibc `install` target to your linker flags and rebuild. 361 | 362 | **My distribution ships a version of elfutils that is newer than 0.176. What should I do?** 363 | 364 | It probably does! The easiest thing to do is to downgrade the package. For instance, if using 365 | Debian, download (at least) the following binary packages from https://snapshot.debian.org: 366 | `libasm1`, `libasm-dev`, `libdw1`, `libelf1`, and `libelf-dev`. Put the packages into a new 367 | directory, `cd` into it, and run `apt -s install ./*.deb`. If it shows any dependency errors, 368 | download additional packages to address them and add them to the directory; you will probably need 369 | `gdb` version 9.2-1, `libpython3.8`, `libpython3.8-minimal`, and `libpython3.8-stdlib`. Then 370 | perform the downgrade by running `sudo apt install ./*.deb`. 371 | 372 | **I get the assertion: `ELF64_ST_TYPE(st->st_info) != STT_GNU_IFUNC)`!** 373 | 374 | This appears to occur on some Ubuntu systems because of the way Canonical packages zlib (`libz.so`). 375 | The easiest workaround is to grab a closely matching package version from 376 | https://packages.debian.org and extract the shared library file. Then either export the 377 | `$LD_LIBRARY_PATH` environment variable to the path to its containing folder when executing your 378 | application, or just place the shared library in the `lib` directory generated by glibc's `install` 379 | target if/when you built it from source. 380 | 381 | **I get some other error at some point.** 382 | 383 | Start by rerunning `./configure` to clean the build. Make sure it finds all the dependencies. Look 384 | back at _System requirements_ and verify that you are not on a `cargo` version affected by the 385 | regression, and that you have the described wrapper script in your `$PATH` if you have a 386 | sufficiently new version of `bindgen` (a shell alias will not suffice). 387 | 388 | Next, try building _libgotcha_ in isolation. Just `cd` into `external/libgotcha` and type `make` 389 | (without passing `-j`). If you get errors here, go back to the previous paragraph, pinch yourself, 390 | and check all that one more time. 391 | 392 | If you get this far, try building _libinger_ without per--preemptive function thread-local variables 393 | by passing `--features notls` to your Cargo command. If you are still stuck, see whether you can 394 | trace the source of the error using any of the techniques under _Debugging tips_ above. 395 | -------------------------------------------------------------------------------- /benches/baseline.rs: -------------------------------------------------------------------------------- 1 | use bencher::Bencher; 2 | use bencher::benchmark_group; 3 | use bencher::benchmark_main; 4 | use libc::exit; 5 | use libc::waitpid; 6 | use std::ffi::c_void; 7 | use std::ptr::null; 8 | use std::ptr::null_mut; 9 | 10 | benchmark_group![bench, fork, vfork, pthread_create, pthread_join]; 11 | 12 | fn fork(lo: &mut Bencher) { 13 | use libc::fork; 14 | 15 | lo.iter(|| { 16 | let pid = unsafe { 17 | fork() 18 | }; 19 | if pid == 0 { 20 | unsafe { 21 | exit(0); 22 | } 23 | } 24 | unsafe { 25 | waitpid(pid, null_mut(), 0); 26 | } 27 | }) 28 | } 29 | 30 | fn vfork(lo: &mut Bencher) { 31 | use libc::vfork; 32 | 33 | lo.iter(|| { 34 | let pid = unsafe { 35 | vfork() 36 | }; 37 | if pid == 0 { 38 | unsafe { 39 | exit(0); 40 | } 41 | } 42 | unsafe { 43 | waitpid(pid, null_mut(), 0); 44 | } 45 | }) 46 | } 47 | 48 | fn pthread_create(lo: &mut Bencher) { 49 | use libc::pthread_create; 50 | use libc::pthread_join; 51 | 52 | lo.iter(|| { 53 | let mut tid = 0; 54 | unsafe { 55 | pthread_create(&mut tid, null(), identity, null_mut()); 56 | pthread_join(tid, null_mut()); 57 | } 58 | }) 59 | } 60 | 61 | fn pthread_join(lo: &mut Bencher) { 62 | use libc::pthread_create; 63 | use libc::pthread_join; 64 | use std::collections::VecDeque; 65 | 66 | let mut tids: VecDeque<_> = (0..100_000).map(|_| { 67 | let mut tid = 0; 68 | unsafe { 69 | pthread_create(&mut tid, null(), identity, null_mut()) 70 | }; 71 | tid 72 | }).collect(); 73 | lo.iter(|| { 74 | let tid = tids.pop_front().unwrap(); 75 | unsafe { 76 | pthread_join(tid, null_mut()); 77 | } 78 | }); 79 | } 80 | 81 | extern fn identity(val: *mut c_void) -> *mut c_void { val } 82 | 83 | benchmark_main! { 84 | bench 85 | } 86 | -------------------------------------------------------------------------------- /benches/bench.c: -------------------------------------------------------------------------------- 1 | #include "../libinger.h" 2 | 3 | #ifdef BREAKDOWN 4 | # include 5 | #endif 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #ifdef BREAKDOWN 13 | # define ITERS 75 14 | #else 15 | # define ITERS 1000000 16 | #endif 17 | 18 | #ifdef BREAKDOWN 19 | # pragma weak ProfilerFlush 20 | # pragma weak ProfilerStart 21 | # pragma weak ProfilerStop 22 | #endif 23 | 24 | struct bothtimes { 25 | unsigned long usr; 26 | unsigned long sys; 27 | long pfs; 28 | }; 29 | 30 | static unsigned long nsnow(void) { 31 | struct timespec tv; 32 | clock_gettime(CLOCK_REALTIME, &tv); 33 | return tv.tv_sec * 1000000000 + tv.tv_nsec; 34 | } 35 | 36 | static struct bothtimes usboth(void) { 37 | struct rusage ru; 38 | getrusage(RUSAGE_SELF, &ru); 39 | return (struct bothtimes) { 40 | .usr = ru.ru_utime.tv_sec * 1000000 + ru.ru_utime.tv_usec, 41 | .sys = ru.ru_stime.tv_sec * 1000000 + ru.ru_stime.tv_usec, 42 | .pfs = ru.ru_majflt + ru.ru_minflt, 43 | }; 44 | } 45 | 46 | static void nop(void *ign) { 47 | (void) ign; 48 | } 49 | 50 | int main(int argc, char **argv) { 51 | unsigned long each[ITERS + 1]; 52 | #ifdef BREAKDOWN 53 | struct bothtimes breakdown[ITERS + 1]; 54 | #endif 55 | 56 | char fname[strlen(*argv) + sizeof ".prof." + 2]; 57 | unsigned long nsthen = nsnow(); 58 | for(int iter = 0; iter < ITERS; ++iter) { 59 | #ifdef BREAKDOWN 60 | if(ProfilerFlush && ProfilerStart && ProfilerStop) 61 | switch(iter) { 62 | case 51: 63 | case 65: 64 | ProfilerFlush(); 65 | ProfilerStop(); 66 | case 5: 67 | sprintf(fname, "%s.prof.%02d", *argv, iter); 68 | ProfilerStart(fname); 69 | } 70 | putc('.', stderr); 71 | each[iter] = nsnow(); 72 | breakdown[iter] = usboth(); 73 | #endif 74 | launch(nop, UINT64_MAX, NULL); 75 | } 76 | each[ITERS] = nsnow(); 77 | #ifdef BREAKDOWN 78 | breakdown[ITERS] = usboth(); 79 | if(ProfilerFlush && ProfilerStop) { 80 | ProfilerFlush(); 81 | ProfilerStop(); 82 | } 83 | putc('\n', stderr); 84 | #endif 85 | 86 | unsigned long nswhen = (nsnow() - nsthen) / ITERS; 87 | #ifdef BREAKDOWN 88 | for(int iter = 0; iter < ITERS; ++iter) { 89 | unsigned long time = each[iter + 1] - each[iter]; 90 | printf("%2d %lu.%03lu μs\n", iter, time / 1000, time % 1000); 91 | 92 | unsigned long usr = breakdown[iter + 1].usr - breakdown[iter].usr; 93 | unsigned long sys = breakdown[iter + 1].sys - breakdown[iter].sys; 94 | long pfs = breakdown[iter + 1].pfs - breakdown[iter].pfs; 95 | printf("(%lu μs usr, %lu μs sys, %ld pfs)\n", usr, sys, pfs); 96 | } 97 | #endif 98 | printf("%ld.%03ld μs\n", nswhen / 1000, nswhen % 1000); 99 | 100 | return 0; 101 | } 102 | -------------------------------------------------------------------------------- /benches/inger.rs: -------------------------------------------------------------------------------- 1 | use bencher::Bencher; 2 | use bencher::benchmark_group; 3 | use bencher::benchmark_main; 4 | use inger::concurrency_limit; 5 | use inger::nsnow; 6 | use inger::pause; 7 | use std::fs::File; 8 | use std::io::Write; 9 | use std::sync::atomic::AtomicBool; 10 | use std::sync::atomic::AtomicU64; 11 | use std::sync::atomic::Ordering; 12 | 13 | benchmark_group![bench, launch, resume, renew]; 14 | 15 | fn launch(lo: &mut Bencher) { 16 | use inger::STACK_N_PREALLOC; 17 | use inger::abort; 18 | use inger::launch; 19 | use inger::resume; 20 | use std::mem::MaybeUninit; 21 | let mut lingers: [MaybeUninit<_>; STACK_N_PREALLOC] = unsafe { 22 | MaybeUninit::uninit().assume_init() 23 | }; 24 | let during = AtomicU64::default(); 25 | 26 | let mut into = 0; 27 | let mut outof = 0; 28 | let mut index = 0; 29 | let paused = AtomicBool::from(false); 30 | lo.iter(|| { 31 | assert!(index < concurrency_limit(), "LIBSETS tunable set too low!"); 32 | 33 | let before = nsnow(); 34 | lingers[index] = MaybeUninit::new(launch(|| { 35 | during.store(nsnow(), Ordering::Relaxed); 36 | pause(); 37 | if ! paused.load(Ordering::Relaxed) { 38 | abort("pause() did not context switch!"); 39 | } 40 | }, u64::max_value()).unwrap()); 41 | 42 | let after = nsnow(); 43 | let during = during.load(Ordering::Relaxed); 44 | into += during - before; 45 | outof += after - during; 46 | 47 | index += 1; 48 | }); 49 | 50 | if let Ok(mut file) = File::create("bench_launch.log") { 51 | let index: u64 = index as _; 52 | writeln!(file, "entry launch ... {} ns/iter", into / index).unwrap(); 53 | writeln!(file, "exit launch ... {} ns/iter", outof / index).unwrap(); 54 | writeln!(file, "(ran for {} iterations)", index).unwrap(); 55 | } 56 | 57 | paused.store(true, Ordering::Relaxed); 58 | for linger in &mut lingers[..index] { 59 | let linger = linger.as_mut_ptr(); 60 | let linger = unsafe { 61 | &mut *linger 62 | }; 63 | resume(linger, u64::max_value()).unwrap(); 64 | } 65 | } 66 | 67 | fn resume(lo: &mut Bencher) { 68 | use inger::launch; 69 | use inger::resume; 70 | 71 | let run = AtomicBool::from(true); 72 | let during = AtomicU64::default(); 73 | let mut lingers: Vec<_> = (0..concurrency_limit()).map(|_| launch(|| while run.load(Ordering::Relaxed) { 74 | pause(); 75 | during.store(nsnow(), Ordering::Relaxed); 76 | }, u64::max_value()).unwrap()).collect(); 77 | let nlingers = lingers.len(); 78 | 79 | let mut into = 0; 80 | let mut outof = 0; 81 | let mut index = 0; 82 | lo.iter(|| { 83 | let before = nsnow(); 84 | resume(&mut lingers[index % nlingers], u64::max_value()).unwrap(); 85 | 86 | let after = nsnow(); 87 | let during = during.load(Ordering::Relaxed); 88 | into += during - before; 89 | outof += after - during; 90 | 91 | index += 1; 92 | }); 93 | 94 | if let Ok(mut file) = File::create("bench_resume.log") { 95 | let index: u64 = index as _; 96 | writeln!(file, "entry resume ... {} ns/iter", into / index).unwrap(); 97 | writeln!(file, "exit resume ... {} ns/iter", outof / index).unwrap(); 98 | writeln!(file, "(ran for {} iterations)", index).unwrap(); 99 | } 100 | 101 | run.store(false, Ordering::Relaxed); 102 | for linger in &mut lingers { 103 | resume(linger, u64::max_value()).unwrap(); 104 | } 105 | } 106 | 107 | fn renew(lo: &mut Bencher) { 108 | use inger::launch; 109 | 110 | let lingers: Vec<_> = (0..concurrency_limit()).map(|_| launch(pause, u64::max_value()).unwrap()).collect(); 111 | let mut lingers = lingers.into_iter(); 112 | lo.iter(|| 113 | drop(lingers.next().expect("LIBSETS tunable set too low!")) 114 | ); 115 | } 116 | 117 | benchmark_main! { 118 | bench 119 | } 120 | -------------------------------------------------------------------------------- /compiler/.gitignore: -------------------------------------------------------------------------------- 1 | /X86InstrInfo.h 2 | /X86RegisterInfo.h 3 | 4 | *.o 5 | -------------------------------------------------------------------------------- /compiler/Makefile: -------------------------------------------------------------------------------- 1 | CXX := c++ 2 | LD := $(CXX) 3 | SED := sed 4 | TBLGEN := llvm-tblgen 5 | 6 | override CPPFLAGS := -DGET_INSTRINFO_ENUM -DGET_REGINFO_ENUM $(CPPFLAGS) 7 | override CXXFLAGS := -std=c++17 -O2 -fpic -fno-rtti -Wall -Wextra -Wpedantic $(CXXFLAGS) 8 | override LDLIBS := -lLLVM-13 -ldl $(LDLIBS) 9 | override TBLGENFLAGS := -I$(LLVM_SOURCE_DIR)/include $(TBLGENFLAGS) 10 | 11 | libingerc.so: X86InstrInfo.h X86RegisterInfo.h 12 | 13 | X86InstrInfo.h: $(LLVM_SOURCE_DIR)/lib/Target/X86/X86.td 14 | $(TBLGEN) -gen-instr-info $(TBLGENFLAGS) -I$(dir $<) -o $@ $< 15 | 16 | X86RegisterInfo.h: $(LLVM_SOURCE_DIR)/lib/Target/X86/X86.td 17 | $(TBLGEN) -gen-register-info $(TBLGENFLAGS) -I$(dir $<) -o $@ $< 18 | $(SED) -i 's/ : \S\+\( {\)/\1/' $@ 19 | 20 | .PHONY: clean 21 | clean: 22 | $(RM) libingerc.*o 23 | 24 | lib%.so: %.o 25 | $(LD) $(LDFLAGS) -shared -zdefs -ztext -o $@ $^ $(LDLIBS) 26 | -------------------------------------------------------------------------------- /compiler/ingerc: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Skip the following LLVM optimization passes, which cause problems for us. 4 | readonly BADPASSES="function-attrs inline prune-eh" 5 | 6 | epiloguefun="" 7 | epiloguevar="" 8 | if [ "$1" = "-e" -a "$#" -ge "2" ] 9 | then 10 | epiloguefun="$2" 11 | epiloguevar="INGERC_EPILOGUE=$epiloguefun" 12 | shift 2 13 | fi 14 | 15 | # Temporarily drop any --emit or -emit because it would prevent us from getting LLVM debug output. 16 | args="`printf "'%s' " "$@" | sed "s/'-\?-emit\(' '\|=\)[^']\+'//"`" 17 | 18 | finalstage="`echo "$*" | sed -n 's/.*-\?-emit[ =]\(\S\+\).*/\1/p'`" 19 | crate="`eval rustc $args --print crate-name 2>/dev/null`" 20 | if [ "$?" -ne "0" ] 21 | then 22 | echo "USAGE: $0 [-e ] [args]... " 23 | echo 24 | echo "If the -e switch is provided, it must come first. These other switches are supported:" 25 | echo "\t--emit " 26 | echo "\trustc's usual options..." 27 | echo 28 | exec rustc --help 29 | fi 30 | 31 | set -e 32 | 33 | # These -Ccodegen-units=1 switches prevent unreadable output due to interleaving. 34 | allpasses="`eval rustc $args -Cllvm-args=-debug-pass=Arguments -Ccodegen-units=1 2>&1`" 35 | autopasses="`eval rustc $args -Cllvm-args=-debug-pass=Arguments -Cno-prepopulate-passes -Ccodegen-units=1 2>&1`" 36 | rm -f "$crate" 37 | 38 | # Each line is a grep-compatible command-line switch. 39 | autoswitches="`echo "$autopasses" | sed -e's/^/-e"/' -e's/: \+/: \\\\+/g' -e's/$/"/'`" 40 | 41 | manualpasses="`echo "$allpasses" | eval grep -vx $autoswitches`" 42 | 43 | # Each line is a pass name, without the leading hyphen. 44 | manualnames="`echo "$manualpasses" | sed -e's/[^-]\+-//' -e's/ -/\n/g'`" 45 | badnames="`echo "$BADPASSES" | tr " " "\n"`" 46 | 47 | # Each line is a one-word grep-compatible command-line switch. 48 | manualswitches="`echo "$badnames" | sed -e's/^/-e/'`" 49 | 50 | passnames="`echo "$manualnames" | grep -vxF $manualswitches`" 51 | 52 | dir="`dirname "$0"`" 53 | std="`LD_TRACE_LOADED_OBJECTS= rustc | grep -o 'std-[^.]\+' | uniq`" 54 | eval rustc $args --emit llvm-ir -Cno-prepopulate-passes -Cpasses="'$passnames'" 55 | if [ "$finalstage" = "llvm-ir" ] 56 | then 57 | exit 0 58 | fi 59 | 60 | "$dir/ingerc.ts" "$crate.ll" $epiloguefun 61 | if [ "$finalstage" = "ir" ] 62 | then 63 | exit 0 64 | fi 65 | 66 | eval LD_PRELOAD="'$dir/libingerc.so'" "$epiloguevar" llc "'$crate.ll'" >/dev/null 67 | if [ "$finalstage" = "asm" ] 68 | then 69 | exit 0 70 | fi 71 | 72 | linkargs="`printf "'%s' " "$@" | sed -e"s/\('-C\)' '\(link-arg=\)/\1\2/g" -e"s/' '/'\n'/g" | sed -n "s/^'-Clink-arg=\(.\+\)'$/\1/p"`" 73 | eval cc -no-pie -Wa,-gdwarf-5 -o "'$crate'" "'$crate.s'" $linkargs "'-l$std'" 74 | -------------------------------------------------------------------------------- /compiler/ingerc.cpp: -------------------------------------------------------------------------------- 1 | #include "X86InstrInfo.h" 2 | #include "X86RegisterInfo.h" 3 | 4 | #include "llvm/CodeGen/TargetInstrInfo.h" 5 | #include "llvm/IR/LegacyPassManager.h" 6 | #include "llvm/IR/Module.h" 7 | #include "llvm/MC/MCContext.h" 8 | 9 | #include 10 | #include 11 | 12 | using llvm::legacy::PassManager; 13 | using llvm::DebugLoc; 14 | using llvm::Function; 15 | using llvm::FunctionType; 16 | using llvm::GlobalValue; 17 | using llvm::MachineBasicBlock; 18 | using llvm::MachineFunction; 19 | using llvm::MachineFunctionPass; 20 | using llvm::MachineInstr; 21 | using llvm::MachineInstrBuilder; 22 | using llvm::MachineInstrBundleIterator; 23 | using llvm::MCContext; 24 | using llvm::MCSymbol; 25 | using llvm::Pass; 26 | using llvm::PassInfo; 27 | using llvm::PassRegistry; 28 | using llvm::PointerType; 29 | using llvm::SmallVector; 30 | using llvm::Type; 31 | using llvm::callDefaultCtor; 32 | using llvm::outs; 33 | 34 | namespace { 35 | struct IngerCancel: public MachineFunctionPass { 36 | virtual bool runOnMachineFunction(MachineFunction &mf) override { 37 | outs() << "FUNCTION: " << mf.getName() << '\n'; 38 | 39 | auto changed = false; 40 | auto &pads = mf.getLandingPads(); 41 | if(getEpilogueFunction()) { 42 | auto epilogue = findInst( 43 | mf, 44 | std::bind(isCallTo, std::placeholders::_1, [](auto *fun) { 45 | return fun && fun->getName() == getEpilogueFunction(); 46 | }) 47 | ); 48 | if(epilogue) { 49 | auto *cfi = findFirstPad(mf); 50 | if(!cfi) { 51 | cfi = pads.front().LandingPadBlock->getPrevNode(); 52 | while(!cfi->front().isCFIInstruction()) 53 | cfi = cfi->getPrevNode(); 54 | } 55 | 56 | auto index = cfi->front().getOperand(0).getCFIIndex(); 57 | auto off = mf.getFrameInstructions()[index].getOffset(); 58 | addInst( 59 | *(*epilogue)->getParent(), 60 | *epilogue, 61 | llvm::X86::LEA64r, 62 | [&off](auto &inst, auto &) { 63 | inst.addReg(llvm::X86::RDI); 64 | 65 | // off(%rsp), see X86InstrBuilder.h:addRegOffset() 66 | inst.addReg(llvm::X86::RSP); 67 | inst.addImm(1); 68 | inst.addReg(0); 69 | inst.addImm(off - 8); 70 | inst.addReg(0); 71 | } 72 | ); 73 | ++*epilogue; 74 | addInst( 75 | *(*epilogue)->getParent(), 76 | *epilogue, 77 | llvm::X86::NOOP, 78 | [](auto &, auto &) {} 79 | ); 80 | changed = true; 81 | } 82 | } 83 | 84 | if(mf.getName().contains("drop_in_place")) 85 | return changed; 86 | 87 | for(auto &pad : pads) { 88 | auto &cleanupBlock = *pad.LandingPadBlock; 89 | outs() << "landing pad: " << *pad.LandingPadLabel << '\n'; 90 | 91 | auto dropCall = std::find_if( 92 | cleanupBlock.begin(), 93 | cleanupBlock.end(), 94 | std::bind(isCallTo, std::placeholders::_1, [](auto *fun) { 95 | return fun && fun->getName().contains("drop_in_place"); 96 | }) 97 | ); 98 | 99 | auto *beginBlock = &mf.front(); 100 | auto beginLoc = beginBlock->begin(); 101 | 102 | auto endIt = mf.end(); 103 | --endIt; 104 | if(&*endIt == &cleanupBlock) 105 | --endIt; 106 | 107 | auto *endBlock = &*endIt; 108 | auto endLoc = endBlock->end(); 109 | 110 | if(dropCall == cleanupBlock.end()) { 111 | auto *sRetType = getFunctionSRetType(mf.getFunction()); 112 | const Function *dtor = nullptr; 113 | if(sRetType) 114 | dtor = findDtor(*sRetType, mf); 115 | if(dtor) { 116 | auto argIndex = -1ul; 117 | auto ctor = findInst(mf, [&sRetType, &argIndex](auto &each) { 118 | auto *fun = getFunction(each); 119 | if(!fun) 120 | return false; 121 | return getFunctionSRetType(*fun, &argIndex) == sRetType; 122 | }); 123 | assert(ctor); 124 | beginBlock = endBlock = (*ctor)->getParent(); 125 | ++*ctor; 126 | beginLoc = endLoc = *ctor; 127 | 128 | // Find the mov or lea before the constructor call. 129 | assert(argIndex == 0); 130 | --*ctor; 131 | --*ctor; 132 | while((*ctor)->getOperand(0).getReg() != llvm::X86::RDI) { 133 | assert(*ctor != (*ctor)->getParent()->begin()); 134 | --*ctor; 135 | } 136 | 137 | auto move = cleanupBlock.begin(); 138 | while(move != cleanupBlock.end() && !move->isMoveReg()) 139 | ++move; 140 | assert(move != cleanupBlock.end()); 141 | 142 | // mov %rax, %rbp ; Save _Unwind_Context pointer. 143 | move->getOperand(0).setReg(llvm::X86::RBP); 144 | 145 | auto unwind = move; 146 | while(unwind != cleanupBlock.end() && !unwind->isCall()) 147 | ++unwind; 148 | assert(unwind != cleanupBlock.end()); 149 | 150 | // (mov|lea) ???, %rdi ; Pass victim to destructor. 151 | addInst( 152 | cleanupBlock, 153 | unwind, 154 | (*ctor)->getOpcode(), 155 | [&ctor](auto &inst, auto &) { 156 | inst.addReg(llvm::X86::RDI); 157 | 158 | auto &src = (*ctor)->getOperand(1); 159 | if(src.isReg()) 160 | inst.addReg(src.getReg()); 161 | else 162 | inst.cloneMemRefs(**ctor); 163 | } 164 | ); 165 | 166 | dropCall = addInst( 167 | cleanupBlock, 168 | unwind, 169 | unwind->getOpcode(), 170 | [&dtor](auto &inst, auto &) { 171 | inst.addGlobalAddress(dtor); 172 | } 173 | ); 174 | 175 | // mov %rbp, %rdi ; Pass _Unwind_Context pointer. 176 | addInst( 177 | cleanupBlock, 178 | unwind, 179 | move->getOpcode(), 180 | [](auto &inst, auto &) { 181 | inst.addReg(llvm::X86::RDI); 182 | inst.addReg(llvm::X86::RBP); 183 | } 184 | ); 185 | } 186 | } 187 | 188 | if(dropCall != cleanupBlock.end()) { 189 | auto &beginLabel = getOrCreateLabel( 190 | mf.getOrCreateLandingPadInfo(&cleanupBlock).BeginLabels, 191 | *beginBlock, 192 | beginLoc, 193 | changed 194 | ); 195 | 196 | auto &endLabel = getOrCreateLabel( 197 | mf.getOrCreateLandingPadInfo(&cleanupBlock).EndLabels, 198 | *endBlock, 199 | endLoc, 200 | changed 201 | ); 202 | outs() << "bounding labels: " << beginLabel << ' ' << endLabel << '\n'; 203 | 204 | auto &dropFun = *dropCall->getOperand(0).getGlobal(); 205 | outs() << "dropFun: " << dropFun.getName() << '\n'; 206 | 207 | auto &dropType = *getFunctionType(*dropCall).params().front(); 208 | outs() << "paramType: " << dropType << '\n'; 209 | 210 | auto labelFinder = [](auto &label) { 211 | return [&label](auto &each) { 212 | return std::any_of( 213 | each.operands_begin(), 214 | each.operands_end(), 215 | [&label](auto &each) { 216 | return each.isMCSymbol() 217 | && each.getMCSymbol() == &label; 218 | } 219 | ); 220 | }; 221 | }; 222 | 223 | auto beginInst = findInst(mf, labelFinder(beginLabel)); 224 | assert(beginInst); 225 | outs() << "beginInst: " << **beginInst << '\n'; 226 | 227 | auto endInst = findInst(mf, labelFinder(endLabel)); 228 | assert(endInst); 229 | outs() << "endInst: " << **endInst << '\n'; 230 | 231 | auto prevInst = (*beginInst)->getPrevNode(); 232 | assert(prevInst); 233 | if(!isCallUsing(*prevInst, dropType)) { 234 | auto nextInst = this->nextInst(*beginInst); 235 | while( 236 | nextInst 237 | && !(*nextInst)->isCall() 238 | && *nextInst != endInst 239 | ) 240 | nextInst = this->nextInst(*nextInst); 241 | if(nextInst && isCallUsing(**nextInst, dropType)) { 242 | outs() << "nextInst: " << **nextInst << '\n'; 243 | 244 | auto *movedInst = (*beginInst)->removeFromParent(); 245 | (*nextInst)->getParent()->insertAfter( 246 | *nextInst, 247 | movedInst 248 | ); 249 | changed = true; 250 | } 251 | } 252 | 253 | auto nextInst = this->nextInst(*endInst); 254 | while( 255 | nextInst 256 | && !isCallTo(**nextInst, [&dropFun](auto *fun) { 257 | return fun == &dropFun; 258 | }) 259 | && !(*nextInst)->isCFIInstruction() 260 | && !(*nextInst)->isReturn() 261 | ) 262 | nextInst = this->nextInst(*nextInst); 263 | assert(nextInst); 264 | outs() << "nextInst: " << **nextInst << '\n'; 265 | 266 | if((*nextInst)->isCFIInstruction()) 267 | --*nextInst; 268 | 269 | auto *movedInst = (*endInst)->removeFromParent(); 270 | (*nextInst)->getParent()->insert(*nextInst, movedInst); 271 | changed = true; 272 | } 273 | } 274 | 275 | return changed; 276 | } 277 | 278 | static char ID; 279 | IngerCancel(): MachineFunctionPass(ID) {} 280 | 281 | private: 282 | static MachineInstr &addInst( 283 | MachineBasicBlock &block, 284 | MachineInstrBundleIterator pos, 285 | unsigned opcode, 286 | std::function operands 287 | ) { 288 | auto &mf = *block.getParent(); 289 | auto &info = mf.getSubtarget().getInstrInfo()->get(opcode); 290 | auto &inst = *mf.CreateMachineInstr(info, DebugLoc()); 291 | MachineInstrBuilder build(mf, inst); 292 | operands(build, mf.getContext()); 293 | block.insert(pos, &inst); 294 | return inst; 295 | } 296 | 297 | static const Function *findDtor(const Type &type, const MachineFunction &fun) { 298 | for(auto &defn : *fun.getFunction().getParent()) { 299 | if(defn.getName().contains("drop_in_place")) { 300 | const auto ¶ms = defn.getFunctionType()->params(); 301 | if( 302 | params.size() 303 | && params.front()->isPointerTy() 304 | && static_cast(params.front())->getElementType() == &type 305 | ) 306 | return &defn; 307 | } 308 | } 309 | return nullptr; 310 | } 311 | 312 | static MachineBasicBlock *findFirstPad(MachineFunction &fun) { 313 | for(auto &pad : fun.getLandingPads()) { 314 | auto &block = *pad.LandingPadBlock; 315 | if(block.front().isCFIInstruction()) 316 | return █ 317 | } 318 | return nullptr; 319 | } 320 | 321 | static std::optional> findInst( 322 | MachineFunction &mf, 323 | std::function pred 324 | ) { 325 | for(auto &block : mf) { 326 | auto inst = std::find_if(block.begin(), block.end(), pred); 327 | if(inst != block.end()) 328 | return std::optional(inst); 329 | } 330 | return {}; 331 | } 332 | 333 | static const char *getEpilogueFunction() { 334 | static const char *epilogue = nullptr; 335 | static bool memo = false; 336 | if(!memo) { 337 | epilogue = getenv("INGERC_EPILOGUE"); 338 | memo = true; 339 | } 340 | return epilogue; 341 | } 342 | 343 | static const Function *getFunction(const MachineInstr &call) { 344 | if(!call.isCall()) 345 | return nullptr; 346 | 347 | return static_cast(call.getOperand(0).getGlobal()); 348 | } 349 | 350 | static const Type *getFunctionSRetType(const Function &fun, size_t *param = nullptr) { 351 | auto params = fun.getFunctionType()->params(); 352 | for(auto index = 0u; index != params.size(); ++index) { 353 | auto *type = fun.getParamStructRetType(index); 354 | if(type) { 355 | if(param) 356 | *param = index; 357 | return type; 358 | } 359 | } 360 | return nullptr; 361 | } 362 | 363 | static const FunctionType &getFunctionType(const MachineInstr &call) { 364 | assert(call.isCall()); 365 | 366 | auto &fun = *call.getOperand(0).getGlobal(); 367 | auto &funType = *fun.getType()->getElementType(); 368 | assert(funType.isFunctionTy()); 369 | 370 | return static_cast(funType); 371 | } 372 | 373 | static MCSymbol &getOrCreateLabel( 374 | SmallVector &labels, 375 | MachineBasicBlock &block, 376 | MachineInstrBundleIterator pos, 377 | bool &changed 378 | ) { 379 | if(labels.size()) 380 | return *labels.front(); 381 | 382 | auto &inst = addInst( 383 | block, 384 | pos, 385 | llvm::TargetOpcode::EH_LABEL, 386 | [](auto &inst, auto &syms) { 387 | inst.addSym(syms.createTempSymbol()); 388 | } 389 | ); 390 | auto &label = *inst.getOperand(0).getMCSymbol(); 391 | labels.push_back(&label); 392 | changed = true; 393 | return label; 394 | } 395 | 396 | static bool isCallTo( 397 | const MachineInstr &inst, 398 | std::function name 399 | ) { 400 | if(!inst.isCall()) 401 | return false; 402 | 403 | auto &operand = inst.getOperand(0); 404 | const GlobalValue *fun = nullptr; 405 | if(operand.isGlobal()) 406 | fun = operand.getGlobal(); 407 | // else it's an indirect call so we cannot know the function statically 408 | return name(fun); 409 | } 410 | 411 | static bool isCallUsing(const MachineInstr &inst, const Type &type) { 412 | if(!inst.isCall()) 413 | return false; 414 | 415 | auto &funType = getFunctionType(inst); 416 | return funType.getReturnType() == &type 417 | || std::any_of( 418 | funType.param_begin(), 419 | funType.param_end(), 420 | [&type](auto &each) { 421 | return each == &type; 422 | } 423 | ); 424 | } 425 | 426 | static std::optional> nextInst( 427 | MachineInstrBundleIterator inst 428 | ) { 429 | auto &block = *inst->getParent(); 430 | auto nextInst = inst; 431 | ++nextInst; 432 | if(nextInst != block.end()) 433 | return std::optional(nextInst); 434 | 435 | auto *nextBlock = inst->getParent()->getNextNode(); 436 | if(nextBlock) 437 | return std::optional(nextBlock->begin()); 438 | 439 | return {}; 440 | } 441 | }; 442 | } 443 | 444 | char IngerCancel::ID; 445 | 446 | typedef void (*PassManager_add_t)(PassManager *, Pass *); 447 | static PassManager_add_t _ZN4llvm6legacy11PassManager3addEPNS_4PassE; 448 | 449 | static void add(PassManager *pm, Pass *p) { 450 | if(p->getPassName() == "X86 Assembly Printer") 451 | _ZN4llvm6legacy11PassManager3addEPNS_4PassE(pm, new IngerCancel()); 452 | _ZN4llvm6legacy11PassManager3addEPNS_4PassE(pm, p); 453 | } 454 | 455 | extern "C" { 456 | PassManager_add_t _ZTVN4llvm6legacy11PassManagerE[5]; 457 | } 458 | 459 | namespace llvm { 460 | void initializeIngerCancelPass(PassRegistry &); 461 | } 462 | 463 | extern "C" void LLVMInitializeX86Target() { 464 | void (*LLVMInitializeX86Target)() = (void (*)()) dlsym(RTLD_NEXT, "LLVMInitializeX86Target"); 465 | LLVMInitializeX86Target(); 466 | 467 | PassRegistry &pr = *PassRegistry::getPassRegistry(); 468 | initializeIngerCancelPass(pr); 469 | 470 | _ZN4llvm6legacy11PassManager3addEPNS_4PassE = (PassManager_add_t) 471 | dlsym(RTLD_NEXT, "_ZN4llvm6legacy11PassManager3addEPNS_4PassE"); 472 | const PassManager_add_t *vtable = (PassManager_add_t *) 473 | dlsym(RTLD_NEXT, "_ZTVN4llvm6legacy11PassManagerE"); 474 | for( 475 | size_t index = 0; 476 | index != sizeof _ZTVN4llvm6legacy11PassManagerE / sizeof *_ZTVN4llvm6legacy11PassManagerE; 477 | ++index 478 | ) 479 | if(vtable[index] == _ZN4llvm6legacy11PassManager3addEPNS_4PassE) 480 | _ZTVN4llvm6legacy11PassManagerE[index] = add; 481 | else 482 | _ZTVN4llvm6legacy11PassManagerE[index] = vtable[index]; 483 | } 484 | 485 | INITIALIZE_PASS(IngerCancel, "llc", "IngerCancel", false, false) 486 | -------------------------------------------------------------------------------- /compiler/ingerc.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S deno run --allow-read --allow-write 2 | 3 | const EPILOGUE = 'llvm.donothing'; 4 | 5 | function processDefine(fun: string[]): string[] { 6 | let personality = Boolean(fun[0].match(/\bpersonality\b/)); 7 | if(!personality) 8 | fun[0] = fun[0].replace( 9 | /(!dbg .+)?{$/, 10 | 'personality i32 (' 11 | + 'i32, ' 12 | + 'i32, ' 13 | + 'i64, ' 14 | + '%"unwind::libunwind::_Unwind_Exception"*, ' 15 | + '%"unwind::libunwind::_Unwind_Context"*' 16 | + ')* @rust_eh_personality $&', 17 | ); 18 | 19 | let cleanup = false; 20 | for(let line = 0; line < fun.length; ++line) 21 | if(fun[line].match(/^ ret\b/)) { 22 | const label = 'ingerc' + line; 23 | fun.splice( 24 | line, 25 | 0, 26 | ' invoke void @"' 27 | + epilogue 28 | + '"() to label %' 29 | + label 30 | + ' unwind label %cleanup', 31 | label + ':', 32 | ); 33 | line += 2; 34 | } else if(fun[line].match(/^cleanup:/)) 35 | cleanup = true; 36 | 37 | if(!cleanup) 38 | fun.splice( 39 | fun.length - 1, 40 | 0, 41 | 'cleanup:', 42 | ' %ingerc = landingpad { i8*, i32 } cleanup', 43 | ' resume { i8*, i32 } %ingerc', 44 | ); 45 | 46 | return fun; 47 | } 48 | 49 | let epilogue = EPILOGUE; 50 | const args = Deno.args.slice(); 51 | if(args.length == 2) 52 | epilogue = args.pop()!; 53 | 54 | if(args.length != 1) { 55 | console.log( 56 | 'USAGE: ' + import.meta.url.replace(/.*\//, '') + ' [epilogue function]\n' 57 | + '\n' 58 | + 'Modify to force llc to generate an LSDA for each function, even\n' 59 | + 'those that statically cannot raise exceptions.' 60 | + 'With [epilogue function], notify the runtime of epilogue entry by invoking the\n' 61 | + 'named function.' 62 | ); 63 | Deno.exit(1); 64 | } 65 | 66 | const filename = args[0]; 67 | let ll = new TextDecoder().decode(Deno.readFileSync(filename)); 68 | let define = '\ndeclare void @"' + epilogue + '"()'; 69 | if(ll.includes(define)) { 70 | console.log('We\'ve already processed this file! Leaving it unchanged.'); 71 | Deno.exit(2); 72 | } 73 | 74 | const funs = ll.split('\ndefine ').flatMap(function(elem) { 75 | const [head, tail] = elem.split('\n}\n'); 76 | if(tail) 77 | return [('define ' + head + '\n}').split('\n'), tail]; 78 | else 79 | return [head]; 80 | }); 81 | 82 | ll = funs.map(function(elem) { 83 | if(!Array.isArray(elem)) 84 | return elem; 85 | 86 | return '\n' + processDefine(elem).join('\n') + '\n'; 87 | }).join(''); 88 | ll += define; 89 | Deno.writeFileSync(filename, new TextEncoder().encode(ll)); 90 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | readonly PIPE="/tmp/configure" 4 | 5 | version() { 6 | echo -n "Looking for $1..." 7 | 8 | local version 9 | echo -n " broken installation" >"$PIPE.err" 10 | version="`"$1" --version 2>>"$PIPE.err" | grep -o '[[:digit:].]\+[[:digit:]]'`" 11 | rm "$PIPE.err" 12 | echo " found `echo "$version" | head -n1`" 13 | } 14 | 15 | helper() { 16 | local file="$1" 17 | local color="$2" 18 | printf %s "[${color}m" >&2 19 | tr -d '\n' <"$file" | sed 's/.*://;a\' >&2 20 | printf %s "" >&2 21 | rm "$file" 22 | } 23 | 24 | handler() { 25 | [ -e "$PIPE.err" ] && helper "$PIPE.err" 31 26 | [ -e "$PIPE.warn" ] && helper "$PIPE.warn" 93 27 | } 28 | 29 | set -e 30 | trap handler EXIT 31 | 32 | version cargo 33 | version rustc 34 | 35 | echo -n "Checking Rust 2018 support..." 36 | echo " not present" >"$PIPE.err" 37 | rustc --help -v | grep -- --edition >/dev/null 38 | echo " present" 39 | 40 | version make 41 | 42 | echo -n "Checking GNU Make language support..." 43 | echo " not present" >"$PIPE.err" 44 | make --version | grep GNU >/dev/null 45 | echo " present" 46 | 47 | version cc 48 | version bindgen 49 | version rustfmt 50 | version ld 51 | version nm 52 | version objcopy 53 | version git 54 | 55 | dir="`dirname "$0"`" 56 | 57 | git() { 58 | "`which git`" -C "$dir" "$@" 59 | } 60 | 61 | cargo() { 62 | local owd="$PWD" 63 | cd "$dir" 64 | "`which cargo`" "$@" 65 | cd "$owd" 66 | } 67 | 68 | echo -n "Checking submodules..." 69 | if git submodule status | grep '^-' >/dev/null 70 | then 71 | echo " initializing" 72 | git submodule update --init --recursive 73 | elif git submodule status | grep '^+' >/dev/null 74 | then 75 | echo " inconsistent" >"$PIPE.warn" 76 | handler 77 | else 78 | echo " consistent" 79 | fi 80 | 81 | echo -n "Configuring toolchain..." 82 | if [ -e "$dir/.cargo/config" ] 83 | then 84 | echo " skipping" 85 | else 86 | cat >"$dir/.cargo/config" <<-tac 87 | `cat "$dir/.cargo/config.toml"` 88 | rustc = ".cargo/rustc" 89 | tac 90 | echo " generated" 91 | fi 92 | 93 | echo -n "Cleaning libinger build..." 94 | cargo clean 95 | echo " done" 96 | 97 | echo -n "Cleaning libgotcha build..." 98 | make -C"$dir/external/libgotcha" clean >/dev/null 99 | echo " done" 100 | 101 | echo -n "Removing lockfile..." 102 | rm -f "$dir/Cargo.lock" 103 | echo " done" 104 | 105 | echo -n "Detecting interpreter..." 106 | if [ -n "$LIBINGER_LINKER" ] 107 | then 108 | file -L "$LIBINGER_LINKER" 2>&1 | tee "$PIPE.err" | grep '\' >/dev/null 109 | echo " using $LIBINGER_LINKER" 110 | elif [ -e "$dir/ld.so" ] 111 | then 112 | echo " using `readlink "$dir/ld.so"`" 113 | else 114 | echo " define LIBINGER_LINKER to a dynamic linker or place one at ./ld.so" >"$PIPE.err" 115 | false 116 | fi 117 | -------------------------------------------------------------------------------- /internal/libsignal/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "signal" 3 | version = "0.0.0" 4 | edition = "2018" 5 | 6 | [dependencies] 7 | libc = "*" 8 | 9 | [features] 10 | libgotcha = [] 11 | -------------------------------------------------------------------------------- /internal/libsignal/src/lib.rs: -------------------------------------------------------------------------------- 1 | mod libgotcha; 2 | pub mod pthread; 3 | 4 | use libc::SIG_BLOCK; 5 | use libc::SIG_SETMASK; 6 | use libc::SIG_UNBLOCK; 7 | use libc::SIGABRT; 8 | use libc::SIGALRM; 9 | use libc::SIGBUS; 10 | use libc::SIGCHLD; 11 | use libc::SIGCONT; 12 | use libc::SIGFPE; 13 | use libc::SIGHUP; 14 | use libc::SIGILL; 15 | use libc::SIGINT; 16 | use libc::SIGKILL; 17 | use libc::SIGPIPE; 18 | use libc::SIGPROF; 19 | use libc::SIGPOLL; 20 | use libc::SIGPWR; 21 | use libc::SIGQUIT; 22 | use libc::SIGSEGV; 23 | use libc::SIGSTKFLT; 24 | use libc::SIGSYS; 25 | use libc::SIGTERM; 26 | use libc::SIGTRAP; 27 | use libc::SIGTSTP; 28 | use libc::SIGTTIN; 29 | use libc::SIGTTOU; 30 | use libc::SIGURG; 31 | use libc::SIGUSR1; 32 | use libc::SIGUSR2; 33 | use libc::SIGVTALRM; 34 | use libc::SIGWINCH; 35 | use libc::SIGXCPU; 36 | use libc::SIGXFSZ; 37 | use libc::ucontext_t; 38 | use std::io::Error; 39 | use std::ptr::null_mut; 40 | pub use libc::sigaction as Sigaction; 41 | pub use libc::siginfo_t; 42 | pub use libc::sigset_t as Sigset; 43 | use std::io::Result; 44 | use std::os::raw::c_int; 45 | 46 | pub type Handler = extern "C" fn(Signal, Option<&siginfo_t>, Option<&mut ucontext_t>); 47 | 48 | #[allow(dead_code)] 49 | #[repr(i32)] 50 | pub enum Operation { 51 | Block = SIG_BLOCK, 52 | Unblock = SIG_UNBLOCK, 53 | SetMask = SIG_SETMASK, 54 | } 55 | 56 | #[allow(dead_code)] 57 | #[derive(Clone)] 58 | #[derive(Copy)] 59 | #[repr(i32)] 60 | pub enum Signal { 61 | Abort = SIGABRT, 62 | Alarm = SIGALRM, 63 | Bus = SIGBUS, 64 | Breakpoint = SIGTRAP, 65 | Child = SIGCHLD, 66 | Continue = SIGCONT, 67 | Coprocessor = SIGSTKFLT, 68 | FilesystemLimit = SIGXFSZ, 69 | FloatingPoint = SIGFPE, 70 | Hangup = SIGHUP, 71 | Illegal = SIGILL, 72 | Interrupt = SIGINT, 73 | Kill = SIGKILL, 74 | Pipe = SIGPIPE, 75 | Pollable = SIGPOLL, 76 | ProfilingTimer = SIGPROF, 77 | Quit = SIGQUIT, 78 | Segfault = SIGSEGV, 79 | Syscall = SIGSYS, 80 | TerminalInput = SIGTTIN, 81 | TerminalOutput = SIGTTOU, 82 | TerminalStop = SIGTSTP, 83 | Terminate = SIGTERM, 84 | PowerFailure = SIGPWR, 85 | ProcessorLimit = SIGXCPU, 86 | UrgentSocket = SIGURG, 87 | User1 = SIGUSR1, 88 | User2 = SIGUSR2, 89 | VirtualAlarm = SIGVTALRM, 90 | WindowResize = SIGWINCH, 91 | } 92 | 93 | impl PartialEq for Signal { 94 | fn eq(&self, other: &Self) -> bool { 95 | *self as c_int == *other as c_int 96 | } 97 | } 98 | impl Eq for Signal {} 99 | 100 | pub trait Set { 101 | fn empty() -> Self; 102 | fn full() -> Self; 103 | fn add(&mut self, _: Signal); 104 | fn del(&mut self, _: Signal); 105 | fn has(&self, _: Signal) -> bool; 106 | } 107 | 108 | fn sigset(fun: fn(&mut Sigset)) -> Sigset { 109 | use std::mem::zeroed; 110 | 111 | let mut my = unsafe { 112 | zeroed() 113 | }; 114 | fun(&mut my); 115 | my 116 | } 117 | 118 | impl Set for Sigset { 119 | fn empty() -> Self { 120 | use libc::sigemptyset; 121 | sigset(|me| unsafe { sigemptyset(me); }) 122 | } 123 | 124 | fn full() -> Self { 125 | use libc::sigfillset; 126 | sigset(|me| unsafe { sigfillset(me); }) 127 | } 128 | 129 | fn add(&mut self, signal: Signal) { 130 | use libc::sigaddset; 131 | unsafe { 132 | sigaddset(self, signal as _); 133 | } 134 | } 135 | 136 | fn del(&mut self, signal: Signal) { 137 | use libc::sigdelset; 138 | unsafe { 139 | sigdelset(self, signal as _); 140 | } 141 | } 142 | 143 | fn has(&self, signal: Signal) -> bool { 144 | use libc::sigismember; 145 | unsafe { 146 | sigismember(self, signal as _) != 0 147 | } 148 | } 149 | } 150 | 151 | pub trait Action { 152 | fn new(_: Handler, _: Sigset, _: c_int) -> Self; 153 | fn sa_sigaction(&self) -> &Handler; 154 | fn sa_sigaction_mut(&mut self) -> &mut Handler; 155 | } 156 | 157 | impl Action for Sigaction { 158 | fn new(sigaction: Handler, mask: Sigset, flags: c_int) -> Self { 159 | Self { 160 | sa_sigaction: sigaction as _, 161 | sa_mask: mask, 162 | sa_flags: flags, 163 | sa_restorer: None, 164 | } 165 | } 166 | 167 | fn sa_sigaction(&self) -> &Handler { 168 | use std::mem::transmute; 169 | 170 | unsafe { 171 | transmute(self.sa_sigaction) 172 | } 173 | } 174 | 175 | fn sa_sigaction_mut(&mut self) -> &mut Handler { 176 | use std::mem::transmute; 177 | 178 | unsafe { 179 | transmute(self.sa_sigaction) 180 | } 181 | } 182 | } 183 | 184 | pub trait Actionable { 185 | fn maybe(&self) -> Option<&Sigaction>; 186 | } 187 | 188 | impl Actionable for Sigaction { 189 | fn maybe(&self) -> Option<&Self> { 190 | Some(self) 191 | } 192 | } 193 | 194 | impl Actionable for () { 195 | fn maybe(&self) -> Option<&Sigaction> { 196 | None 197 | } 198 | } 199 | 200 | pub fn sigaction(signal: Signal, new: &dyn Actionable, old: Option<&mut Sigaction>) -> Result<()> { 201 | use libc::sigaction; 202 | 203 | if unsafe { 204 | sigaction( 205 | signal as c_int, 206 | if let Some(new) = new.maybe() { new } else { null_mut() }, 207 | if let Some(old) = old { old } else { null_mut() }, 208 | ) 209 | } == 0 { 210 | Ok(()) 211 | } else { 212 | Err(Error::last_os_error()) 213 | } 214 | } 215 | 216 | pub fn sigprocmask(how: Operation, new: &Sigset, old: Option<&mut Sigset>) -> Result<()> { 217 | pthread_sigmask(how, new, old) 218 | } 219 | 220 | pub fn pthread_sigmask(how: Operation, new: &Sigset, old: Option<&mut Sigset>) -> Result<()> { 221 | use libc::pthread_sigmask; 222 | 223 | let code = unsafe { 224 | pthread_sigmask(how as c_int, new, if let Some(old) = old { old } else { null_mut() }) 225 | }; 226 | if code == 0 { 227 | Ok(()) 228 | } else { 229 | Err(Error::from_raw_os_error(code)) 230 | } 231 | } 232 | 233 | #[cfg(test)] 234 | mod tests { 235 | use super::*; 236 | 237 | use pthread::pthread_kill; 238 | use pthread::pthread_self; 239 | 240 | #[test] 241 | fn sigaction_usr1() { 242 | use std::sync::atomic::AtomicBool; 243 | use std::sync::atomic::Ordering; 244 | 245 | thread_local! { 246 | static RAN: AtomicBool = AtomicBool::new(false); 247 | } 248 | 249 | extern "C" fn handler(signum: Signal, _: Option<&siginfo_t>, _: Option<&mut ucontext_t>) { 250 | RAN.with(|ran| ran.store(signum == Signal::User1, Ordering::Relaxed)); 251 | } 252 | 253 | let conf = Sigaction::new(handler, Sigset::empty(), 0); 254 | sigaction(Signal::User1, &conf, None).unwrap(); 255 | 256 | pthread_kill(pthread_self(), Signal::User1).unwrap(); 257 | 258 | assert!(RAN.with(|ran| ran.load(Ordering::Relaxed))); 259 | } 260 | 261 | #[test] 262 | fn sigprocmask_usr2() { 263 | use libc::sigsuspend; 264 | use std::sync::atomic::AtomicBool; 265 | use std::sync::atomic::Ordering; 266 | 267 | thread_local! { 268 | static RAN: AtomicBool = AtomicBool::new(false); 269 | } 270 | 271 | extern "C" fn handler(signum: Signal, _: Option<&siginfo_t>, _: Option<&mut ucontext_t>) { 272 | RAN.with(|ran| ran.store(signum == Signal::User2, Ordering::Relaxed)); 273 | } 274 | 275 | let mut mask = Sigset::empty(); 276 | mask.add(Signal::User2); 277 | sigprocmask(Operation::Block, &mask, None).unwrap(); 278 | 279 | let conf = Sigaction::new(handler, Sigset::empty(), 0); 280 | sigaction(Signal::User2, &conf, None).unwrap(); 281 | 282 | pthread_kill(pthread_self(), Signal::User2).unwrap(); 283 | 284 | assert!(!RAN.with(|ran| ran.load(Ordering::Relaxed))); 285 | 286 | let mut mask = Sigset::full(); 287 | mask.del(Signal::User2); 288 | unsafe { 289 | sigsuspend(&mask); 290 | } 291 | 292 | assert!( RAN.with(|ran| ran.load(Ordering::Relaxed))); 293 | } 294 | } 295 | -------------------------------------------------------------------------------- /internal/libsignal/src/libgotcha.rs: -------------------------------------------------------------------------------- 1 | ../../../external/libtimetravel/src/libgotcha.rs -------------------------------------------------------------------------------- /internal/libsignal/src/pthread.rs: -------------------------------------------------------------------------------- 1 | pub use crate::Signal; 2 | 3 | use libc::c_int; 4 | use libc::pthread_t; 5 | use std::io::Error; 6 | use std::io::Result; 7 | 8 | pub struct PThread (pthread_t); 9 | 10 | pub fn pthread_kill(thread: PThread, signal: Signal) -> Result<()> { 11 | use crate::libgotcha::libgotcha_pthread_kill; 12 | 13 | let code = unsafe { 14 | libgotcha_pthread_kill(thread.0, signal as c_int) 15 | }; 16 | if code == 0 { 17 | Ok(()) 18 | } else { 19 | Err(Error::from_raw_os_error(code)) 20 | } 21 | } 22 | 23 | pub fn pthread_self() -> PThread { 24 | use libc::pthread_self; 25 | 26 | PThread (unsafe { 27 | pthread_self() 28 | }) 29 | } 30 | -------------------------------------------------------------------------------- /ld: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cc="cc" 4 | interp="" 5 | for arg in "$@" 6 | do 7 | case "$arg" in 8 | -linger) 9 | symlink="`dirname "$0"`/`basename "$0"`.so" 10 | if [ -z "$LIBINGER_LINKER" ] && readlink -e "$symlink" >/dev/null 11 | then 12 | LIBINGER_LINKER="$symlink" 13 | fi 14 | 15 | if [ -n "$LIBINGER_LINKER" ] 16 | then 17 | interp="-Wl,-I`readlink -f "$LIBINGER_LINKER"`" 18 | fi 19 | ;; 20 | */libgotcha-*.rlib) 21 | cc="`dirname "$0"`/cc" 22 | ;; 23 | esac 24 | done 25 | 26 | exec "$cc" $interp "$@" 27 | -------------------------------------------------------------------------------- /ldd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "$#" -eq "0" ] 4 | then 5 | echo "$0: missing file arguments" 6 | exit 1 7 | fi 8 | 9 | while [ "$#" -ne "0" ] 10 | do 11 | prog="$1" 12 | shift 13 | 14 | echo "$prog:" 15 | case "$prog" in 16 | /*) 17 | ;; 18 | *) 19 | prog="./$prog" 20 | esac 21 | LD_TRACE_LOADED_OBJECTS= "$prog" 22 | done 23 | -------------------------------------------------------------------------------- /profile: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "$#" -lt "2" ] 4 | then 5 | cat <<-tac 6 | USAGE: $0 ... 7 | 8 | is which benchmark to run (or an empty string for all) 9 | is the profiler and command-line arguments, e.g.,: 10 | time 11 | strace -c 12 | valgrind --tool=callgrind --collect-atstart=no --toggle-collect=bencher::Bencher::iter 13 | 14 | Only release builds of the inger benchmark are supported. Note that you probably 15 | want to enable debug symbols by adding the following to Cargo.toml before building: 16 | [profile.bench] 17 | debug = true 18 | [profile.release] 19 | debug = true 20 | tac 21 | exit 1 22 | fi 23 | 24 | test="$1" 25 | shift 26 | 27 | LIBGOTCHA_NOGLOBALS= LD_LIBRARY_PATH=target/release/deps exec "$@" target/release/inger-*[!.]? $test 28 | -------------------------------------------------------------------------------- /src/compile_assert.rs: -------------------------------------------------------------------------------- 1 | #[inline] 2 | pub fn assert_sync(_: &T) {} 3 | -------------------------------------------------------------------------------- /src/ffi.rs: -------------------------------------------------------------------------------- 1 | use crate::linger::Linger as Lingerer; 2 | 3 | use std::ffi::c_void; 4 | use std::process::abort; 5 | use std::thread::Result; 6 | 7 | #[repr(C)] 8 | pub struct Linger { 9 | is_complete: bool, 10 | continuation: Lingerer<(), dyn FnMut(*mut Option>) + Send>, 11 | } 12 | 13 | #[no_mangle] 14 | extern fn launch(fun: unsafe extern fn(*mut c_void), us: u64, args: *mut c_void) -> Linger { 15 | use crate::force::AssertSend; 16 | use crate::linger::launch; 17 | 18 | let args = unsafe { 19 | AssertSend::new(args) 20 | }; 21 | let timed = launch(move || unsafe { 22 | fun(*args) 23 | }, us); 24 | if let Ok(timed) = timed { 25 | Linger { 26 | is_complete: timed.is_completion(), 27 | continuation: timed.erase(), 28 | } 29 | } else { 30 | abort() 31 | } 32 | } 33 | 34 | #[no_mangle] 35 | extern fn resume(timed: Option<&mut Linger>, us: u64) { 36 | use crate::linger::resume; 37 | 38 | if let Some(timed) = timed { 39 | if resume(&mut timed.continuation, us).is_err() { 40 | abort(); 41 | } 42 | timed.is_complete = timed.continuation.is_completion(); 43 | } else { 44 | abort(); 45 | } 46 | } 47 | 48 | #[no_mangle] 49 | extern fn cancel(timed: Option<&mut Linger>) { 50 | if let Some(timed) = timed { 51 | if timed.continuation.is_continuation() { 52 | timed.continuation = Lingerer::Completion(()); 53 | timed.is_complete = true; 54 | } 55 | } else { 56 | abort(); 57 | } 58 | } 59 | 60 | #[no_mangle] 61 | extern fn pause() { 62 | use crate::linger::pause; 63 | 64 | pause(); 65 | } 66 | -------------------------------------------------------------------------------- /src/force.rs: -------------------------------------------------------------------------------- 1 | use std::ops::Deref; 2 | use std::ops::DerefMut; 3 | 4 | #[repr(transparent)] 5 | pub struct AssertSend (T); 6 | 7 | unsafe impl Send for AssertSend {} 8 | 9 | impl AssertSend { 10 | pub unsafe fn new(t: T) -> Self { 11 | Self (t) 12 | } 13 | } 14 | 15 | impl Deref for AssertSend { 16 | type Target = T; 17 | 18 | fn deref(&self) -> &Self::Target { 19 | let Self (this) = self; 20 | this 21 | } 22 | } 23 | 24 | impl DerefMut for AssertSend { 25 | fn deref_mut(&mut self) -> &mut Self::Target { 26 | let Self (this) = self; 27 | this 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/future.rs: -------------------------------------------------------------------------------- 1 | use crate::linger::Linger; 2 | 3 | use std::future::Future; 4 | use std::io::Result; 5 | use std::pin::Pin; 6 | use std::sync::mpsc::SyncSender; 7 | use std::task::Context; 8 | use std::task::Poll; 9 | use std::thread::Result as ThdResult; 10 | 11 | pub struct PreemptiveFuture< 12 | T, 13 | F: FnMut(*mut Option>) + Send, 14 | P: Fn() -> I + Unpin, 15 | I: FnMut() + Unpin, 16 | > { 17 | fun: Option>, 18 | us: u64, 19 | poll: P, 20 | pre: SyncSender, 21 | } 22 | 23 | pub fn poll_fn(fun: impl FnMut() -> Poll + Send, us: u64) 24 | -> Result>) + Send, impl Fn() -> fn(), fn()>> { 25 | fn nop() {} 26 | fn nope() -> fn() { nop } 27 | poll_fns(nope, fun, us) 28 | } 29 | 30 | pub fn poll_fns( 31 | poll: impl Fn() -> I + Unpin, 32 | mut fun: impl FnMut() -> Poll + Send, 33 | us: u64, 34 | ) -> Result>) + Send, impl Fn() -> I, I>> { 35 | use crate::linger::launch; 36 | use crate::linger::pause; 37 | 38 | use std::hint::unreachable_unchecked; 39 | use std::sync::mpsc::sync_channel; 40 | 41 | let (pre, prep): (SyncSender, _) = sync_channel(1); 42 | let fun = Some(launch(move || { 43 | let mut res; 44 | while { 45 | prep.recv().unwrap()(); 46 | res = fun(); 47 | res.is_pending() 48 | } { 49 | pause(); 50 | } 51 | if let Poll::Ready(res) = res { 52 | res 53 | } else { 54 | unsafe { 55 | unreachable_unchecked() 56 | } 57 | } 58 | }, 0)?); 59 | Ok(PreemptiveFuture { 60 | fun, 61 | us, 62 | poll, 63 | pre, 64 | }) 65 | } 66 | 67 | impl>) + Send, P: Fn() -> I + Unpin, I: FnMut() + Unpin> 68 | Future for PreemptiveFuture { 69 | type Output = Result; 70 | 71 | fn poll(mut self: Pin<&mut Self>, context: &mut Context) -> Poll { 72 | use crate::linger::resume; 73 | 74 | if let Some(mut fun) = self.fun.take() { 75 | self.pre.send((self.poll)()).unwrap(); 76 | if let Err(or) = resume(&mut fun, self.us) { 77 | Poll::Ready(Err(or)) 78 | } else { 79 | if let Linger::Completion(ready) = fun { 80 | Poll::Ready(Ok(ready)) 81 | } else { 82 | let timeout = ! fun.yielded(); 83 | self.fun.replace(fun); 84 | if timeout { 85 | // The preemptible function timed out rather than blocking 86 | // on some other future, so it's already ready to run again. 87 | context.waker().wake_by_ref(); 88 | } 89 | Poll::Pending 90 | } 91 | } 92 | } else { 93 | Poll::Pending 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/groups.rs: -------------------------------------------------------------------------------- 1 | use crate::reusable::SyncResult; 2 | 3 | use gotcha::Group as GotchaGroup; 4 | 5 | pub fn assign_group() -> SyncResult<'static, GotchaGroup> { 6 | use crate::compile_assert::assert_sync; 7 | use crate::reusable::SyncPool; 8 | 9 | use std::convert::TryInto; 10 | use std::sync::Once; 11 | 12 | static mut GROUPS: Option> = None; 13 | static INIT: Once = Once::new(); 14 | INIT.call_once(|| unsafe { 15 | GROUPS.replace(SyncPool::new(GotchaGroup::new)); 16 | }); 17 | 18 | let groups = unsafe { 19 | GROUPS.as_ref() 20 | }.unwrap(); 21 | assert_sync(&groups); 22 | groups.try_into() 23 | } 24 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | mod compile_assert; 2 | pub mod ffi; 3 | pub mod force; 4 | pub mod future; 5 | mod groups; 6 | mod lifetime; 7 | mod linger; 8 | mod localstores; 9 | mod preemption; 10 | pub mod profiler; 11 | mod reusable; 12 | mod signals; 13 | mod stacks; 14 | #[cfg(not(feature = "notls"))] 15 | mod tcb; 16 | mod timer; 17 | mod unfurl; 18 | 19 | #[cfg(feature = "notls")] 20 | mod tcbstub; 21 | #[cfg(feature = "notls")] 22 | mod tcb { 23 | pub use crate::tcbstub::*; 24 | } 25 | 26 | pub use linger::*; 27 | 28 | use gotcha::Group; 29 | 30 | const QUANTUM_MICROSECS: u64 = 100; 31 | 32 | #[doc(hidden)] 33 | pub const STACK_N_PREALLOC: usize = Group::LIMIT; 34 | const STACK_SIZE_BYTES: usize = 2 * 1_024 * 1_024; 35 | 36 | #[no_mangle] 37 | static libgotcha_exitanalysis: bool = true; 38 | 39 | pub fn concurrency_limit() -> usize { 40 | Group::limit() 41 | } 42 | 43 | #[cfg(test)] 44 | fn main() {} 45 | -------------------------------------------------------------------------------- /src/lifetime.rs: -------------------------------------------------------------------------------- 1 | #![allow(dead_code)] 2 | 3 | pub unsafe fn unbound<'a, T>(bounded: *const T) -> &'a T { 4 | &*bounded 5 | } 6 | 7 | pub unsafe fn unbound_mut<'a, T>(bounded: *mut T) -> &'a mut T { 8 | &mut *bounded 9 | } 10 | -------------------------------------------------------------------------------- /src/linger.rs: -------------------------------------------------------------------------------- 1 | use crate::preemption::RealThreadId; 2 | use crate::preemption::defer_preemption; 3 | use crate::preemption::disable_preemption; 4 | use crate::preemption::is_preemptible; 5 | use crate::preemption::thread_signal; 6 | use crate::reusable::ReusableSync; 7 | use crate::stacks::DerefAdapter; 8 | use crate::tcb::ThreadControlBlock; 9 | 10 | use gotcha::Group; 11 | use signal::Set; 12 | use signal::Signal; 13 | use signal::siginfo_t; 14 | use std::cell::Cell; 15 | use std::cell::RefCell; 16 | use std::io::Result; 17 | use std::os::raw::c_int; 18 | use std::ptr::NonNull; 19 | use std::thread::Result as ThdResult; 20 | use timetravel::errno::errno; 21 | use timetravel::Context; 22 | use timetravel::HandlerContext; 23 | 24 | #[repr(C)] 25 | pub enum Linger>) + Send + ?Sized> { 26 | Completion(T), 27 | Continuation(Continuation), 28 | Poison, 29 | } 30 | 31 | impl>) + Send + ?Sized> Unpin for Linger {} 32 | 33 | impl>) + Send + ?Sized> Linger { 34 | pub fn is_completion(&self) -> bool { 35 | if let Linger::Completion(_) = self { 36 | true 37 | } else { 38 | false 39 | } 40 | } 41 | 42 | pub fn is_continuation(&self) -> bool { 43 | if let Linger::Continuation(_) = self { 44 | true 45 | } else { 46 | false 47 | } 48 | } 49 | 50 | pub fn yielded(&self) -> bool { 51 | if let Linger::Continuation(continuation) = self { 52 | continuation.stateful.yielded 53 | } else { 54 | false 55 | } 56 | } 57 | } 58 | 59 | impl<'a, T, F: FnMut(*mut Option>) + Send + 'a> Linger { 60 | /// Erase the type of the contained closure, thereby allowing insertion of the instance into 61 | /// a polymorphic data structure containing other instances. After this operation, the 62 | /// instance behaves as before: only its type is changed. 63 | pub fn erase(self) -> Linger>) + Send + 'a> { 64 | use std::mem::MaybeUninit; 65 | 66 | if let Linger::Completion(this) = self { 67 | Linger::Completion(this) 68 | } else if let Linger::Continuation(this) = self { 69 | let this = MaybeUninit::new(this); 70 | let this = this.as_ptr(); 71 | unsafe { 72 | let functional: *const _ = &(*this).functional; 73 | let stateful: *const _ = &(*this).stateful; 74 | let group: *const _ = &(*this).group; 75 | let tls: *const _ = &(*this).tls; 76 | 77 | Linger::Continuation(Continuation { 78 | functional: functional.read(), 79 | stateful: stateful.read(), 80 | group: group.read(), 81 | tls: tls.read(), 82 | }) 83 | } 84 | } else { 85 | Linger::Poison 86 | } 87 | } 88 | } 89 | 90 | pub struct Continuation { 91 | // First they called for the preemptible function to be executed, and I did not read the 92 | // argument because it was not present. Then they called for the return value, and I did 93 | // not call the preemptible function because it was not present. Then they called for me, 94 | // and I did not call or return because there was no one left to call and I had nothing left 95 | // to give. (Only call this twice!) 96 | // 97 | // Because the whole Continuation might be moved between this function's preemption and its 98 | // resumption, we must heap allocate it so its captured environment has a stable address. 99 | functional: Box, 100 | stateful: Task, 101 | group: ReusableSync<'static, Group>, 102 | tls: ReusableSync<'static, Option>, 103 | } 104 | 105 | unsafe impl Send for Continuation {} 106 | 107 | impl Drop for Continuation { 108 | fn drop(&mut self) { 109 | debug_assert!(! is_preemptible(), "libinger: dropped from preemptible function"); 110 | 111 | let group = &self.group; 112 | if self.stateful.errno.is_some() { 113 | // We're canceling a paused preemptible function. Clean up the group! 114 | assert!(group.renew(), "libinger: failed to reinitialize library group"); 115 | } 116 | } 117 | } 118 | 119 | #[derive(Default)] 120 | struct Task { 121 | // Also indicates the state of execution. If we have a Continuation instance, we know the 122 | // computatation cannot have completed earlier, so it must be in one of these states... 123 | // * None on entry to resume() means it hasn't yet started running. 124 | // * None at end of resume() means it has completed. 125 | // * Some at either point means it timed out and is currently paused. 126 | errno: Option, 127 | checkpoint: Option>>>>, 128 | yielded: bool, 129 | } 130 | 131 | /// Run `fun` with the specified time budget, in `us`econds. 132 | /// 133 | /// If the budget is `0`, the timed function is initialized but not invoked; if it is `max_value()`, 134 | /// it is run to completion. 135 | pub fn launch(fun: impl FnOnce() -> T + Send, us: u64) 136 | -> Result>) + Send>> { 137 | use crate::localstores::alloc_localstore; 138 | use crate::groups::assign_group; 139 | 140 | use std::panic::AssertUnwindSafe; 141 | use std::panic::catch_unwind; 142 | 143 | enum Completion { 144 | Function(F), 145 | Return(T), 146 | Empty, 147 | } 148 | 149 | impl Completion { 150 | fn take(&mut self) -> Self { 151 | use std::mem::replace; 152 | 153 | replace(self, Completion::Empty) 154 | } 155 | } 156 | 157 | // Danger, W.R.! Although in theory libgotcha ensures there's only one copy of our library, 158 | // exported parameterized functions are actually monomorphized into the *caller's* object 159 | // file! This means that this function has an inconsistent view of the worldstate if called 160 | // from a preemptible function, but that any non-generic (name brand?) functions it calls 161 | // are guaranteed to run in the libgotcha's shared group. To guard against heisenbugs 162 | // arising from the former case, we first assert that no one has attempted a nested call. 163 | debug_assert!(! is_preemptible(), "launch(): called from preemptible function"); 164 | 165 | let mut fun = Completion::Function(AssertUnwindSafe (fun)); 166 | let fun = Box::new(move |ret: *mut Option>| { 167 | fun = match fun.take() { 168 | Completion::Function(fun) => 169 | // We haven't yet moved the closure. This means schedule() is invoking us 170 | // as a *nullary* function, implying ret is undefined and mustn't be used. 171 | Completion::Return(catch_unwind(fun)), 172 | Completion::Return(val) => { 173 | debug_assert!(! ret.is_null()); 174 | let ret = unsafe { 175 | &mut *ret 176 | }; 177 | ret.replace(val); 178 | Completion::Empty 179 | }, 180 | Completion::Empty => 181 | Completion::Empty, 182 | } 183 | }); 184 | 185 | let group = assign_group().expect("launch(): too many active timed functions"); 186 | let mut linger = Linger::Continuation(Continuation { 187 | functional: fun, 188 | stateful: Task::default(), 189 | group, 190 | tls: alloc_localstore(), 191 | }); 192 | if us != 0 { 193 | resume(&mut linger, us)?; 194 | } 195 | Ok(linger) 196 | } 197 | 198 | // Note that it is only safe to use these while the virtual thread-control block is installed! 199 | thread_local! { 200 | static BOOTSTRAP: Cell, Group)>> = Cell::default(); 201 | static TASK: RefCell = RefCell::default(); 202 | static DEADLINE: Cell = Cell::default(); 203 | } 204 | 205 | /// Let `fun` continue running for the specified time budget, in `us`econds. 206 | /// 207 | /// If the budget is `0`, this is a no-op; if it is `max_value()`, the timed function is run to 208 | /// completion. This function is idempotent once the timed function completes. 209 | pub fn resume(fun: &mut Linger>) + Send + ?Sized>, us: u64) 210 | -> Result<&mut Linger>) + Send + ?Sized>> { 211 | use std::panic::resume_unwind; 212 | 213 | // Danger, W.R! The same disclaimer from launch() applies here. 214 | debug_assert!(! is_preemptible(), "resume(): called from preemptible function"); 215 | 216 | if let Linger::Continuation(continuation) = fun { 217 | let task = &mut continuation.stateful; 218 | let group = *continuation.group; 219 | let thread = RealThreadId::current(); 220 | 221 | // Install the virtual thread-control block. It is only safe to use thread-locals 222 | // between the end of this block and when we uninstall it again! 223 | let tls = continuation.tls.take().expect("libinger: continuation with missing TCB"); 224 | let tls = unsafe { 225 | tls.install(group)? 226 | }; 227 | 228 | if let Err(or) = setup_thread(thread) { 229 | abort(&format!("resume(): failure in thread_setup(): {}", or)); 230 | } 231 | 232 | // Are we launching this preemptible function for the first time? 233 | if task.errno.is_none() { 234 | let fun = &mut continuation.functional; 235 | task.checkpoint = setup_stack()?; 236 | BOOTSTRAP.with(|bootstrap| { 237 | // The schedule() function is polymorphic across "return" types, but 238 | // we expect a storage area appropriate for our own type. To remove 239 | // the specialization from our signature, we reduce our arity by 240 | // casting away our parameter. This is safe because schedule() 241 | // represents our first caller, so we know not to read the argument. 242 | let fun: *mut (dyn FnMut(_) + Send) = fun; 243 | let fun: *mut (dyn FnMut() + Send) = fun as _; 244 | let no_fun = bootstrap.replace(NonNull::new(fun).map(|fun| 245 | (fun, group) 246 | )); 247 | debug_assert!( 248 | no_fun.is_none(), 249 | "resume(): bootstraps without an intervening schedule()", 250 | ); 251 | }); 252 | } 253 | 254 | DEADLINE.with(|deadline| deadline.replace(us)); 255 | 256 | // Transfer control into the libinger module before running the function! 257 | let finished = switch_stack(task, group)?; 258 | continuation.tls.replace(unsafe { 259 | tls.uninstall()? 260 | }); 261 | if finished { 262 | // The preemptible function finished (either ran to completion or panicked). 263 | // Since we know the closure is no longer running concurrently, it's now 264 | // safe to call it again to retrieve the return value. 265 | let mut retval = None; 266 | (continuation.functional)(&mut retval); 267 | 268 | match retval.expect("resume(): return value was already retrieved") { 269 | Ok(retval) => *fun = Linger::Completion(retval), 270 | Err(panic) => { 271 | *fun = Linger::Poison; 272 | resume_unwind(panic); 273 | }, 274 | } 275 | } 276 | } 277 | 278 | Ok(fun) 279 | } 280 | 281 | /// Set up preemption for a kernel execution thread. Call after installing a virtual TCB! 282 | #[inline(never)] 283 | fn setup_thread(thread: RealThreadId) -> Result<()> { 284 | use crate::preemption::thread_setup; 285 | use super::QUANTUM_MICROSECS; 286 | thread_setup(thread, preempt, QUANTUM_MICROSECS) 287 | } 288 | 289 | /// Set up the oneshot execution stack. Always returns a Some when things are Ok. 290 | #[inline(never)] 291 | fn setup_stack() 292 | -> Result>>>>> { 293 | use crate::stacks::alloc_stack; 294 | 295 | use timetravel::makecontext; 296 | 297 | let mut checkpoint = None; 298 | makecontext( 299 | DerefAdapter::from(alloc_stack()), 300 | |goto| drop(checkpoint.replace(goto)), 301 | schedule, 302 | )?; 303 | Ok(checkpoint) 304 | } 305 | 306 | /// Jump to the preemptible function, reenabling preemption if the function was previously paused. 307 | /// Runs on the main execution stack. Returns whether the function "finished" without a timeout. 308 | #[inline(never)] 309 | fn switch_stack(task: &mut Task, group: Group) -> Result { 310 | use crate::lifetime::unbound_mut; 311 | 312 | use gotcha::group_thread_set; 313 | use signal::Operation; 314 | use signal::Sigset; 315 | use signal::pthread_sigmask; 316 | use timetravel::restorecontext; 317 | use timetravel::sigsetcontext; 318 | 319 | let mut error = None; 320 | restorecontext( 321 | task.checkpoint.take().expect("switch_stack(): continuation is missing"), 322 | |pause| { 323 | let resume = TASK.with(|task| { 324 | let mut task = task.borrow_mut(); 325 | debug_assert!( 326 | task.checkpoint.is_none(), 327 | "switch_stack(): this continuation would nest?!" 328 | ); 329 | 330 | let resume = task.checkpoint.get_or_insert(pause); 331 | unsafe { 332 | unbound_mut(resume) 333 | } 334 | }); 335 | 336 | // Are we resuming a paused preemptible function? 337 | if let Some(erryes) = task.errno { 338 | // Do everything to enable preemption short of unblocking the 339 | // signal, which will be don atomically by sigsetcontext() as it 340 | // jumps into the continuation. 341 | let mut old = Sigset::empty(); 342 | let mut new = Sigset::empty(); 343 | let sig = thread_signal().unwrap_or_else(|_| 344 | abort("switch_stack(): error accessing TLS variable") 345 | ); 346 | new.add(sig); 347 | if let Err(or) = pthread_sigmask( 348 | Operation::Block, 349 | &new, 350 | Some(&mut old), 351 | ) { 352 | error.replace(or); 353 | } 354 | old.del(sig); 355 | *resume.mask() = old; 356 | 357 | // The clock is ticking from this "start" point. 358 | stamp(); 359 | 360 | // The order of these two lines, with respect to both each other and 361 | // the rest of the program, is very important. We need to switch to 362 | // the new group before we restore errno so we get the right one, 363 | // and there cannot be any standard library calls between its 364 | // restoration and the call to sigsetcontext(). 365 | group_thread_set!(group); 366 | *errno() = erryes; 367 | } 368 | 369 | // No library calls may be made before this one! 370 | let failure = sigsetcontext(resume); 371 | error.replace(failure.expect("resume(): continuation is invalid")); 372 | }, 373 | )?; 374 | if let Some(error) = error { 375 | Err(error)?; 376 | } 377 | 378 | let descheduled = TASK.with(|task| task.replace(Task::default())); 379 | let preempted = descheduled.errno.is_some(); 380 | if preempted { 381 | *task = descheduled; 382 | } else { 383 | // Prevent namespace reinitialization on drop of Continuation containing the Task. 384 | task.errno.take(); 385 | } 386 | 387 | Ok(! preempted) 388 | } 389 | 390 | /// Enable preemption and call the preemptible function. Runs on the oneshot execution stack. 391 | fn schedule() { 392 | use crate::preemption::enable_preemption; 393 | 394 | let (mut fun, group) = BOOTSTRAP.with(|bootstrap| bootstrap.take()).unwrap_or_else(|| 395 | abort("schedule(): called without bootstrapping") 396 | ); 397 | let fun = unsafe { 398 | fun.as_mut() 399 | }; 400 | stamp(); 401 | if let Ok(signal) = enable_preemption(group.into()) { 402 | debug_assert!(signal.is_some()); 403 | 404 | fun(); 405 | disable_preemption(signal); 406 | } else { 407 | abort("schedule(): error accessing TLS variable"); 408 | } 409 | 410 | // It's important that we haven't saved any errno, since we'll check it to determine whether 411 | // the preemptible function ran to completion. 412 | debug_assert!( 413 | TASK.with(|task| task.borrow().errno.is_none()), 414 | "schedule(): finished leaving errno", 415 | ); 416 | } 417 | 418 | /// Signal handler that pauses the preemptible function on timeout. Runs on the oneshot stack. 419 | extern fn preempt(no: Signal, _: Option<&siginfo_t>, uc: Option<&mut HandlerContext>) { 420 | use crate::unfurl::Unfurl; 421 | 422 | use timetravel::Swap; 423 | 424 | let erryes = *errno(); 425 | let uc = unsafe { 426 | uc.unfurl() 427 | }; 428 | let relevant = thread_signal().map(|signal| no == signal).unwrap_or(false); 429 | if relevant && is_preemptible() { 430 | let deadline = DEADLINE.with(|deadline| deadline.get()); 431 | if nsnow() >= deadline { 432 | TASK.with(|task| { 433 | // It's time to pause the function. We need to save its state. 434 | let mut task = task.borrow_mut(); 435 | 436 | // Did it cooperatively yield instead of being preempted? 437 | task.yielded = deadline == 0; 438 | 439 | // Configure us to return into the checkpoint for its call site. 440 | let checkpoint = task.checkpoint.as_mut(); 441 | let checkpoint = unsafe { 442 | checkpoint.unfurl() 443 | }; 444 | checkpoint.swap(uc); 445 | 446 | // Instead of restoring errno, save it for if and when we resume. 447 | task.errno.replace(erryes); 448 | 449 | // Block this signal and disable preemption. 450 | uc.uc_sigmask.add(no); 451 | disable_preemption(None); 452 | }); 453 | } else { 454 | *errno() = erryes; 455 | } 456 | } else { 457 | if relevant { 458 | // The timed function has called into a nonpreemptible library function. 459 | // We'll need to intercept it immediately upon the function's return. 460 | defer_preemption((&mut uc.uc_sigmask, no).into()); 461 | } else { 462 | // We still want to block this signal so it doesn't disturb us again. 463 | uc.uc_sigmask.add(no); 464 | } 465 | 466 | *errno() = erryes; 467 | } 468 | } 469 | 470 | /// Bump the deadline forward by the current wall-clock time, unless the timeout is unlimited. 471 | fn stamp() { 472 | DEADLINE.with(|deadline| 473 | deadline.replace(deadline.get().checked_mul(1_000).map(|timeout| 474 | nsnow() + timeout 475 | ).unwrap_or(deadline.get())) 476 | ); 477 | } 478 | 479 | /// Immediately yield the calling preemptible function. 480 | #[inline(never)] 481 | pub fn pause() { 482 | // Note that this function is not parameterized, so it is itself nonpreemptible. This is 483 | // key because we need to be able to request preemption atomically (so we only get one). 484 | DEADLINE.with(|deadline| deadline.take()); 485 | 486 | // Get preempted when we return. 487 | defer_preemption(None); 488 | } 489 | 490 | /// Read the current wall-clock time, in nanoseconds. 491 | #[doc(hidden)] 492 | pub fn nsnow() -> u64 { 493 | use std::time::UNIX_EPOCH; 494 | use std::time::SystemTime; 495 | 496 | let now = SystemTime::now().duration_since(UNIX_EPOCH).expect("libinger: wall clock error"); 497 | let mut sum = now.subsec_nanos().into(); 498 | sum += now.as_secs() * 1_000_000_000; 499 | sum 500 | } 501 | 502 | #[doc(hidden)] 503 | pub fn abort(err: &str) -> ! { 504 | use std::process::abort; 505 | 506 | let abort: fn() -> _ = abort; 507 | eprintln!("{}", err); 508 | abort() 509 | } 510 | -------------------------------------------------------------------------------- /src/localstores.rs: -------------------------------------------------------------------------------- 1 | use crate::reusable::ReusableSync; 2 | use crate::tcb::ThreadControlBlock; 3 | 4 | pub fn alloc_localstore() -> ReusableSync<'static, Option> { 5 | use crate::compile_assert::assert_sync; 6 | use crate::reusable::SyncPool; 7 | 8 | use gotcha::Group; 9 | use std::convert::TryInto; 10 | use std::sync::Once; 11 | 12 | static mut LOCALSTORES: Option>> = None; 13 | static INIT: Once = Once::new(); 14 | INIT.call_once(|| { 15 | let localstores: fn() -> _ = || Some(Some(ThreadControlBlock::new())); 16 | let localstores = SyncPool::new(localstores); 17 | localstores.prealloc(Group::limit()) 18 | .expect("libinger: TCB allocator lock was poisoned during init"); 19 | unsafe { 20 | LOCALSTORES.replace(localstores); 21 | } 22 | }); 23 | 24 | let localstores = unsafe { 25 | LOCALSTORES.as_ref() 26 | }.unwrap(); 27 | assert_sync(&localstores); 28 | localstores.try_into().expect("libinger: TCB allocator lock is poisoned") 29 | } 30 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use inger::ffi::Linger; 2 | use std::mem::size_of; 3 | 4 | fn main() { 5 | print!( 6 | "\ 7 | #ifndef LIBINGER_H_\n\ 8 | #define LIBINGER_H_\n\ 9 | \n\ 10 | #include \n\ 11 | #include \n\ 12 | \n\ 13 | typedef struct {{\n\ 14 | bool is_complete;\n\ 15 | uint8_t continuation[{}];\n\ 16 | }} linger_t;\n\ 17 | \n\ 18 | linger_t launch(void (*)(void *), uint64_t, void *);\n\ 19 | void resume(linger_t *, uint64_t);\n\ 20 | void cancel(linger_t *);\n\ 21 | \n\ 22 | void pause(void);\n\ 23 | \n\ 24 | #endif\n\ 25 | ", 26 | size_of::(), 27 | ); 28 | } 29 | -------------------------------------------------------------------------------- /src/preemption.rs: -------------------------------------------------------------------------------- 1 | use crate::reusable::ReusableSync; 2 | use crate::timer::Timer; 3 | 4 | use gotcha::Group; 5 | use gotcha::group_thread_set; 6 | use signal::pthread::pthread_kill; 7 | use signal::pthread::pthread_self; 8 | use signal::Handler; 9 | use signal::Operation; 10 | use signal::Set; 11 | use signal::Signal; 12 | use signal::Sigset; 13 | use signal::sigaction; 14 | use std::cell::RefCell; 15 | use std::io::Result as IoResult; 16 | use std::os::raw::c_int; 17 | use std::sync::atomic::AtomicBool; 18 | use std::sync::atomic::Ordering; 19 | 20 | thread_local! { 21 | static SIGNAL: RefCell> = RefCell::default(); 22 | 23 | // Whether we had to delay preemption checks until the end of a nonpreemptible call. 24 | static DEFERRED: AtomicBool = AtomicBool::new(false); 25 | } 26 | 27 | pub fn thread_signal() -> Result { 28 | // Because this is called from signal handlers, it might happen during thread teardown, when 29 | // the thread-local variable is being/has been destructed. In such a case, we simply report 30 | // that the current thread has no preemption signal assigned (any longer). 31 | SIGNAL.try_with(|signal| 32 | signal.borrow().as_ref().map(|signal| { 33 | let RealThreadId (signal) = signal; 34 | signal.borrow().as_ref().map(|signal| 35 | *signal.signal 36 | ) 37 | }).unwrap_or(None).ok_or(()) 38 | ).unwrap_or(Err(())) 39 | } 40 | 41 | extern fn resume_preemption() { 42 | // Skip if this trampoline is running in a destructor during thread teardown. 43 | drop(enable_preemption(None)); 44 | } 45 | 46 | pub fn enable_preemption(group: Option) -> Result, ()> { 47 | use timetravel::errno::errno; 48 | 49 | // We must access errno_group() even when we won't use it so that the initial 50 | // enable_preemption() call bootstraps later resume_preemption() ones! 51 | let erryes = errno_group(group); 52 | let errno = *errno(); 53 | 54 | // We can only call thread_signal() if the preemption signal is already blocked; otherwise, 55 | // the signal handler might race on the thread-local SIGNAL variable. It's fine to do when: 56 | // * We have been passed a group, because in this case preemption was previously disabled. 57 | // - OR - 58 | // * Preemption has been deferred, because when setting the flag, the signal handler will 59 | // have masked out the signal. 60 | let mut unblock = None; 61 | if let Some(group) = group { 62 | // It's important we don't unmask the preemption signal until we've switched groups; 63 | // otherwise, its handler may run immediately and remask it! 64 | group_thread_set!(group); 65 | unblock.replace(thread_signal()?); 66 | } 67 | // else the caller is asserting the group change has already been performed. 68 | 69 | if DEFERRED.with(|deferred| deferred.swap(false, Ordering::Relaxed)) { 70 | let signal = unblock.get_or_insert(thread_signal()?); 71 | drop(pthread_kill(pthread_self(), *signal)); 72 | } 73 | 74 | if let Some(signal) = unblock { 75 | drop(mask(Operation::Unblock, signal)); 76 | } 77 | 78 | // Propagate any errors encountered during our libc replacements back to the calling libset. 79 | if group.is_none() && errno != 0 { 80 | *erryes = errno; 81 | } 82 | Ok(unblock) 83 | } 84 | 85 | pub fn disable_preemption(block: Option) { 86 | group_thread_set!(Group::SHARED); 87 | if let Some(signal) = block { 88 | // Mask the preemption signal without calling thread_signal(), which would by racy. 89 | drop(mask(Operation::Block, signal)); 90 | } 91 | 92 | SIGNAL.with(|signal| signal.replace(None)); 93 | DEFERRED.with(|deferred| deferred.store(false, Ordering::Relaxed)); 94 | } 95 | 96 | pub fn is_preemptible() -> bool { 97 | use gotcha::group_thread_get; 98 | 99 | ! group_thread_get!().is_shared() 100 | } 101 | 102 | // It is only safe to call this function while preemption is (temporarily) disabled! 103 | pub fn defer_preemption(signum: Option<(&mut Sigset, Signal)>) { 104 | debug_assert!(! is_preemptible()); 105 | 106 | // We must first mask the signal so no attempted preemption races on DEFERRED! 107 | if let Some((sigmask, signo)) = signum { 108 | // Caller is asserting we are beneath a signal handler, so we should only update the 109 | // outside world's mask. 110 | sigmask.add(signo); 111 | } else { 112 | drop(mask(Operation::Block, thread_signal().unwrap())); 113 | } 114 | 115 | DEFERRED.with(|deferred| deferred.store(true, Ordering::Relaxed)); 116 | } 117 | 118 | pub fn thread_setup(thread: RealThreadId, handler: Handler, quantum: u64) -> IoResult<()> { 119 | use gotcha::shared_hook; 120 | use std::sync::Once; 121 | 122 | let RealThreadId (signal) = thread; 123 | if signal.borrow().is_none() { 124 | signal.replace(Some(PreemptionSignal::new(handler, quantum)?)); 125 | } 126 | SIGNAL.with(|signal| signal.replace(Some(thread))); 127 | 128 | static INIT: Once = Once::new(); 129 | INIT.call_once(|| shared_hook(resume_preemption)); 130 | 131 | Ok(()) 132 | } 133 | 134 | fn errno_group(group: Option) -> &'static mut c_int { 135 | use gotcha::group_lookup_symbol_fn; 136 | use libc::__errno_location; 137 | thread_local! { 138 | static ERRNO_LOCATION: RefCell *mut c_int>> = 139 | RefCell::new(None); 140 | } 141 | 142 | // We save the location in a thread-local variable so the next time we need to find its 143 | // *location,* we won't clear its *value* in the process! 144 | ERRNO_LOCATION.with(|errno_location| { 145 | let mut errno_location = errno_location.borrow_mut(); 146 | if let Some(group) = group { 147 | errno_location.replace(unsafe { 148 | group_lookup_symbol_fn!(group, __errno_location) 149 | }.unwrap()); 150 | } 151 | let __errno_location = errno_location.unwrap(); 152 | unsafe { 153 | &mut *__errno_location() 154 | } 155 | }) 156 | } 157 | 158 | fn mask(un: Operation, no: Signal) -> IoResult<()> { 159 | use signal::pthread_sigmask; 160 | 161 | let mut set = Sigset::empty(); 162 | set.add(no); 163 | pthread_sigmask(un, &set, None) 164 | } 165 | 166 | pub struct RealThreadId (&'static RefCell>); 167 | 168 | impl RealThreadId { 169 | pub fn current() -> Self { 170 | use crate::lifetime::unbound; 171 | 172 | thread_local! { 173 | static SIGNALER: RefCell> = RefCell::default(); 174 | } 175 | 176 | Self (SIGNALER.with(|signaler| unsafe { 177 | unbound(signaler) 178 | })) 179 | } 180 | } 181 | 182 | struct PreemptionSignal { 183 | signal: ReusableSync<'static, Signal>, 184 | timer: Timer, 185 | } 186 | 187 | impl PreemptionSignal { 188 | fn new(handler: Handler, quantum: u64) -> IoResult { 189 | use crate::signals::assign_signal; 190 | use crate::timer::Clock; 191 | use crate::timer::Sigevent; 192 | use crate::timer::itimerspec; 193 | use crate::timer::timer_create; 194 | use crate::timer::timer_settime; 195 | 196 | use libc::SA_RESTART; 197 | use libc::SA_SIGINFO; 198 | use libc::timespec; 199 | use signal::Action; 200 | use signal::Sigaction; 201 | 202 | let signal = assign_signal().expect("libinger: no available signal for preempting this thread"); 203 | let sa = Sigaction::new(handler, Sigset::empty(), SA_SIGINFO | SA_RESTART); 204 | sigaction(*signal, &sa, None)?; 205 | mask(Operation::Block, *signal)?; 206 | 207 | let mut se = Sigevent::signal(*signal); 208 | let timer = timer_create(Clock::Real, &mut se)?; 209 | let quantum: i64 = quantum as _; 210 | let mut it = itimerspec { 211 | it_interval: timespec { 212 | tv_sec: 0, 213 | tv_nsec: quantum * 1_000, 214 | }, 215 | it_value: timespec { 216 | tv_sec: 0, 217 | tv_nsec: quantum * 1_000, 218 | }, 219 | }; 220 | timer_settime(timer, false, &mut it, None)?; 221 | 222 | Ok(Self { 223 | signal, 224 | timer, 225 | }) 226 | } 227 | } 228 | 229 | impl Drop for PreemptionSignal { 230 | fn drop(&mut self) { 231 | use crate::timer::timer_delete; 232 | 233 | if let Err(or) = timer_delete(self.timer) { 234 | eprintln!("libinger: unable to delete POSIX timer: {}", or); 235 | } 236 | if let Err(or) = sigaction(*self.signal, &(), None) { 237 | eprintln!("libinger: unable to unregister signal handler: {}", or); 238 | } 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /src/profiler.rs: -------------------------------------------------------------------------------- 1 | use crate::linger::nsnow; 2 | 3 | use libc::ucontext_t; 4 | use signal::Set; 5 | use signal::Signal; 6 | use signal::Sigset; 7 | use signal::siginfo_t; 8 | use std::cell::RefCell; 9 | use std::collections::VecDeque; 10 | 11 | pub fn with_profiler(fun: impl Fn(&mut Profiler) -> T) -> T { 12 | thread_local! { 13 | static PROFILER: RefCell> = RefCell::default(); 14 | } 15 | PROFILER.with(|profiler| 16 | fun(profiler.borrow_mut().get_or_insert_with(Profiler::default)) 17 | ) 18 | } 19 | 20 | #[derive(Default)] 21 | pub struct Profiler { 22 | past: VecDeque, 23 | present: u64, 24 | } 25 | 26 | impl Profiler { 27 | pub fn begin(&mut self) { 28 | use signal::Action; 29 | use signal::Sigaction; 30 | use signal::sigaction; 31 | use std::sync::Once; 32 | 33 | static ONCE: Once = Once::new(); 34 | ONCE.call_once(|| 35 | drop(sigaction(Signal::Interrupt, &mut Sigaction::new(interrupt, Sigset::empty(), 0), None)) 36 | ); 37 | self.present = nsnow(); 38 | } 39 | 40 | pub fn end(&mut self) -> bool { 41 | if self.present != 0 { 42 | self.past.push_back(nsnow() - self.present); 43 | self.present = 0; 44 | true 45 | } else { 46 | false 47 | } 48 | } 49 | } 50 | 51 | impl Drop for Profiler { 52 | fn drop(&mut self) { 53 | use std::ffi::CString; 54 | use std::os::raw::c_char; 55 | 56 | extern { 57 | fn puts(_: *const c_char); 58 | } 59 | 60 | let len: f64 = self.past.len() as _; 61 | let sum: u64 = self.past.iter().sum(); 62 | let sum: f64 = sum as _; 63 | let msg = CString::new(format!("Profiler ave. = {} us", sum / len / 1_000.0)).unwrap(); 64 | unsafe { 65 | puts(msg.as_ptr()); 66 | } 67 | } 68 | } 69 | 70 | extern fn interrupt(_: Signal, _: Option<&siginfo_t>, _: Option<&mut ucontext_t>) { 71 | use signal::Operation; 72 | use signal::pthread_sigmask; 73 | use std::os::raw::c_int; 74 | use std::process::abort; 75 | use std::thread::current; 76 | extern { 77 | fn exit(_: c_int); 78 | } 79 | if current().name().unwrap_or_else(|| abort()) != "main" { 80 | unsafe { 81 | exit(100); 82 | } 83 | } else { 84 | let mut sig = Sigset::empty(); 85 | sig.add(Signal::Interrupt); 86 | drop(pthread_sigmask(Operation::Block, &sig, None)) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/reusable.rs: -------------------------------------------------------------------------------- 1 | use std::cell::BorrowMutError; 2 | use std::cell::RefCell; 3 | use std::cell::RefMut; 4 | use std::convert::TryFrom; 5 | use std::fmt::Debug; 6 | use std::marker::PhantomData; 7 | use std::ops::Deref; 8 | use std::ops::DerefMut; 9 | use std::result::Result as StdResult; 10 | use std::sync::Mutex; 11 | use std::sync::MutexGuard; 12 | use std::sync::PoisonError; 13 | 14 | type Sync = Mutex>; 15 | type Unsync = RefCell>; 16 | 17 | pub struct Reusable<'a, T, A = Unsync> 18 | where &'a A: SharedMut> { 19 | value: Option, 20 | pool: &'a A, 21 | } 22 | 23 | pub type ReusableSync<'a, T> = Reusable<'a, T, Sync>; 24 | 25 | impl<'a, T, A, B: Fn() -> Option> TryFrom<&'a Pool> for Reusable<'a, T, A> 26 | where &'a A: SharedMut> { 27 | type Error = Option<<&'a A as SharedMut>>::Error>; 28 | 29 | fn try_from(pool: &'a Pool) -> StdResult, Self::Error> { 30 | let builder = &pool.builder; 31 | let pool = &pool.allocated; 32 | let value = pool.try_into_inner()?.pop().or_else(builder); 33 | if value.is_some() { 34 | Ok(Self { 35 | value, 36 | pool, 37 | }) 38 | } else { 39 | Err(None) 40 | } 41 | } 42 | } 43 | 44 | impl<'a, T, A> Deref for Reusable<'a, T, A> 45 | where &'a A: SharedMut> { 46 | type Target = T; 47 | 48 | fn deref(&self) -> &Self::Target { 49 | // Can only be None if we're called while being dropped! 50 | self.value.as_ref().unwrap() 51 | } 52 | } 53 | 54 | impl<'a, T, A> DerefMut for Reusable<'a, T, A> 55 | where &'a A: SharedMut> { 56 | fn deref_mut(&mut self) -> &mut Self::Target { 57 | // Can only be None if we're called while being dropped! 58 | self.value.as_mut().unwrap() 59 | } 60 | } 61 | 62 | impl<'a, T, A> Drop for Reusable<'a, T, A> 63 | where &'a A: SharedMut> { 64 | fn drop(&mut self) { 65 | // Can only be None on a double drop! 66 | let value = self.value.take().unwrap(); 67 | 68 | // Panic instead of losing this value. 69 | self.pool.try_into_inner().unwrap().push(value) 70 | } 71 | } 72 | 73 | pub struct Pool Option, A = Unsync> { 74 | _type: PhantomData, 75 | allocated: A, 76 | builder: B, 77 | } 78 | 79 | pub type SyncPool Option> = Pool>; 80 | 81 | impl<'a, T, F: Fn() -> Option, C: Default + 'a> Pool 82 | where &'a C: SharedMut> { 83 | pub fn new(builder: F) -> Self { 84 | Self { 85 | _type: PhantomData::default(), 86 | allocated: C::default(), 87 | builder: builder, 88 | } 89 | } 90 | 91 | pub fn prealloc(&'a self, count: usize) 92 | -> StdResult<(), Option<<&'a C as SharedMut>>::Error>> { 93 | use std::collections::LinkedList; 94 | use std::convert::TryInto; 95 | 96 | let swap: LinkedList, _>> = (0..count).map(|_| 97 | self.try_into() 98 | ).collect(); 99 | for temp in swap { 100 | temp?; 101 | } 102 | Ok(()) 103 | } 104 | } 105 | 106 | impl<'a, T: Default, C: Default + 'a> Default for Pool Option, C> 107 | where &'a C: SharedMut> { 108 | fn default() -> Self { 109 | Self::new(|| Some(T::default())) 110 | } 111 | } 112 | 113 | pub type Result<'a, T, A = Unsync> = StdResult< 114 | Reusable<'a, T, A>, 115 | Option<<&'a A as SharedMut>>::Error>, 116 | >; 117 | 118 | pub type SyncResult<'a, T> = Result<'a, T, Sync>; 119 | 120 | #[doc(hidden)] 121 | pub trait SharedMut { 122 | type Okay: DerefMut; 123 | type Error: Debug; 124 | 125 | fn try_into_inner(self) -> StdResult; 126 | } 127 | 128 | impl<'a, T> SharedMut for &'a RefCell { 129 | type Okay = RefMut<'a, T>; 130 | type Error = BorrowMutError; 131 | 132 | fn try_into_inner(self) -> StdResult { 133 | self.try_borrow_mut() 134 | } 135 | } 136 | 137 | impl<'a, T> SharedMut for &'a Mutex { 138 | type Okay = MutexGuard<'a, T>; 139 | type Error = PoisonError; 140 | 141 | fn try_into_inner(self) -> StdResult { 142 | self.lock() 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/signals.rs: -------------------------------------------------------------------------------- 1 | use crate::reusable::SyncResult; 2 | 3 | use signal::Signal; 4 | 5 | static NOTIFICATION_SIGNALS: [Signal; 16] = [ 6 | Signal::Alarm, 7 | Signal::VirtualAlarm, 8 | Signal::ProfilingTimer, 9 | Signal::ProcessorLimit, 10 | Signal::FilesystemLimit, 11 | Signal::TerminalInput, 12 | Signal::TerminalOutput, 13 | Signal::PowerFailure, 14 | Signal::User1, 15 | Signal::User2, 16 | 17 | // A stretch... 18 | Signal::UrgentSocket, 19 | Signal::Pollable, 20 | Signal::Syscall, 21 | Signal::FloatingPoint, 22 | Signal::Hangup, 23 | Signal::Child, 24 | ]; 25 | 26 | pub fn assign_signal() -> SyncResult<'static, Signal> { 27 | use crate::compile_assert::assert_sync; 28 | use crate::reusable::SyncPool; 29 | 30 | use std::convert::TryInto; 31 | use std::sync::atomic::AtomicUsize; 32 | use std::sync::atomic::Ordering; 33 | use std::sync::Once; 34 | 35 | static mut SIGNALS: Option Option + Sync>>> = None; 36 | static INIT: Once = Once::new(); 37 | INIT.call_once(|| unsafe { 38 | let free = AtomicUsize::new(0); 39 | SIGNALS.replace(SyncPool::new(Box::new(move || 40 | NOTIFICATION_SIGNALS.get(free.fetch_add(1, Ordering::Relaxed)).copied() 41 | ))); 42 | }); 43 | 44 | let signals = unsafe { 45 | SIGNALS.as_ref() 46 | }.unwrap(); 47 | assert_sync(&signals); 48 | signals.try_into() 49 | } 50 | -------------------------------------------------------------------------------- /src/stacks.rs: -------------------------------------------------------------------------------- 1 | use crate::reusable::ReusableSync; 2 | 3 | use std::marker::PhantomData; 4 | use std::ops::Deref; 5 | use std::ops::DerefMut; 6 | use timetravel::stable::StableAddr; 7 | use timetravel::stable::StableMutAddr; 8 | 9 | pub fn alloc_stack() -> ReusableSync<'static, Box<[u8]>> { 10 | use crate::compile_assert::assert_sync; 11 | use crate::reusable::SyncPool; 12 | use super::STACK_SIZE_BYTES; 13 | 14 | use gotcha::Group; 15 | use std::convert::TryInto; 16 | use std::sync::Once; 17 | 18 | static mut STACKS: Option>> = None; 19 | static INIT: Once = Once::new(); 20 | INIT.call_once(|| { 21 | let stacks: fn() -> _ = || Some(vec![0; STACK_SIZE_BYTES].into_boxed_slice()); 22 | let stacks = SyncPool::new(stacks); 23 | stacks.prealloc(Group::limit()) 24 | .expect("libinger: stack allocator lock was poisoned during init"); 25 | unsafe { 26 | STACKS.replace(stacks); 27 | } 28 | }); 29 | 30 | let stacks = unsafe { 31 | STACKS.as_ref() 32 | }.unwrap(); 33 | assert_sync(&stacks); 34 | stacks.try_into().expect("libinger: stack allocator lock is poisoned") 35 | } 36 | 37 | pub struct DerefAdapter<'a, T> (T, PhantomData<&'a ()>); 38 | 39 | impl From for DerefAdapter<'_, T> { 40 | fn from(t: T) -> Self { 41 | Self (t, PhantomData::default()) 42 | } 43 | } 44 | 45 | impl<'a, T: Deref, U: Deref + 'a, V: ?Sized> Deref for DerefAdapter<'a, T> { 46 | type Target = V; 47 | 48 | fn deref(&self) -> &Self::Target { 49 | let Self (t, _) = self; 50 | &***t 51 | } 52 | } 53 | 54 | impl<'a, T: DerefMut, U: DerefMut + 'a, V: ?Sized> DerefMut for DerefAdapter<'a, T> { 55 | fn deref_mut(&mut self) -> &mut Self::Target { 56 | let Self (t, _) = self; 57 | &mut ***t 58 | } 59 | } 60 | 61 | unsafe impl<'a, T: Deref, U: StableAddr + 'a> StableAddr for DerefAdapter<'a, T> {} 62 | unsafe impl<'a, T: DerefMut, U: StableMutAddr + 'a> StableMutAddr for DerefAdapter<'a, T> {} 63 | -------------------------------------------------------------------------------- /src/tcb.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused)] 2 | 3 | use gotcha::prctl::ARCH_GET_CPUID; 4 | use gotcha::prctl::ARCH_GET_FS; 5 | use gotcha::prctl::ARCH_GET_GS; 6 | use gotcha::prctl::ARCH_SET_CPUID; 7 | use gotcha::prctl::ARCH_SET_FS; 8 | use gotcha::prctl::ARCH_SET_GS; 9 | use gotcha::Group; 10 | use std::io::Error; 11 | use std::io::Result; 12 | use std::os::raw::c_int; 13 | use std::os::raw::c_ulong; 14 | 15 | #[must_use] 16 | pub struct ThreadControlBlock (Option>); 17 | 18 | impl ThreadControlBlock { 19 | pub fn current() -> Result { 20 | unsafe { 21 | arch_prctl_get(GetOp::Fs).map(|fs| Self (Some(MaybeMut::Ref(fs)))) 22 | } 23 | } 24 | 25 | pub fn new() -> Self { 26 | extern { 27 | fn _dl_allocate_tls(_: Option<&mut TCB>) -> Option<&mut TCB>; 28 | } 29 | 30 | #[repr(C)] 31 | struct TCB { 32 | tls_ptr: usize, 33 | _unused: usize, 34 | self_ptr: usize, 35 | } 36 | 37 | let fs = unsafe { 38 | _dl_allocate_tls(None) 39 | }.expect("libinger: could not allocate thread-control block"); 40 | let auto: *mut _ = fs; 41 | fs.tls_ptr = auto as _; 42 | fs.self_ptr = auto as _; 43 | 44 | let auto: *mut _ = auto as _; 45 | Self (Some(MaybeMut::Mut(unsafe { 46 | &mut *auto 47 | }))) 48 | } 49 | 50 | pub unsafe fn install(mut self, group: Group) -> Result { 51 | let parent = unguarded_parent(self.install_unguarded(group.into()))?; 52 | Ok(ThreadControlBlockGuard { 53 | this: self, 54 | parent, 55 | }) 56 | } 57 | 58 | unsafe fn install_unguarded(&mut self, group: Option) -> Result> { 59 | use crate::linger::abort; 60 | 61 | use gotcha::group_lookup_symbol_fn; 62 | use std::slice; 63 | extern { 64 | fn __ctype_init(); 65 | } 66 | 67 | const POINTER_GUARD: usize = 6; 68 | const KERNEL_THREAD: usize = 90; 69 | 70 | let Self (fs) = self; 71 | let fs = fs.as_mut().unwrap(); 72 | let mut cur = None; 73 | let mut custom = false; 74 | if let MaybeMut::Mut(fs) = fs { 75 | let fs = unsafe { 76 | slice::from_raw_parts_mut(*fs, KERNEL_THREAD + 1) 77 | }; 78 | let cur = cur.get_or_insert(Self::current()?); 79 | let Self (cur) = &cur; 80 | let cur: &_ = cur.as_ref().unwrap().into(); 81 | let cur = unsafe { 82 | slice::from_raw_parts(cur, KERNEL_THREAD + 1) 83 | }; 84 | fs[POINTER_GUARD] = cur[POINTER_GUARD]; 85 | fs[KERNEL_THREAD] = cur[KERNEL_THREAD]; 86 | custom = true; 87 | } 88 | 89 | let fs = (&*fs).into(); 90 | arch_prctl_set(SetOp::Fs, fs)?; 91 | if custom { 92 | __ctype_init(); 93 | if let Some(group) = group { 94 | let __ctype_init: Option = group_lookup_symbol_fn!(group, __ctype_init); 95 | if let Some(__ctype_init) = __ctype_init { 96 | __ctype_init(); 97 | } else { 98 | abort("install(): could not get address of __ctype_init()"); 99 | } 100 | } 101 | } 102 | Ok(cur) 103 | } 104 | 105 | fn take(&mut self) -> Option> { 106 | let Self (this) = self; 107 | this.take() 108 | } 109 | } 110 | 111 | impl Drop for ThreadControlBlock { 112 | fn drop(&mut self) { 113 | let Self (this) = self; 114 | if let Some(MaybeMut::Mut(_)) = this.as_mut() { 115 | if let Ok (parent) = unguarded_parent(unsafe { 116 | self.install_unguarded(None) 117 | }) { 118 | drop(ThreadControlBlockGuard { 119 | this: ThreadControlBlock (self.take()), 120 | parent, 121 | }); 122 | } else { 123 | eprintln!("libinger: could not install TCB to run TLS destructors"); 124 | } 125 | } 126 | let Self (fs) = self; 127 | } 128 | } 129 | 130 | fn unguarded_parent(this: Result>) -> Result { 131 | this?.ok_or(()).or_else(|_| ThreadControlBlock::current()) 132 | } 133 | 134 | #[must_use] 135 | pub struct ThreadControlBlockGuard { 136 | this: ThreadControlBlock, 137 | parent: ThreadControlBlock, 138 | } 139 | 140 | impl ThreadControlBlockGuard { 141 | pub unsafe fn uninstall(mut self) -> Result { 142 | Ok(ThreadControlBlock (self.this.take())) 143 | } 144 | } 145 | 146 | impl Drop for ThreadControlBlockGuard { 147 | fn drop(&mut self) { 148 | extern { 149 | fn __call_tls_dtors(); 150 | fn _dl_deallocate_tls(_: &mut usize, _: bool); 151 | } 152 | 153 | let mut dealloc = None; 154 | if let Some(MaybeMut::Mut(fs)) = self.this.take() { 155 | unsafe { 156 | __call_tls_dtors(); 157 | } 158 | dealloc = Some(fs); 159 | } 160 | unsafe { 161 | self.parent.install_unguarded(None).unwrap(); 162 | } 163 | if let Some(fs) = dealloc { 164 | unsafe { 165 | _dl_deallocate_tls(fs, true); 166 | } 167 | } 168 | } 169 | } 170 | 171 | enum MaybeMut<'a> { 172 | Ref(&'a usize), 173 | Mut(&'a mut usize), 174 | } 175 | 176 | impl<'a> From<&'a MaybeMut<'a>> for &'a usize { 177 | fn from(other: &'a MaybeMut) -> Self { 178 | match other { 179 | MaybeMut::Ref(other) => other, 180 | MaybeMut::Mut(other) => other, 181 | } 182 | } 183 | } 184 | 185 | enum GetOp { 186 | Cpuid = ARCH_GET_CPUID as _, 187 | Fs = ARCH_GET_FS as _, 188 | Gs = ARCH_GET_GS as _, 189 | } 190 | 191 | enum SetOp { 192 | Cpuid = ARCH_SET_CPUID as _, 193 | Fs = ARCH_SET_FS as _, 194 | Gs = ARCH_SET_GS as _, 195 | } 196 | 197 | unsafe fn arch_prctl_get<'a>(op: GetOp) -> Result<&'a usize> { 198 | use std::mem::MaybeUninit; 199 | extern { 200 | fn arch_prctl(_: c_int, _: *mut c_ulong) -> c_int; 201 | } 202 | 203 | let mut addr = MaybeUninit::uninit(); 204 | if arch_prctl(op as _, addr.as_mut_ptr()) == 0 { 205 | let addr: *const _ = addr.assume_init() as _; 206 | Ok(&*addr) 207 | } else { 208 | Err(Error::last_os_error()) 209 | } 210 | } 211 | 212 | unsafe fn arch_prctl_set(op: SetOp, val: &usize) -> Result<()> { 213 | extern { 214 | fn libgotcha_arch_prctl(_: c_int, _: c_ulong) -> c_int; 215 | } 216 | 217 | let val: *const _ = val; 218 | if libgotcha_arch_prctl(op as _, val as _) == 0 { 219 | Ok(()) 220 | } else { 221 | Err(Error::last_os_error()) 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /src/tcbstub.rs: -------------------------------------------------------------------------------- 1 | use gotcha::Group; 2 | use std::io::Result; 3 | 4 | #[must_use] 5 | pub struct ThreadControlBlock; 6 | 7 | impl ThreadControlBlock { 8 | pub fn new() -> Self { Self } 9 | pub unsafe fn install(self, _: Group) -> Result { Ok(ThreadControlBlockGuard) } 10 | } 11 | 12 | impl Drop for ThreadControlBlock { 13 | fn drop(&mut self) {} 14 | } 15 | 16 | #[must_use] 17 | pub struct ThreadControlBlockGuard; 18 | 19 | impl ThreadControlBlockGuard { 20 | pub unsafe fn uninstall(self) -> Result { Ok(ThreadControlBlock) } 21 | } 22 | 23 | impl Drop for ThreadControlBlockGuard { 24 | fn drop(&mut self) {} 25 | } 26 | -------------------------------------------------------------------------------- /src/timer.rs: -------------------------------------------------------------------------------- 1 | use libc::CLOCK_BOOTTIME; 2 | use libc::CLOCK_BOOTTIME_ALARM; 3 | use libc::CLOCK_MONOTONIC; 4 | use libc::CLOCK_PROCESS_CPUTIME_ID; 5 | use libc::CLOCK_REALTIME; 6 | use libc::CLOCK_REALTIME_ALARM; 7 | use libc::CLOCK_THREAD_CPUTIME_ID; 8 | pub use libc::itimerspec; 9 | use libc::sigevent; 10 | use signal::Signal; 11 | use std::io::Error; 12 | use std::io::Result; 13 | use std::mem::MaybeUninit; 14 | use std::os::raw::c_int; 15 | 16 | #[allow(dead_code)] 17 | pub enum Clock { 18 | Boot = CLOCK_BOOTTIME as _, 19 | BootAlarm = CLOCK_BOOTTIME_ALARM as _, 20 | Mono = CLOCK_MONOTONIC as _, 21 | Process = CLOCK_PROCESS_CPUTIME_ID as _, 22 | Real = CLOCK_REALTIME as _, 23 | RealAlarm = CLOCK_REALTIME_ALARM as _, 24 | Thread = CLOCK_THREAD_CPUTIME_ID as _, 25 | } 26 | 27 | #[repr(transparent)] 28 | pub struct Sigevent (sigevent); 29 | 30 | impl Sigevent { 31 | #[allow(dead_code)] 32 | pub fn none() -> Self { 33 | use libc::SIGEV_NONE; 34 | 35 | Self (Self::new(SIGEV_NONE)) 36 | } 37 | 38 | #[allow(dead_code)] 39 | pub fn signal(signal: Signal) -> Self { 40 | use libc::SIGEV_SIGNAL; 41 | 42 | let mut this = Self::new(SIGEV_SIGNAL); 43 | this.sigev_signo = signal as _; 44 | Self (this) 45 | } 46 | 47 | #[allow(dead_code)] 48 | pub fn thread_id(signal: Signal, thread: c_int) -> Self { 49 | use libc::SIGEV_THREAD_ID; 50 | 51 | let mut this = Self::new(SIGEV_THREAD_ID); 52 | this.sigev_signo = signal as _; 53 | this.sigev_notify_thread_id = thread; 54 | Self (this) 55 | } 56 | 57 | fn new(notify: c_int) -> sigevent { 58 | let mut event: sigevent = unsafe { 59 | uninitialized() 60 | }; 61 | event.sigev_notify = notify; 62 | event 63 | } 64 | } 65 | 66 | #[derive(Clone, Copy)] 67 | #[repr(transparent)] 68 | pub struct Timer (usize); 69 | 70 | pub fn timer_create(clockid: Clock, sevp: &mut Sigevent) -> Result { 71 | extern { 72 | fn timer_create(_: c_int, _: *mut sigevent, _: *mut Timer) -> c_int; 73 | } 74 | 75 | let mut timer = MaybeUninit::uninit(); 76 | let Sigevent (sevp) = sevp; 77 | if unsafe { 78 | timer_create(clockid as _, sevp, timer.as_mut_ptr()) 79 | } != 0 { 80 | Err(Error::last_os_error())?; 81 | } 82 | 83 | Ok(unsafe { 84 | timer.assume_init() 85 | }) 86 | } 87 | 88 | pub fn timer_settime(timerid: Timer, absolute: bool, new: &itimerspec, old: Option<&mut itimerspec>) -> Result<()> { 89 | use libc::TIMER_ABSTIME; 90 | extern { 91 | fn timer_settime(_: Timer, _: c_int, _: *const itimerspec, _: Option<&mut itimerspec>) -> c_int; 92 | } 93 | 94 | let absolute = if absolute { TIMER_ABSTIME } else { 0 }; 95 | if unsafe { 96 | timer_settime(timerid, absolute, new, old) 97 | } != 0 { 98 | Err(Error::last_os_error())?; 99 | } 100 | 101 | Ok(()) 102 | } 103 | 104 | #[allow(dead_code)] 105 | pub fn timer_delete(timerid: Timer) -> Result<()> { 106 | extern { 107 | fn timer_delete(_: Timer) -> c_int; 108 | } 109 | 110 | if unsafe { 111 | timer_delete(timerid) 112 | } != 0 { 113 | Err(Error::last_os_error())?; 114 | } 115 | 116 | Ok(()) 117 | } 118 | 119 | unsafe fn uninitialized() -> T { 120 | MaybeUninit::uninit().assume_init() 121 | } 122 | -------------------------------------------------------------------------------- /src/unfurl.rs: -------------------------------------------------------------------------------- 1 | pub trait Unfurl { 2 | unsafe fn unfurl(self) -> T; 3 | } 4 | 5 | impl Unfurl for Option { 6 | unsafe fn unfurl(self) -> T { 7 | use std::hint::unreachable_unchecked; 8 | 9 | if let Some(t) = self { 10 | t 11 | } else { 12 | unreachable_unchecked() 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tests/inger/lock.rs: -------------------------------------------------------------------------------- 1 | use std::sync::MutexGuard; 2 | 3 | pub fn exclusive(fun: fn() -> T) { 4 | let lock = lock(); 5 | fun(); 6 | drop(lock); 7 | } 8 | 9 | fn lock() -> MutexGuard<'static, ()> { 10 | use std::sync::Mutex; 11 | use std::sync::Once; 12 | 13 | static INIT: Once = Once::new(); 14 | static mut LOCK: Option> = None; 15 | 16 | INIT.call_once(|| unsafe { 17 | LOCK.replace(Mutex::new(())); 18 | }); 19 | 20 | // The lock might be poisened because a previous test failed. This is safe to ignore 21 | // because we should no longer have a race (since the other test's thread is now 22 | // dead) and we don't need to fail the current test as well. 23 | unsafe { 24 | LOCK.as_ref() 25 | }.unwrap().lock().unwrap_or_else(|poison| poison.into_inner()) 26 | } 27 | -------------------------------------------------------------------------------- /tests/inger/main.rs: -------------------------------------------------------------------------------- 1 | #![cfg_attr(bench, feature(test))] 2 | #[cfg(bench)] 3 | extern crate test; 4 | 5 | mod lock; 6 | 7 | use lock::exclusive; 8 | 9 | use inger::launch; 10 | use inger::nsnow; 11 | use inger::resume; 12 | #[cfg(bench)] 13 | use test::Bencher; 14 | 15 | #[test] 16 | fn launch_completion() { 17 | exclusive(|| 18 | assert!(launch(|| (), 1_000).unwrap().is_completion()) 19 | ); 20 | } 21 | 22 | #[test] 23 | fn launch_continuation() { 24 | exclusive(|| 25 | assert!(launch(|| timeout(1_000_000), 10).unwrap().is_continuation()) 26 | ); 27 | } 28 | 29 | #[test] 30 | fn launch_union() { 31 | exclusive(|| 32 | launch(|| -> Result> { Ok(false) }, 1_000).unwrap() 33 | ); 34 | } 35 | 36 | #[should_panic(expected = "PASS")] 37 | #[test] 38 | fn launch_panic() { 39 | exclusive(|| 40 | drop(launch(|| panic!("PASS"), 1_000)) 41 | // Lock becomes poisoned. 42 | ); 43 | } 44 | 45 | #[ignore] 46 | #[should_panic(expected = "PASS")] 47 | #[test] 48 | fn launch_panic_outer() { 49 | exclusive(|| 50 | drop(launch(|| { 51 | drop(launch(|| (), 1_000)); 52 | panic!("PASS"); 53 | }, 1_000)) 54 | // Lock becomes poisoned. 55 | ); 56 | } 57 | 58 | #[ignore] 59 | #[should_panic(expected = "PASS")] 60 | #[test] 61 | fn launch_panic_inner() { 62 | exclusive(|| 63 | drop(launch(|| drop(launch(|| panic!("PASS"), 1_000)), 1_000)) 64 | // Lock becomes poisoned. 65 | ); 66 | } 67 | 68 | #[ignore] 69 | #[test] 70 | fn launch_completions() { 71 | exclusive(|| 72 | assert!(launch(|| assert!(launch(|| (), 1_000).unwrap().is_completion()), 1_000).unwrap().is_completion()) 73 | ); 74 | } 75 | 76 | #[ignore] 77 | #[test] 78 | fn launch_continuations() { 79 | exclusive(|| { 80 | assert!(launch(|| { 81 | assert!(launch(|| timeout(1_000_000), 10).unwrap().is_continuation()); 82 | timeout(1_000_000); 83 | }, 1_000).unwrap().is_continuation()); 84 | }); 85 | } 86 | 87 | #[ignore] 88 | #[test] 89 | fn resume_completion() { 90 | exclusive(|| { 91 | let mut cont = launch(|| timeout(1_000_000), 10).unwrap(); 92 | assert!(cont.is_continuation(), "completion instead of continuation"); 93 | assert!(resume(&mut cont, 10_000_000).unwrap().is_completion()); 94 | }); 95 | } 96 | 97 | #[ignore] 98 | #[test] 99 | fn resume_completion_drop() { 100 | exclusive(|| { 101 | let mut cont = launch(|| timeout(1_000_000), 100).unwrap(); 102 | assert!(cont.is_continuation(), "completion instead of continuation"); 103 | assert!(resume(&mut cont, 10_000).unwrap().is_continuation()); 104 | }); 105 | } 106 | 107 | #[ignore] 108 | #[test] 109 | fn resume_completion_repeat() { 110 | exclusive(|| { 111 | let mut cont = launch(|| timeout(1_000_000), 10).unwrap(); 112 | assert!(cont.is_continuation(), "launch(): returned completion instead of continuation"); 113 | resume(&mut cont, 10).unwrap(); 114 | assert!(cont.is_continuation(), "resume(): returned completion instead of continuation"); 115 | assert!(resume(&mut cont, 10_000_000).unwrap().is_completion()); 116 | }); 117 | } 118 | 119 | #[test] 120 | fn setup_only() { 121 | use std::sync::atomic::AtomicBool; 122 | use std::sync::atomic::Ordering; 123 | 124 | exclusive(|| { 125 | let run = AtomicBool::new(false); 126 | let mut prep = launch(|| run.store(true, Ordering::Relaxed), 0).unwrap(); 127 | assert!(! run.load(Ordering::Relaxed)); 128 | resume(&mut prep, 1_000).unwrap(); 129 | assert!(run.load(Ordering::Relaxed)); 130 | }); 131 | } 132 | 133 | #[should_panic(expected = "launch(): too many active timed functions: None")] 134 | #[test] 135 | fn launch_toomany() { 136 | exclusive(|| { 137 | use std::collections::LinkedList; 138 | 139 | let mut orphans = LinkedList::default(); 140 | loop { 141 | orphans.push_back(launch(|| timeout(1_000_000), 0).unwrap()); 142 | // Lock becomes poisoned. 143 | } 144 | }); 145 | } 146 | 147 | #[test] 148 | fn launch_toomany_reinit() { 149 | exclusive(|| { 150 | let thing_one = launch(|| timeout(1_000_000), 0).unwrap(); 151 | let _thing_two = launch(|| timeout(1_000_000), 0).unwrap(); 152 | drop(thing_one); 153 | let _thing_three = launch(|| timeout(1_000_000), 0).unwrap(); 154 | }); 155 | } 156 | 157 | #[ignore] 158 | #[test] 159 | fn abuse_preemption() { 160 | for _ in 0..25 { 161 | launch_continuation(); 162 | } 163 | } 164 | 165 | fn timeout(mut useconds: u64) { 166 | useconds *= 1_000; 167 | 168 | let mut elapsed = 0; 169 | let mut last = nsnow(); 170 | while elapsed < useconds { 171 | let mut this = nsnow(); 172 | while this < last || this - last > 1_000 { 173 | last = this; 174 | this = nsnow(); 175 | } 176 | elapsed += this - last; 177 | last = this; 178 | } 179 | } 180 | 181 | #[bench] 182 | #[cfg(bench)] 183 | fn timeout_10(lo: &mut Bencher) { 184 | lo.iter(|| timeout(10)); 185 | } 186 | 187 | #[bench] 188 | #[cfg(bench)] 189 | fn timeout_100(lo: &mut Bencher) { 190 | lo.iter(|| timeout(100)); 191 | } 192 | 193 | #[bench] 194 | #[cfg(bench)] 195 | fn timeout_1000(lo: &mut Bencher) { 196 | lo.iter(|| timeout(1_000)); 197 | } 198 | -------------------------------------------------------------------------------- /testsuite/.gitignore: -------------------------------------------------------------------------------- 1 | /** 2 | 3 | !/.gitignore 4 | !/build 5 | !/custom-glibc.patch 6 | !/test 7 | !/testinger.c 8 | -------------------------------------------------------------------------------- /testsuite/build: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | readonly LIBINGER=".." 4 | 5 | if [ "$#" -eq "0" -o \( "$1" != "release" -a "$1" != "debug" -a -z "`echo "$1" | grep ^-`" \) ] 6 | then 7 | echo "USAGE: $0 ...\"> [cc flag]..." 8 | exit 1 9 | fi 10 | cargoflags="$1" 11 | shift 12 | buildtype="debug" 13 | cflags="-g3 -Og" 14 | case "$cargoflags" in 15 | -*-release*) 16 | buildtype="release" 17 | ;; 18 | release) 19 | buildtype="release" 20 | cargoflags="--$cargoflags" 21 | ;; 22 | debug) 23 | cargoflags="" 24 | ;; 25 | esac 26 | if [ "$buildtype" = "release" ] 27 | then 28 | cflags="-O2" 29 | fi 30 | 31 | cd "`dirname "$0"`" 32 | cd "$LIBINGER" 33 | 34 | set -ve 35 | ./configure || true 36 | mkdir -p "target/$buildtype" 37 | cargo build $cargoflags 38 | cargo run $cargoflags >"target/$buildtype/libinger.h" 39 | cp external/libgotcha/libgotcha_api.h "target/$buildtype/libgotcha.h" 40 | cp external/libgotcha/libgotcha_repl.h "target/$buildtype" 41 | objcopy -Wsignal --globalize-symbol libgotcha_dlsym --globalize-symbol libgotcha_signal "target/$buildtype/deps/libgotcha-"*.rlib 2>/dev/null 42 | rm "target/$buildtype/deps/libinger.so" 43 | cd - 44 | c99 $cflags -Wall -Wextra -Wpedantic -Werror "$@" -c -fpic -fno-optimize-sibling-calls -D_GNU_SOURCE -Wno-missing-attributes -I"$OLDPWD/target/$buildtype" testinger.c 45 | cd - 46 | cargo rustc $cargoflags --lib -- -Clink-arg="$OLDPWD/testinger.o" 47 | cd - 48 | mv "$OLDPWD/target/$buildtype/libinger.so" libtestinger.so 49 | rm "$OLDPWD/target/$buildtype/deps/libinger.so" 50 | rm testinger.o 51 | cd - 52 | ./configure >/dev/null || true 53 | -------------------------------------------------------------------------------- /testsuite/custom-glibc.patch: -------------------------------------------------------------------------------- 1 | diff --git i/testsuite/build w/testsuite/build 2 | index cf760c0..fe575b8 100755 3 | --- i/testsuite/build 4 | +++ w/testsuite/build 5 | @@ -11,6 +11,7 @@ cargoflags="$1" 6 | shift 7 | buildtype="debug" 8 | cflags="-g3 -Og" 9 | +rustflags="" 10 | case "$cargoflags" in 11 | -*-release*) 12 | buildtype="release" 13 | @@ -27,6 +28,10 @@ if [ "$buildtype" = "release" ] 14 | then 15 | cflags="-O2" 16 | fi 17 | +if [ -e lib ] 18 | +then 19 | + rustflags="-Clink-arg=-L$PWD/lib" 20 | +fi 21 | 22 | cd "`dirname "$0"`" 23 | cd "$LIBINGER" 24 | @@ -34,8 +39,11 @@ cd "$LIBINGER" 25 | set -ve 26 | ./configure || true 27 | mkdir -p "target/$buildtype" 28 | -cargo build $cargoflags 29 | cargo run $cargoflags >"target/$buildtype/libinger.h" 30 | + 31 | +export RUSTFLAGS="-Cprefer-dynamic=no" 32 | +cargo build $cargoflags --lib 33 | +cargo build $cargoflags --lib 34 | cp external/libgotcha/libgotcha_api.h "target/$buildtype/libgotcha.h" 35 | cp external/libgotcha/libgotcha_repl.h "target/$buildtype" 36 | objcopy -Wsignal --globalize-symbol libgotcha_dlsym --globalize-symbol libgotcha_signal "target/$buildtype/deps/libgotcha-"*.rlib 2>/dev/null 37 | @@ -43,7 +51,7 @@ rm "target/$buildtype/deps/libinger.so" 38 | cd - 39 | c99 $cflags -Wall -Wextra -Wpedantic -Werror "$@" -c -fpic -fno-optimize-sibling-calls -D_GNU_SOURCE -Wno-missing-attributes -I"$OLDPWD/target/$buildtype" testinger.c 40 | cd - 41 | -cargo rustc $cargoflags --lib -- -Clink-arg="$OLDPWD/testinger.o" 42 | +eval cargo rustc "$cargoflags" --lib -- -Clink-arg="$OLDPWD/testinger.o" "$rustflags" "`sed -n -e's/",/"/g' -e's/^rustflags = \[\(.\+\)\]$/\1/p' .cargo/config`" 43 | cd - 44 | mv "$OLDPWD/target/$buildtype/libinger.so" libtestinger.so 45 | rm "$OLDPWD/target/$buildtype/deps/libinger.so" 46 | diff --git i/testsuite/test w/testsuite/test 47 | index cfef791..0c83013 100755 48 | --- i/testsuite/test 49 | +++ w/testsuite/test 50 | @@ -1,5 +1,7 @@ 51 | #!/bin/sh 52 | 53 | +readonly VERSION="2.29" 54 | + 55 | GNULIB="$*" 56 | if [ -z "$GNULIB" ] 57 | then 58 | @@ -7,9 +9,38 @@ then 59 | fi 60 | 61 | set -ve 62 | -[ ! -e libtestinger.so ] && ./build release 63 | [ ! -e gnulib/configure ] && "$GNULIB/gnulib-tool" --create-testdir --dir gnulib --single-configure `"$GNULIB/posix-modules"` 64 | -[ ! -e Makefile ] && gnulib/configure CFLAGS="-fpic -g3" 65 | +if [ ! -e Makefile ] 66 | +then 67 | + cflags="" 68 | + ldflags="" 69 | + version="`ldd --version | head -n1 | rev | cut -d" " -f1 | rev`" 70 | + if [ "$version" != "$VERSION" ] 71 | + then 72 | + echo >&2 73 | + echo "!!! It looks like your system uses glibc $version." >&2 74 | + echo "!!! We recommend running this suite on version $VERSION!" >&2 75 | + echo >&2 76 | + printf %s "Path to an alternative ld-linux.so (enter to use system's)? " 77 | + read interp 78 | + if [ -n "$interp" ] 79 | + then 80 | + interp="`realpath "$interp"`" 81 | + ldflags="-Wl,-I$interp" 82 | + 83 | + lib="`dirname "$interp"`" 84 | + ldflags="-L$lib $ldflags" 85 | + ln -s "$lib" . 86 | + rm -f libtestinger.so 87 | + 88 | + cflags="-I." 89 | + mkdir sys 90 | + echo "#error" >sys/single_threaded.h 91 | + fi 92 | + fi 93 | + gnulib/configure CFLAGS="-fpic -g3 $cflags" LDFLAGS="$ldflags" 94 | +fi 95 | +[ ! -e libtestinger.so ] && ./build release 96 | make -j"`getconf _NPROCESSORS_ONLN`" 97 | [ ! -e gltests/test-suite.log ] && make check || true 98 | make check LD_PRELOAD="$PWD/libtestinger.so" LIBGOTCHA_NUMGROUPS="1" LIBGOTCHA_SKIP="`cat <<-tac 99 | -------------------------------------------------------------------------------- /testsuite/test: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | GNULIB="$*" 4 | if [ -z "$GNULIB" ] 5 | then 6 | GNULIB="/usr/share/gnulib" 7 | fi 8 | 9 | set -ve 10 | [ ! -e libtestinger.so ] && ./build release 11 | [ ! -e gnulib/configure ] && "$GNULIB/gnulib-tool" --create-testdir --dir gnulib --single-configure `"$GNULIB/posix-modules"` 12 | [ ! -e Makefile ] && gnulib/configure CFLAGS="-fpic -g3" 13 | make -j"`getconf _NPROCESSORS_ONLN`" 14 | [ ! -e gltests/test-suite.log ] && make check || true 15 | make check LD_PRELOAD="$PWD/libtestinger.so" LIBGOTCHA_NUMGROUPS="1" LIBGOTCHA_SKIP="`cat <<-tac 16 | /bin/bash 17 | /usr/bin/cat 18 | /usr/bin/chmod 19 | /usr/bin/cmp 20 | /usr/bin/diff 21 | /usr/bin/env 22 | /usr/bin/expr 23 | /usr/bin/gawk 24 | /usr/bin/grep 25 | /usr/bin/head 26 | /usr/bin/make 27 | /usr/bin/mkdir 28 | /usr/bin/mv 29 | /usr/bin/sed 30 | /usr/bin/sh 31 | /usr/bin/sleep 32 | /usr/bin/rm 33 | /usr/bin/tr 34 | /usr/bin/wc 35 | tac`" 36 | -------------------------------------------------------------------------------- /testsuite/testinger.c: -------------------------------------------------------------------------------- 1 | #include "libgotcha.h" 2 | #include "libgotcha_repl.h" 3 | #include "libinger.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | struct inout { 17 | int argc; 18 | char **argv; 19 | char **envp; 20 | int retval; 21 | }; 22 | 23 | static int (*mainfunc)(int, char **, char **); 24 | 25 | static void testinging(void *inout) { 26 | struct inout *argret = inout; 27 | argret->retval = mainfunc(argret->argc, argret->argv, argret->envp); 28 | } 29 | 30 | static int testinger(int argc, char **argv, char **envp) { 31 | struct inout argret = { 32 | .argc = argc, 33 | .argv = argv, 34 | .envp = envp, 35 | }; 36 | launch(testinging, UINT64_MAX, &argret); 37 | return argret.retval; 38 | } 39 | 40 | #pragma weak libtestinger_libc_start_main = __libc_start_main 41 | int __libc_start_main(int (*main)(int, char **, char **), int argc, char **argv, int (*init)(int, char **, char **), void (*fini)(void), void (*rtld_fini)(void), void *stack_end) { 42 | const char *skiplist = getenv("LIBGOTCHA_SKIP"); 43 | if(skiplist && strstr(skiplist, *argv)) 44 | return __libc_start_main(main, argc, argv, init, fini, rtld_fini, stack_end); 45 | 46 | struct link_map *lm = dlmopen(1, *argv, RTLD_LAZY); 47 | dlclose(lm); 48 | 49 | const struct link_map *l = dlopen(NULL, RTLD_LAZY); 50 | uintptr_t offset = (uintptr_t) main - l->l_addr; 51 | mainfunc = (int (*)(int, char **, char **)) (lm->l_addr + offset); 52 | 53 | return __libc_start_main(testinger, argc, argv, init, fini, rtld_fini, stack_end); 54 | } 55 | 56 | #pragma weak libtestinger_signal = signal 57 | void (*signal(int signum, void (*handler)(int)))(int) { 58 | if(handler == SIG_DFL) 59 | return handler; 60 | 61 | return libgotcha_signal(signum, handler); 62 | } 63 | 64 | static bool intrsleep; 65 | 66 | #pragma weak libtestinger_alarm = alarm 67 | unsigned int alarm(unsigned int seconds) { 68 | intrsleep = true; 69 | return alarm(seconds); 70 | } 71 | 72 | #pragma weak libtestinger_nanosleep = nanosleep 73 | int nanosleep(const struct timespec *req, struct timespec *rem) { 74 | // This should really be preemptible, but that causes the gnulib testsuite to fail some 75 | // fragile timing assertions. 76 | //libgotcha_group_thread_set(libgotcha_group_caller()); 77 | 78 | struct timespec ours; 79 | if(!rem) 80 | rem = &ours; 81 | 82 | int stat = nanosleep(req, rem); 83 | if(intrsleep) { 84 | intrsleep = false; 85 | return stat; 86 | } 87 | while(stat && errno == EINTR) { 88 | struct timespec next; 89 | memcpy(&next, rem, sizeof next); 90 | stat = nanosleep(&next, rem); 91 | } 92 | return stat; 93 | } 94 | 95 | int libtestinger_nanosleep(const struct timespec *, struct timespec *); 96 | 97 | #pragma weak libtestinger_usleep = usleep 98 | int usleep(useconds_t usec) { 99 | struct timespec nsec = { 100 | .tv_sec = usec / 1000000, 101 | .tv_nsec = (usec % 1000000) * 1000, 102 | }; 103 | return libtestinger_nanosleep(&nsec, NULL); 104 | } 105 | 106 | #pragma weak libtestinger_sleep = sleep 107 | unsigned int sleep(unsigned int seconds) { 108 | struct timespec nsec = { 109 | .tv_sec = seconds, 110 | }; 111 | return libtestinger_nanosleep(&nsec, NULL); 112 | } 113 | 114 | #pragma weak libtestinger_write = write 115 | ssize_t write(int fd, const void *buf, size_t count) { 116 | // This should really be preemptible, but that causes the gnulib testsuite to fail some 117 | // fragile timing assertions. 118 | //libgotcha_group_thread_set(libgotcha_group_caller()); 119 | 120 | int flags = fcntl(fd, F_GETFL); 121 | if(flags & O_NONBLOCK) 122 | return write(fd, buf, count); 123 | 124 | size_t written = 0; 125 | while(written != count) { 126 | ssize_t update = write(fd, (void *) ((uintptr_t) buf + written), count - written); 127 | if(update < 0) 128 | return update; 129 | written += update; 130 | } 131 | return written; 132 | } 133 | 134 | #pragma weak libtestinger_select = select 135 | int select(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, struct timeval *timeout) { 136 | libgotcha_group_thread_set(libgotcha_group_caller()); 137 | 138 | int res = 0; 139 | while((res = select(nfds, readfds, writefds, exceptfds, timeout)) < 0 && errno == EINTR); 140 | return res; 141 | } 142 | 143 | void _dl_signal_error(int, const char *, const char *, const char *); 144 | void libtestinger_dl_signal_exception(int error, const char *const *module, const char *message); 145 | 146 | extern const char *__progname; 147 | 148 | // It seems that glibc has a bug: __libc_dlsym() calls from ancillary namespaces abort the process 149 | // if they cannot find the target symbol, even if they would ordinarily only return an error code! 150 | // This happens because _dl_signal_cexception() calls are always redirected back to the base 151 | // namespace, so we work around it by proxying them and redirecting to the correct namespace. 152 | #pragma weak libtestinger_dl_signal_exception = _dl_signal_exception 153 | void _dl_signal_exception(int error, const char *const *module, const char *message) { 154 | // If we're still bootstrapping dlsym(), just jump back to libc however we can get there! 155 | if(dlsym == libgotcha_dlsym) { 156 | if(_dl_signal_exception == libtestinger_dl_signal_exception) 157 | _dl_signal_error(error, module[0], message, module[1]); 158 | else 159 | _dl_signal_exception(error, module, message); 160 | } 161 | 162 | // Otherwise, jump to the copy of libc in the namespace we came from. 163 | libgotcha_group_t group = libgotcha_group_caller(); 164 | void (*_dl_signal_exception)(int, const char *const *, const char *) = 165 | (void (*)(int, const char *const *, const char *)) (uintptr_t) 166 | libgotcha_group_symbol_from(group, "_dl_signal_exception", "libc.so.6"); 167 | libgotcha_group_thread_set(group); 168 | assert(_dl_signal_exception); 169 | assert(_dl_signal_exception != libtestinger_dl_signal_exception); 170 | if(mainfunc) 171 | fprintf(stderr, "./%s: symbol lookup warning: %s: %s (code %d)\n", __progname, *module, message, error); 172 | _dl_signal_exception(error, module, message); 173 | } 174 | --------------------------------------------------------------------------------