├── vis ├── __init__.py ├── demo.png ├── __main__.py └── template.html ├── .gitignore ├── examples ├── fork-buf.py ├── hello.py ├── parallel-inc.py ├── xv6-log.py ├── fs-crash.py ├── tocttou.py ├── cond-var.py └── _reproduce.py ├── LICENSE ├── README.md └── mosaic.py /vis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *~ 3 | *.pyc 4 | __pycache__ 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /vis/demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiangyy/mosaic/HEAD/vis/demo.png -------------------------------------------------------------------------------- /examples/fork-buf.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | heap.buf = '' 3 | for _ in range(N): 4 | pid = sys_fork() 5 | sys_sched() 6 | heap.buf += f'️{pid}\n' 7 | sys_write(heap.buf) 8 | -------------------------------------------------------------------------------- /examples/hello.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | pid = sys_fork() 3 | sys_sched() # non-deterministic context switch 4 | if pid == 0: 5 | sys_write('World\n') 6 | else: 7 | sys_write('Hello\n') 8 | -------------------------------------------------------------------------------- /examples/parallel-inc.py: -------------------------------------------------------------------------------- 1 | def Tworker(): 2 | for _ in range(N): 3 | tmp = heap.tot 4 | sys_sched() 5 | heap.tot = tmp + 1 6 | sys_sched() 7 | 8 | def main(): 9 | heap.tot = 0 10 | for _ in range(T): 11 | sys_spawn(Tworker) 12 | -------------------------------------------------------------------------------- /examples/xv6-log.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | sys_bwrite(0, {}) 3 | sys_sync() 4 | 5 | # 1. log the write to block #B 6 | log = sys_bread(0) # read log head 7 | for i in range(N): 8 | free = max(log.values(), default=0) + 1 # allocate log block 9 | sys_bwrite(free, f'contents for #{i+1}') 10 | log = log | {i+1: free} 11 | b = sys_crash() 12 | sys_write(b) 13 | sys_sync() 14 | 15 | # 2. write updated log head 16 | sys_bwrite(0, log) 17 | sys_sync() 18 | 19 | # 3. install transactions 20 | for k, v in log.items(): 21 | content = sys_bread(v) 22 | sys_bwrite(k, content) 23 | sys_sync() 24 | 25 | # 4. update log head 26 | sys_bwrite(0, {}) 27 | sys_sync() 28 | -------------------------------------------------------------------------------- /examples/fs-crash.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | # intially, file has a single block #1 3 | sys_bwrite('file.inode', 'i [#1]') 4 | sys_bwrite('used', '#1') 5 | sys_bwrite('#1', '#1 (old)') 6 | sys_sync() 7 | 8 | # append a block #2 to the file 9 | xs = [i + 1 for i in range(N)] 10 | xb = ['#{x}' for x in xs] 11 | 12 | sys_bwrite('file.inode', f'i [{xb}]') # inode 13 | sys_bwrite('used', xb) # bitmap 14 | for i in range(N): 15 | sys_bwrite(f'#{i + 1}', f'#{i + 1} (new)') # data block 16 | sys_crash() # system crash 17 | 18 | # display file system state at crash recovery 19 | inode = sys_bread('file.inode') 20 | used = sys_bread('used') 21 | sys_write(f'{inode}; used: {used} | ') 22 | for i in range(N): 23 | if f'#{i+1}' in inode: 24 | b = sys_bread(f'#{i+1}') 25 | sys_write(f'{b} ') 26 | -------------------------------------------------------------------------------- /examples/tocttou.py: -------------------------------------------------------------------------------- 1 | def main(): 2 | sys_bwrite('/etc/passwd', ('plain', 'secret...')) 3 | sys_bwrite('file', ('plain', 'data...')) 4 | 5 | for i in range(P - 1): 6 | pid = sys_fork() 7 | sys_sched() 8 | 9 | if pid == 0: # attacker: symlink file -> /etc/passwd 10 | sys_bwrite('file', ('symlink', '/etc/passwd')) 11 | break 12 | else: 13 | if i == 0: # sendmail (root): write to plain file 14 | filetype, contents = sys_bread('file') # for check 15 | if filetype == 'plain': 16 | sys_sched() # TOCTTOU interval 17 | filetype, contents = sys_bread('file') # for use 18 | match filetype: 19 | case 'symlink': filename = contents 20 | case 'plain': filename = 'file' 21 | sys_bwrite(filename, 'mail') 22 | sys_write(f'{filename} written') 23 | else: 24 | sys_write('rejected') 25 | -------------------------------------------------------------------------------- /examples/cond-var.py: -------------------------------------------------------------------------------- 1 | def Tworker(name, delta): 2 | for _ in range(N): 3 | while heap.mutex == '🔒': # mutex_lock() 4 | sys_sched() 5 | heap.mutex = '🔒' 6 | 7 | while not (0 <= heap.count + delta <= 1): 8 | sys_sched() 9 | heap.mutex = '🔓' # cond_wait() 10 | heap.cond.append(name) 11 | while name in heap.cond: # wait 12 | sys_sched() 13 | while heap.mutex == '🔒': # reacquire lock 14 | sys_sched() 15 | heap.mutex = '🔒' 16 | 17 | if heap.cond: # cond_signal() 18 | t = sys_choose(heap.cond) 19 | heap.cond.remove(t) # wake up anyone 20 | sys_sched() 21 | 22 | heap.count += delta # produce or consume 23 | 24 | heap.mutex = '🔓' # mutex_unlock() 25 | sys_sched() 26 | 27 | def main(): 28 | heap.mutex = '🔓' # 🔓 or 🔒 29 | heap.count = 0 # filled buffer 30 | heap.cond = [] # condition variable's wait list 31 | for i in range(T_p): 32 | sys_spawn(Tworker, f'Tp{i}', 1) # delta=1, producer 33 | for i in range(T_c): 34 | sys_spawn(Tworker, f'Tc{i}', -1) # delta=-1, consumer 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Yanyan Jiang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /vis/__main__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import jinja2 3 | import sys 4 | import pygments 5 | import pygments.lexers 6 | from pathlib import Path 7 | 8 | if __name__ == '__main__': 9 | model_str = sys.stdin.read() 10 | m = json.loads(model_str) 11 | outputs = set() 12 | for v in m['vertices']: 13 | if not any(v['contexts']): 14 | if (out := v['stdout']): 15 | outputs.add(out.strip().replace("\n", "\\n")) 16 | 17 | src = m['source'].strip() + '\n' 18 | if len(outputs) > 0: 19 | outputs = list(outputs) 20 | outputs.sort() 21 | 22 | src += '\n# Outputs:' 23 | for out in outputs: 24 | src += f'\n# {out}' 25 | 26 | hl = pygments.highlight(src.replace(' ', ' '), 27 | lexer=pygments.lexers.guess_lexer_for_filename('src.py', src), 28 | formatter=pygments.formatters.html.HtmlFormatter( 29 | style='xcode', 30 | linenos='inline', 31 | linespans=True 32 | ), 33 | ) 34 | 35 | template_file = Path(__file__).parent / 'template.html' 36 | template_text = template_file.read_text() 37 | template = jinja2.Template(template_text) 38 | res = template.render( 39 | title='State Transitions', 40 | model=model_str, 41 | hl=hl, 42 | ) 43 | print(res) 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MOSAIC Operating System Model and Checker 2 | 3 | This is the artifact for Paper #202 of USENIX ATC'23 "The Hitchhiker’s Guide to Operating Systems". (Cherry-picked from the backend of my [course homepage](http://jyywiki.cn/OS/2023/build/lect4.ipynb).) 4 | 5 | - [mosaic.py](mosaic.py) - The model checker (zero dependency; self-documented) 6 | - [vis/](vis/) - The visualization script of an interactive state space explorer 7 | - [examples/](examples/) - The code examples evaluated in the paper 8 | 9 | ## The Operating System Model 10 | 11 | MOSAIC supports simple applications with "system calls". The program entry is `main()`: 12 | 13 | ```python 14 | def main(): 15 | pid = sys_fork() 16 | sys_sched() # non-deterministic context switch 17 | if pid == 0: 18 | sys_write('World\n') 19 | else: 20 | sys_write('Hello\n') 21 | ``` 22 | 23 | MOSAIC can interpret these system calls, or model-check it: 24 | 25 | python3 mosaic.py --run foo.py 26 | python3 mosaic.py --check bar.py 27 | 28 | A JSON file (state transition graph) will be printed to stdout. 29 | The output (state transition graph) can be piped to another tool, e.g., a 30 | visualization tool: 31 | 32 | ```bash 33 | # quick and dirty check 34 | python3 mosaic.py --check examples/hello.py | grep stdout | sort | uniq 35 | ``` 36 | 37 | ```bash 38 | # interactive state explorer 39 | python3 mosaic.py --check examples/hello.py | python3 -m vis 40 | ``` 41 | 42 | ![](vis/demo.png) 43 | 44 | ## Modeled System Calls 45 | 46 | System Call | Behavior 47 | --------------------|----------------------------------------------- 48 | `sys_fork()` | create current thread's heap and context clone 49 | `sys_spawn(f, xs)` | spawn a heap-sharing thread executing `f(xs)` 50 | `sys_write(xs)` | write string `xs` to a shared console 51 | `sys_bread(k)` | return the value of block id `k` 52 | `sys_bwrite(k, v)` | write block `k` with value `v` 53 | `sys_sync()` | persist all outstanding block writes 54 | `sys_sched()` | perform a non-deterministic context switch 55 | `sys_choose(xs)` | return a non-deterministic choice in `xs` 56 | `sys_crash()` | perform a non-deterministic crash 57 | 58 | Limitation: system calls are implemented by `yield`. To keep the model checker minimal, one cannot perform system call in a function. (This is not a fundamental limitation and will be addressed in the future.) 59 | 60 | ## Reproducing Results 61 | 62 | ```bash 63 | python3 examples/_reproduce.py 64 | ``` 65 | 66 | Similar results in Table 2 are expected. Tested on: Python 3.10.9 (macOS Ventura), Python 3.11.2 (Ubuntu 22.04) 67 | -------------------------------------------------------------------------------- /examples/_reproduce.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import subprocess 4 | from collections import namedtuple 5 | from pathlib import Path 6 | import json 7 | import datetime 8 | 9 | # (Run experiments once to save time) 10 | TIMEOUT = 60 # 60s 11 | LIMIT = 0.1 # display "<0.1s" for negligible time 12 | RUN = 1 # run = 10 in the experiments 13 | 14 | Subject = namedtuple('Subject', 'name configs') 15 | 16 | # Same as Table 2 17 | SUBJECTS = [ 18 | Subject( 19 | 'fork-buf', 20 | configs=[ 21 | 'N=1', 22 | 'N=2', 23 | 'N=3', 24 | ] 25 | ), 26 | Subject( 27 | 'cond-var', 28 | configs=[ 29 | 'N=1; T_p=1; T_c=1', 30 | 'N=1; T_p=1; T_c=2', 31 | 'N=2; T_p=1; T_c=2', 32 | 'N=2; T_p=2; T_c=1', 33 | ] 34 | ), 35 | Subject( 36 | 'xv6-log', 37 | configs=[ 38 | 'N=2', 39 | 'N=4', 40 | 'N=8', 41 | 'N=10', 42 | ] 43 | ), 44 | Subject( 45 | 'tocttou', 46 | configs=[ 47 | 'P=2', 48 | 'P=3', 49 | 'P=4', 50 | 'P=5', 51 | ] 52 | ), 53 | Subject( 54 | 'parallel-inc', 55 | configs=[ 56 | 'N=1; T=2', 57 | 'N=2; T=2', 58 | 'N=2; T=3', 59 | 'N=3; T=3', 60 | ] 61 | ), 62 | Subject( 63 | 'fs-crash', 64 | configs=[ 65 | 'N=2', 66 | 'N=4', 67 | 'N=8', 68 | 'N=10', 69 | ] 70 | ), 71 | ] 72 | 73 | def run(conf, src, n=RUN): 74 | src = conf + '\n\n' + src 75 | 76 | PROFILING = ''' 77 | import atexit, psutil, sys 78 | 79 | def mem(): 80 | import os 81 | process = psutil.Process(os.getpid()) 82 | mem_info = process.memory_info() 83 | print(mem_info.rss, file=sys.stderr) 84 | 85 | atexit.register(mem) 86 | ''' 87 | 88 | def run_once(src, profiling=False): 89 | if profiling: 90 | src = PROFILING + src 91 | p = subprocess.run( 92 | ['python3', 'mosaic.py', '--check', '/dev/stdin'], 93 | input=src.encode(), 94 | stdout=subprocess.PIPE, 95 | stderr=subprocess.PIPE, 96 | timeout=TIMEOUT 97 | ) 98 | assert p.returncode == 0 99 | return p 100 | 101 | # First run, for warm up and memory statistics 102 | p = run_once(src, profiling=True) 103 | G = json.loads(p.stdout) 104 | V = G['vertices'] 105 | mem = int(p.stderr) / 1024 / 1024 106 | 107 | size, times = len(V), [] 108 | 109 | # Repeated runs 110 | for _ in range(n): 111 | t1 = datetime.datetime.now() 112 | run_once(src, profiling=False) 113 | t2 = datetime.datetime.now() 114 | runtime = (t2 - t1).total_seconds() 115 | times.append(runtime) 116 | 117 | return (size, mem, times) 118 | 119 | def evaluate(subj): 120 | src = Path(__file__).parent \ 121 | .glob(f'**/{subj.name}.py').__next__() \ 122 | .read_text() 123 | LOC = len(src.strip().splitlines()) 124 | print(f' {subj.name} ({LOC} LOC) '.center(62, '-')) 125 | 126 | for conf in subj.configs: 127 | try: 128 | states, mem, ts = run(conf, src) 129 | avg = sum(ts) / len(ts) 130 | per = int(states / avg) 131 | if avg < LIMIT: 132 | time_text = f'<{LIMIT}s' 133 | else: 134 | time_text = f'{avg:.1f}s ({per} st/s)' 135 | except subprocess.TimeoutExpired: 136 | time_text = f'Timeout (>{TIMEOUT}s)' 137 | print(f'{conf:>20}{time_text:>42}') 138 | else: 139 | print(f'{conf:>20}{states:>10}{time_text:>20}{mem:>10.2f}MB') 140 | 141 | for _, subj in enumerate(SUBJECTS): 142 | evaluate(subj) 143 | -------------------------------------------------------------------------------- /mosaic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Mosaic Emulator and Checker 4 | 5 | import argparse 6 | import ast 7 | import copy 8 | import inspect 9 | import json 10 | import random 11 | from dataclasses import dataclass 12 | from itertools import compress, product 13 | from pathlib import Path 14 | from typing import Callable, Generator 15 | 16 | ## 1. Mosaic system calls 17 | 18 | ### 1.1 Process, thread, and context switching 19 | 20 | sys_fork = lambda: os.sys_fork() 21 | sys_spawn = lambda fn, *args: os.sys_spawn(fn, *args) 22 | sys_sched = lambda: os.sys_sched() 23 | 24 | ### 1.2 Virtual character device 25 | 26 | sys_choose = lambda choices: os.sys_choose(choices) 27 | sys_write = lambda *args: os.sys_write(*args) 28 | 29 | ### 1.3 Virtual block storage device 30 | 31 | sys_bread = lambda k: os.sys_bread(k) 32 | sys_bwrite = lambda k, v: os.sys_bwrite(k, v) 33 | sys_sync = lambda: os.sys_sync() 34 | sys_crash = lambda: os.sys_crash() 35 | 36 | ### 1.4 System call helpers 37 | 38 | SYSCALLS = [] 39 | 40 | def syscall(func): # @syscall decorator 41 | SYSCALLS.append(func.__name__) 42 | return func 43 | 44 | ## 2. Mosaic operating system emulator 45 | 46 | ### 2.1 Data structures 47 | 48 | class Heap: 49 | pass # no member: self.__dict__ is the heap 50 | 51 | @dataclass 52 | class Thread: 53 | context: Generator # program counter, local variables, etc. 54 | heap: Heap # a pointer to thread's "memory" 55 | 56 | @dataclass 57 | class Storage: 58 | persist: dict # persisted storage state 59 | buf: dict # outstanding operations yet to be persisted 60 | 61 | ### 2.2 The OperatingSystem class 62 | 63 | class OperatingSystem: 64 | """An executable operating system model. 65 | 66 | The operating system model hosts a single Python application with a 67 | main() function accessible to a shared heap and 9 system calls 68 | (marked by the @syscall decorator). An example: 69 | 70 | def main(): 71 | pid = sys_fork() 72 | sys_sched() # non-deterministic context switch 73 | if pid == 0: 74 | sys_write('World') 75 | else: 76 | sys_write('Hello') 77 | 78 | At any moment, this model keeps tracking a set of threads and a 79 | "currently running" one. Each thread consists of a reference to a 80 | heap object (may be shared with other threads) and a private context 81 | (program counters, local variables, etc.). A thread context is a 82 | Python generator object, i.e., a stack-less coroutine [1] that can 83 | save the running context and yield itself. 84 | 85 | For applications, the keyword "yield" is reserved for system calls. 86 | For example, a "choose" system call [2]: 87 | 88 | sys_choose(['A', 'B']) 89 | 90 | is transpiled as yielding the string "sys_choose" followed by its 91 | parameters (choices): 92 | 93 | res = yield 'sys_choose', ['A', 'B']. 94 | 95 | Yield will transfer the control to the OS for system call handling 96 | and eventually returning a value ('A' or 'B') to the application. 97 | 98 | Right after transferring control to the OS by "yield", the function 99 | state is "frozen", where program counters and local variables are 100 | accessible via the generator object. Therefore, OS can serialize its 101 | internal state--all thread's contexts, heaps, and virtual device 102 | states at this moment. 103 | 104 | In this sense, operating system is a system-call driven state 105 | transition system: 106 | 107 | (s0) --run first thread (main)-> (s1) 108 | --sys_choose and application execution-> (s2) 109 | --sys_sched and application execution-> (s3) ... 110 | 111 | Real operating systems can be preemptive--context switching can 112 | happen non-deterministically at any program point, simply because 113 | processor can non-deterministically interrupt its currently running 114 | code and transfer the control to the operating system. 115 | 116 | The OS internal implementation does NOT immediately process the 117 | system call: it returns all possible choices available at the moment 118 | and their corresponding processing logic as callbacks. For the 119 | example above, the "choose" system call returns a non-deterministic 120 | choice among given choices. The internal implementation thus returns 121 | 122 | choices = { 123 | 'A': (lambda: 'A'), 124 | 'B': (lambda: 'B'), 125 | } 126 | 127 | for later processing. Another example is non-deterministic context 128 | switching by yielding 'sys_sched'. Suppose there are threads t1 and 129 | t2 at the moment. The system call handler will return 130 | 131 | choices = { 132 | 't1': (lambda: switch_to(t1)), 133 | 't2': (lambda: switch_to(t2)), 134 | } 135 | 136 | in which switch_to(th) replaces the OS's current running thread with 137 | th (changes the global "heap" variable). Such deferred execution of 138 | system calls separates the mechanism of non-deterministic choices 139 | from the actual decision makers (e.g., an interpreter or a model 140 | checker). Once the decision is made, the simply call step(choice) 141 | and the OS will execute this choice by 142 | 143 | choices[choice]() 144 | 145 | with the application code (generator) being resumed. 146 | 147 | This model provides "write" system call to immediately push data to 148 | a hypothetical character device like a tty associated with stdout. 149 | We model a block device (key-value store) that may lose data upon 150 | crash. The model assumes atomicity of each single block write (a 151 | key-value update). However, writes are merely to a volatile buffer 152 | which may non-deterministically lose data upon crash 3]. The "sync" 153 | system call persists buffered writes. 154 | 155 | References: 156 | 157 | [1] N. Schemenauer, T. Peters, and M. L. Hetland. PEP 255 - 158 | Simple generators. https://peps.python.org/pep-0255/ 159 | [2] J. Yang, C. Sar, and D. Engler. eXplode: a lightweight, general 160 | system for finding serious storage system errors. OSDI'06. 161 | [3] T. S. Pillai, V. Chidambaram, R. Alagappan, A. Al-Kiswany, A. C. 162 | Arpaci-Dusseau, and R. H. Arpaci-Dusseau. All file systems are 163 | not created equal: On the complexity of crafting crash 164 | consistent applications. OSDI'14. 165 | """ 166 | 167 | def __init__(self, init: Callable): 168 | """Create a new OS instance with pending-to-execute init thread.""" 169 | # Operating system states 170 | self._threads = [Thread(context=init(), heap=Heap())] 171 | self._current = 0 172 | self._choices = {init.__name__: lambda: None} 173 | self._stdout = '' 174 | self._storage = Storage(persist={}, buf={}) 175 | 176 | # Internal states 177 | self._init = init 178 | self._trace = [] 179 | self._newfork = set() 180 | 181 | ### 2.3 System call implementation 182 | 183 | #### 2.3.1 Process, thread, and context switching 184 | 185 | @syscall 186 | def sys_spawn(self, func: Callable, *args): 187 | """Spawn a heap-sharing new thread executing func(args).""" 188 | def do_spawn(): 189 | self._threads.append( 190 | Thread( 191 | context=func(*args), # func() returns a new generator 192 | heap=self.current().heap, # shared heap 193 | ) 194 | ) 195 | return {'spawn': (lambda: do_spawn())} 196 | 197 | @syscall 198 | def sys_fork(self): 199 | """Create a clone of the current thread with a copied heap.""" 200 | if all(not f.frame.f_locals['fork_child'] 201 | for f in inspect.stack() 202 | if f.function == '_step'): # this is parent; do fork 203 | # Deep-copying generators causes troubles--they are twined with 204 | # Python's runtime state. We use an (inefficient) hack here: replay 205 | # the trace and override the last fork to avoid infinite recursion. 206 | os_clone = OperatingSystem(self._init) 207 | os_clone.replay(self._trace[:-1]) 208 | os_clone._step(self._trace[-1], fork_child=True) 209 | 210 | # Now os_clone._current is the forked process. Cloned thread just 211 | # yields a sys_fork and is pending for fork()'s return value. It 212 | # is necessary to mark cloned threads (in self._newfork) and send 213 | # child's fork() return value when they are scheduled for the 214 | # first time. 215 | def do_fork(): 216 | self._threads.append(os_clone.current()) 217 | self._newfork.add((pid := len(self._threads)) - 1) 218 | return 1000 + pid # returned pid starts from 1000 219 | 220 | return {'fork': (lambda: do_fork())} 221 | else: 222 | return None # overridden fork; this value is never used because 223 | # os_clone is destroyed immediately after fork() 224 | 225 | @syscall 226 | def sys_sched(self): 227 | """Return a non-deterministic context switch to a runnable thread.""" 228 | return { 229 | f't{i+1}': (lambda i=i: self._switch_to(i)) 230 | for i, th in enumerate(self._threads) 231 | if th.context.gi_frame is not None # thread still alive? 232 | } 233 | 234 | ### 2.3.2 Virtual character device (byte stream) 235 | 236 | @syscall 237 | def sys_choose(self, choices): 238 | """Return a non-deterministic value from choices.""" 239 | return {f'choose {c}': (lambda c=c: c) for c in choices} 240 | 241 | @syscall 242 | def sys_write(self, *args): 243 | """Write strings (space separated) to stdout.""" 244 | def do_write(): 245 | self._stdout += ' '.join(str(arg) for arg in args) 246 | return {'write': (lambda: do_write())} 247 | 248 | ### 2.3.3 Virtual block storage device 249 | 250 | @syscall 251 | def sys_bread(self, key): 252 | """Return the specific key's associated value in block device.""" 253 | storage = self._storage 254 | return {'bread': (lambda: 255 | storage.buf.get(key, # always try to read from buffer first 256 | storage.persist.get(key, None) # and then persistent storage 257 | ) 258 | )} 259 | 260 | @syscall 261 | def sys_bwrite(self, key, value): 262 | """Write (key, value) pair to block device's buffer.""" 263 | def do_bwrite(): 264 | self._storage.buf[key] = value 265 | return {'bwrite': (lambda: do_bwrite())} 266 | 267 | @syscall 268 | def sys_sync(self): 269 | """Persist all buffered writes.""" 270 | def do_sync(): 271 | store = self._storage 272 | self._storage = Storage( 273 | persist=store.persist | store.buf, # write back 274 | buf={} 275 | ) 276 | return {'sync': (lambda: do_sync())} 277 | 278 | @syscall 279 | def sys_crash(self): 280 | """Simulate a system crash that non-deterministically persists 281 | outstanding writes in the buffer. 282 | """ 283 | persist = self._storage.persist 284 | btrace = self._storage.buf.items() # block trace 285 | 286 | crash_sites = ( 287 | lambda subset=subset: 288 | setattr(self, '_storage', 289 | Storage( # persist only writes in the subset 290 | persist=persist | dict(compress(btrace, subset)), 291 | buf={} 292 | ) 293 | ) for subset in # Mosaic allows persisting any subset of 294 | product( # pending blocks in the buffer 295 | *([(0, 1)] * len(btrace)) 296 | ) 297 | ) 298 | return dict(enumerate(crash_sites)) 299 | 300 | ### 2.4 Operating system as a state machine 301 | 302 | def replay(self, trace: list) -> dict: 303 | """Replay an execution trace and return the resulting state.""" 304 | for choice in trace: 305 | self._step(choice) 306 | return self.state_dump() 307 | 308 | def _step(self, choice, fork_child=False): 309 | self._switch_to(self._current) 310 | self._trace.append(choice) # keep all choices for replay-based fork() 311 | action = self._choices[choice] # return value of sys_xxx: a lambda 312 | res = action() 313 | 314 | try: # Execute current thread for one step 315 | func, args = self.current().context.send(res) 316 | assert func in SYSCALLS 317 | self._choices = getattr(self, func)(*args) 318 | except StopIteration: # ... and thread terminates 319 | self._choices = self.sys_sched() 320 | 321 | # At this point, the operating system's state is 322 | # (self._threads, self._current, self._stdout, self._storage) 323 | # and outgoing transitions are saved in self._choices. 324 | 325 | ### 2.5 Misc and helper functions 326 | 327 | def state_dump(self) -> dict: 328 | """Create a serializable Mosaic state dump with hash code.""" 329 | heaps = {} 330 | for th in self._threads: 331 | if (i := id(th.heap)) not in heaps: # unique heaps 332 | heaps[i] = len(heaps) + 1 333 | 334 | os_state = { 335 | 'current': self._current, 336 | 'choices': sorted(list(self._choices.keys())), 337 | 'contexts': [ 338 | { 339 | 'name': th.context.gi_frame.f_code.co_name, 340 | 'heap': heaps[id(th.heap)], # the unique heap id 341 | 'pc': th.context.gi_frame.f_lineno, 342 | 'locals': th.context.gi_frame.f_locals, 343 | } if th.context.gi_frame is not None else None 344 | for th in self._threads 345 | ], 346 | 'heaps': { 347 | heaps[id(th.heap)]: th.heap.__dict__ 348 | for th in self._threads 349 | }, 350 | 'stdout': self._stdout, 351 | 'store_persist': self._storage.persist, 352 | 'store_buffer': self._storage.buf, 353 | } 354 | 355 | h = hash(json.dumps(os_state, sort_keys=True)) + 2**63 356 | return (copy.deepcopy(os_state) # freeze the runtime state 357 | | dict(hashcode=f'{h:016x}')) 358 | 359 | def current(self) -> Thread: 360 | """Return the current running thread object.""" 361 | return self._threads[self._current] 362 | 363 | def _switch_to(self, tid: int): 364 | self._current = tid 365 | globals()['os'] = self 366 | globals()['heap'] = self.current().heap 367 | if tid in self._newfork: 368 | self._newfork.remove(tid) # tricky: forked process must receive 0 369 | return 0 # to indicate a child 370 | 371 | ## 3. The Mosaic runtime 372 | 373 | class Mosaic: 374 | """The operating system interpreter and model checker. 375 | 376 | The operating system model is a state transition system: os.replay() 377 | maps any trace to a state (with its outgoing transitions). Based 378 | on this model, two state space explorers are implemented: 379 | 380 | - run: Choose outgoing transitions uniformly at random, yielding a 381 | single execution trace. 382 | - check: Exhaustively explore all reachable states by a breadth- 383 | first search. Duplicated states are not visited twice. 384 | 385 | Both explorers produce the visited portion of the state space as a 386 | serializable object containing: 387 | 388 | - source: The application source code 389 | - vertices: A list of operating system state dumps. The first vertex 390 | in the list is the initial state. Each vertex has a 391 | unique "hashcode" id. 392 | - edges: A list of 3-tuples: (source, target, label) denoting an 393 | explored source --[label]-> target edge. Both source and 394 | target are state hashcode ids. 395 | """ 396 | 397 | ### 3.1 Model interpreter and checker 398 | 399 | def run(self) -> dict: 400 | """Interpret the model with non-deterministic choices.""" 401 | os = OperatingSystem(self.entry) 402 | V, E = [os.state_dump() | dict(depth=0)], [] 403 | 404 | while (choices := V[-1]['choices']): 405 | choice = random.choice(choices) # uniformly at random 406 | V.append(os.replay([choice]) | dict(depth=len(V))) 407 | E.append((V[-2]['hashcode'], V[-1]['hashcode'], choice)) 408 | 409 | return dict(source=self.src, vertices=V, edges=E) 410 | 411 | def check(self) -> dict: 412 | """Exhaustively explore the state space.""" 413 | class State: 414 | entry = self.entry 415 | 416 | def __init__(self, trace): 417 | self.trace = trace 418 | self.state = OperatingSystem(State.entry).replay(trace) 419 | self.state |= dict(depth=0) 420 | self.hashcode = self.state['hashcode'] 421 | 422 | def extend(self, c): 423 | st = State(self.trace + (c,)) 424 | st.state = st.state | dict(depth=self.state['depth'] + 1) 425 | return st 426 | 427 | st0 = State(tuple()) # initial state of empty trace 428 | queued, V, E = [st0], {st0.hashcode: st0.state}, [] 429 | 430 | while queued: 431 | st = queued.pop(0) 432 | for choice in st.state['choices']: 433 | st1 = st.extend(choice) 434 | if st1.hashcode not in V: # found an unexplored state 435 | V[st1.hashcode] = st1.state 436 | queued.append(st1) 437 | E.append((st.hashcode, st1.hashcode, choice)) 438 | 439 | return dict( 440 | source=self.src, 441 | vertices=sorted(V.values(), key=lambda st: st['depth']), 442 | edges=E 443 | ) 444 | 445 | ### 3.1 Source code parsing and rewriting 446 | 447 | class Transformer(ast.NodeTransformer): 448 | def visit_Call(self, node): 449 | # Rewrite system calls as yields 450 | if (isinstance(node.func, ast.Name) and 451 | node.func.id in SYSCALLS): # rewrite system calls 452 | return ast.Yield(ast.Tuple( # -> yield ('sys_xxx', args) 453 | elts=[ 454 | ast.Constant(value=node.func.id), 455 | ast.Tuple(elts=node.args), 456 | ] 457 | )) 458 | else: 459 | return node 460 | 461 | def __init__(self, src: str): 462 | tree = ast.parse(src) 463 | hacked_ast = self.Transformer().visit(tree) 464 | hacked_src = ast.unparse(hacked_ast) 465 | 466 | context = {} 467 | exec(hacked_src, globals(), context) 468 | globals().update(context) 469 | 470 | self.src = src 471 | self.entry = context['main'] # must have a main() 472 | 473 | ## 4. Utilities 474 | 475 | if __name__ == '__main__': 476 | parser = argparse.ArgumentParser( 477 | description='The modeled operating system and state explorer.' 478 | ) 479 | parser.add_argument( 480 | 'source', 481 | help='application code (.py) to be checked; must have a main()' 482 | ) 483 | parser.add_argument('-r', '--run', action='store_true') 484 | parser.add_argument('-c', '--check', action='store_true') 485 | args = parser.parse_args() 486 | 487 | src = Path(args.source).read_text() 488 | mosaic = Mosaic(src) 489 | if args.check: 490 | explored = mosaic.check() 491 | else: 492 | explored = mosaic.run() # run is the default option 493 | 494 | # Serialize the explored states and write to stdout. This encourages piping 495 | # the results to another tool following the UNIX philosophy. Examples: 496 | # 497 | # mosaic --run foo.py | grep stdout | tail -n 1 # quick and dirty check 498 | # mosaic --check bar.py | fx # or any other interactive visualizer 499 | # 500 | print(json.dumps(explored, ensure_ascii=False, indent=2)) 501 | -------------------------------------------------------------------------------- /vis/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {{title}} 6 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 502 | 503 |
504 |
505 |
{{hl}}
506 |
507 | 508 | 511 | 512 | 513 | 514 | --------------------------------------------------------------------------------