├── .editorconfig ├── src ├── golden │ ├── temp.nim │ ├── lmdb.nim │ ├── plot.nim │ ├── linkedlists.nim │ ├── fsm.nim │ ├── compilation.nim │ ├── running.nim │ ├── invoke.nim │ ├── db.nim │ ├── benchmark.nim │ ├── lm.nim │ └── spec.nim ├── golden.nim.cfg └── golden.nim ├── tests ├── example.nim ├── nim.cfg ├── tspec.nim ├── tstats.nim ├── tinvoke.nim └── tdb.nim ├── LICENSE ├── golden.nimble └── README.md /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | insert_final_newline = true 4 | indent_size = 2 5 | trim_trailing_whitespace = true 6 | -------------------------------------------------------------------------------- /src/golden/temp.nim: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | proc createTemporaryFile*(prefix: string; suffix: string): string = 4 | ## it should create the file, but so far, it doesn't 5 | let temp = getTempDir() 6 | result = temp / "golden-" & $getCurrentProcessId() & prefix & suffix 7 | -------------------------------------------------------------------------------- /tests/example.nim: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # whatfer testing params and failed invocations 4 | if paramCount() == 1 and paramStr(1) == "quit": 5 | quit(3) 6 | 7 | if paramCount() == 1 and paramStr(1) == "hello": 8 | echo "world" 9 | 10 | if paramCount() == 1 and paramStr(1) == "goodbye": 11 | stderr.writeLine "cruel world" 12 | 13 | sleep 100 14 | -------------------------------------------------------------------------------- /tests/nim.cfg: -------------------------------------------------------------------------------- 1 | --path="$config/../src/" 2 | 3 | --define:threadsafe 4 | --define:threads=on 5 | # use a better poller against process termination 6 | --define:useProcessSignal 7 | 8 | --define:useSHA 9 | --define:LongWayHome 10 | 11 | # nimgit stuff 12 | --define:git2Git 13 | --define:git2SetVer:"master" 14 | 15 | # build lmdb into the executable 16 | #--define:lmdbStatic 17 | -------------------------------------------------------------------------------- /src/golden.nim.cfg: -------------------------------------------------------------------------------- 1 | --define:threadsafe 2 | --define:threads=on 3 | --opt:speed 4 | 5 | # use a better poller against process termination 6 | --define:useProcessSignal 7 | 8 | --define:useSHA 9 | --define:LongWayHome 10 | #--define:Heapster 11 | #--define:debug 12 | #--define:nimTypeNames 13 | #--define:useGcAssert 14 | 15 | # nimgit stuff 16 | --define:git2Git 17 | --define:git2SetVer:"v1.0.1" 18 | -------------------------------------------------------------------------------- /src/golden/lmdb.nim: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import nimterop/[build, cimport] 4 | 5 | const 6 | baseDir = getProjectCacheDir("nimlmdb") 7 | 8 | static: 9 | #cDebug() 10 | 11 | gitPull( 12 | "https://github.com/LMDB/lmdb", 13 | outdir = baseDir, 14 | checkout = "mdb.master" 15 | ) 16 | 17 | getHeader( 18 | "lmdb.h", 19 | outdir = baseDir / "libraries" / "liblmdb" 20 | ) 21 | 22 | type 23 | mode_t = uint32 24 | 25 | when defined(lmdbStatic): 26 | cImport(lmdbPath) 27 | else: 28 | cImport(lmdbPath, dynlib = "lmdbLPath") 29 | -------------------------------------------------------------------------------- /tests/tspec.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncdispatch 3 | import unittest 4 | import strutils 5 | 6 | import golden 7 | import golden/spec 8 | import golden/compilation 9 | 10 | suite "spec": 11 | setup: 12 | var 13 | golden = newGolden() 14 | exampleNim = newFileDetailWithInfo("tests/example.nim") 15 | nimPath = newFileDetailWithInfo(getCurrentCompilerExe()) 16 | compiler = newCompiler() 17 | let 18 | targets = @[exampleNim.file.path] 19 | storage {.used.} = golden.storageForTargets(targets) 20 | 21 | test "set compiler": 22 | nimPath.compiler = compiler 23 | let c = nimPath[aCompiler] 24 | check compiler.oid == c.oid 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Andy Davidoff 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/golden/plot.nim: -------------------------------------------------------------------------------- 1 | #import plotly 2 | import nimetry 3 | 4 | import running 5 | import temp 6 | 7 | proc consolePlot*(stat: RunningStat; histogram: seq[int]; 8 | dimensions: ClassDimensions): string = 9 | ## create a simple plot for display on the console via kitty 10 | result = createTemporaryFile("-image", ".png") 11 | var 12 | data: seq[XY] 13 | p: Plot = newPlot(1600, 1600) 14 | 15 | # use this to rescale small values 16 | var m: float64 = 1.0 17 | while stat.min * m < 1: 18 | m *= 10.0 19 | 20 | p.setX(stat.min.float * m, stat.max.float * m) 21 | p.setY(min(histogram).float * 0.8, max(histogram).float) 22 | p.setXtic(dimensions.size * m) 23 | p.setYtic(max(histogram).float / 10.0) 24 | 25 | for class, value in histogram.pairs: 26 | # the X value is the minimum plus (the class * the class size) 27 | # the Y value is simply the count in the histogram 28 | data.add (m * (stat.min.float + (class.float * dimensions.size)), value.float) 29 | p.setTitle("benchmark") 30 | p.setFontTtf("fonts/Vera.ttf") # sorry! 31 | p.addPlot(data, Line, rgba(0, 0, 255, 255)) 32 | p.save(result) 33 | -------------------------------------------------------------------------------- /golden.nimble: -------------------------------------------------------------------------------- 1 | version = "3.0.15" 2 | author = "disruptek" 3 | description = "a benchmark tool" 4 | license = "MIT" 5 | requires "nim >= 1.0.4" 6 | 7 | requires "foreach >= 1.0.2" 8 | requires "bump >= 1.8.15" 9 | requires "msgpack4nim 0.2.9" 10 | requires "terminaltables#82ee5890c13e381de0f11c8ba6fe484d7c0c2f19" 11 | requires "https://github.com/disruptek/gittyup >= 2.4.4 & < 3.0.0" 12 | requires "nimterop >= 0.6.2 & < 1.0.0" 13 | 14 | # we need this one for csize reasons 15 | requires "cligen >= 0.9.40" 16 | 17 | bin = @["golden"] 18 | srcDir = "src" 19 | 20 | proc execCmd(cmd: string) = 21 | echo "execCmd:" & cmd 22 | exec cmd 23 | 24 | proc execTest(test: string) = 25 | execCmd "nim c -f -r " & test 26 | execCmd "nim c -d:release -r " & test 27 | execCmd "nim c -d:danger -r " & test 28 | execCmd "nim cpp -r " & test 29 | execCmd "nim cpp -d:danger -r " & test 30 | when NimMajor >= 1 and NimMinor >= 1: 31 | execCmd "nim c --useVersion:1.0 -d:danger -r " & test 32 | execCmd "nim c --gc:arc -r " & test 33 | execCmd "nim cpp --gc:arc -r " & test 34 | 35 | task test, "run tests for travis": 36 | execTest("tests/tstats.nim") 37 | execTest("tests/tspec.nim") 38 | execTest("tests/tdb.nim") 39 | execTest("tests/tinvoke.nim") 40 | -------------------------------------------------------------------------------- /tests/tstats.nim: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import golden/spec 4 | import golden/running 5 | 6 | suite "statistics": 7 | test "basic histogram pruning": 8 | var running = newRunningResult[Gold]() 9 | for t in [2, 3, 4, 5, 6]: 10 | var i = newInvocation() 11 | i.invokation.wall = initDuration(seconds = t) 12 | running.add i 13 | if t == 2: 14 | check running.stat.classSize(1) == 2.0 15 | if t == 5: 16 | check running.stat.classSize(2) == 1.5 17 | check running.len == 5 18 | for i in [2, 4, 8]: 19 | let dims = running.stat.makeDimensions(i) 20 | if i == 8: 21 | check dims.count == 5 22 | check dims.size == 0.8 23 | else: 24 | check dims.count == i 25 | 26 | check running.stat.max == 6.0 27 | check running.stat.min == 2.0 28 | var 29 | dims = running.stat.makeDimensions(3) 30 | histo = running.crudeHistogram dims 31 | check histo == @[2, 1, 2] 32 | 33 | for t in [2, 3, 4]: 34 | for u in 0 .. 10_000: 35 | var i = newInvocation() 36 | i.invokation.wall = initDuration(seconds = t) 37 | running.add i 38 | dims = running.stat.makeDimensions(9) 39 | histo = running.crudeHistogram dims 40 | check histo == @[10002, 0, 10002, 0, 10002, 0, 1, 0, 1] 41 | check true == running.maybePrune(histo, dims, 0.001) 42 | check histo == @[10002, 0, 10002, 0, 10002] 43 | -------------------------------------------------------------------------------- /src/golden/linkedlists.nim: -------------------------------------------------------------------------------- 1 | #[ 2 | 3 | just some comfort for lists 4 | 5 | ]# 6 | 7 | import lists 8 | export lists 9 | 10 | import msgpack4nim 11 | 12 | proc isEmpty*[T](list: SinglyLinkedList[T]): bool = 13 | result = list.head == nil 14 | 15 | proc len*[T](list: SinglyLinkedList[T]): int = 16 | var head = list.head 17 | while head != nil: 18 | result.inc 19 | head = head.next 20 | 21 | proc first*[T: ref](list: SinglyLinkedList[T]): T = 22 | if list.head != nil: 23 | result = list.head.value 24 | 25 | proc removeNext*(head: var SinglyLinkedNode) = 26 | ## remove the next node in a list 27 | if head != nil: 28 | if head.next != nil: 29 | if head.next.next != nil: 30 | head.next = head.next.next 31 | else: 32 | head.next = nil 33 | 34 | proc pack_type*[ByteStream, T](s: ByteStream; x: ref SinglyLinkedNodeObj[T]) = 35 | {.error: "this can't be right...".} 36 | s.pack(x.value) 37 | s.pack(x.next) 38 | 39 | proc unpack_type*[ByteStream, T](s: ByteStream; x: var ref SinglyLinkedNodeObj[T]) = 40 | {.error: "this can't be right...".} 41 | s.unpack_type(x.value) 42 | s.unpack_type(x.next) 43 | 44 | proc pack_type*[ByteStream, T](s: ByteStream; x: SinglyLinkedList[T]) = 45 | s.pack(x.head) 46 | s.pack(x.tail) 47 | 48 | proc unpack_type*[ByteStream, T](s: ByteStream; x: var SinglyLinkedList[T]) = 49 | s.unpack_type(x.head) 50 | s.unpack_type(x.tail) 51 | -------------------------------------------------------------------------------- /tests/tinvoke.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncdispatch 3 | import unittest 4 | 5 | import golden/spec 6 | import golden/invoke 7 | import golden/compilation 8 | 9 | suite "compile and invoke": 10 | setup: 11 | let 12 | exampleNim = newFileDetailWithInfo("tests/example.nim") 13 | var 14 | compiler = newCompiler() 15 | gold {.used.} = newCompilation(compiler, exampleNim.file.path) 16 | 17 | test "assumptions": 18 | check gold.compiler.version != "" 19 | check gold.argumentsForCompilation(@[]) == @["c", "-d:danger"] 20 | check gold.argumentsForCompilation(@["umm"]) == @["c", "umm"] 21 | check gold.argumentsForCompilation(@["cpp", "-d:debug"]) == @["cpp", "-d:debug"] 22 | 23 | test "compilation": 24 | let 25 | target = pathToCompilationTarget(exampleNim.file.path) 26 | if fileExists(target): 27 | removeFile(target) 28 | let 29 | args = @["c", "--outdir=" & target.parentDir] 30 | simple = waitfor compileFile(exampleNim.file.path, args) 31 | check simple.okay 32 | check simple.target.file.path == target 33 | check simple.target.file.path.fileExists 34 | 35 | test "invocation": 36 | var binary = gold.target 37 | var invocation = waitfor invoke(binary) 38 | check invocation.okay 39 | invocation = waitfor invoke(binary, @["quit"]) 40 | check not invocation.okay 41 | check invocation.invokation.code == 3 42 | invocation = waitfor invoke(binary, @["hello"]) 43 | check invocation.okay 44 | check invocation.invokation.stdout == "world\n" 45 | invocation = waitfor invoke(binary, @["goodbye"]) 46 | check invocation.invokation.stderr == "cruel world\n" 47 | -------------------------------------------------------------------------------- /tests/tdb.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncdispatch 3 | import unittest 4 | import strutils 5 | 6 | import golden 7 | import golden/spec 8 | import golden/lm 9 | 10 | suite "database": 11 | setup: 12 | var 13 | golden = newGolden() 14 | db: GoldenDatabase = nil 15 | exampleNim = newFileDetailWithInfo("tests/example.nim") 16 | 17 | let 18 | targets = @[exampleNim.file.path] 19 | storage {.used.} = golden.storageForTargets(targets) 20 | 21 | teardown: 22 | db.close 23 | 24 | test "assumptions": 25 | check exampleNim.dirty 26 | check storage.endsWith "/example" 27 | 28 | test "create, destroy once": 29 | let 30 | paths = storage.rsplit("/example", maxSplit=1) 31 | storagePath = paths.join("") & "/.example.golden-lmdb" 32 | db = waitfor golden.openDatabase(targets) 33 | db.close 34 | check existsDir(storagePath) 35 | golden.removeDatabase(targets) 36 | check not existsDir(storagePath) 37 | 38 | test "write": 39 | db = waitfor golden.openDatabase(targets) 40 | 41 | exampleNim.dirty = true 42 | db.write(exampleNim) 43 | check not exampleNim.dirty 44 | 45 | expect AssertionError: 46 | # because it's not dirty 47 | db.write(exampleNim) 48 | 49 | test "write, dupe, read, write": 50 | var 51 | foo = newFileDetail("foo") 52 | db = waitfor golden.openDatabase(targets) 53 | 54 | exampleNim.dirty = true 55 | db.write(exampleNim) 56 | check not exampleNim.dirty 57 | 58 | # make sure we can't write duplicates 59 | expect Exception: 60 | exampleNim.dirty = true 61 | db.write(exampleNim) 62 | 63 | foo.oid = exampleNim.oid 64 | foo.dirty = false 65 | db.read(foo) 66 | check not foo.dirty 67 | 68 | expect AssertionError: 69 | # because it's not dirty 70 | db.write(foo) 71 | 72 | #check foo.file.path == exampleNim.file.path 73 | check foo.oid == exampleNim.oid 74 | when defined(StoreEntry): 75 | check foo.entry == exampleNim.entry 76 | check foo.created == exampleNim.created 77 | check foo.file.digest == exampleNim.file.digest 78 | check foo.file.size == exampleNim.file.size 79 | check foo.file.mtime == exampleNim.file.mtime 80 | 81 | test "dry run": 82 | golden.options.flags.incl DryRun 83 | 84 | db = waitfor golden.openDatabase(targets) 85 | 86 | exampleNim.dirty = true 87 | expect Exception: 88 | db.write(exampleNim) 89 | check exampleNim.dirty 90 | 91 | # remove flag for future tests 92 | golden.options.flags.excl DryRun 93 | 94 | test "create, destroy, leak": 95 | let 96 | d = 6 97 | opens = d * 80 98 | 99 | checkpoint "expecting a leak of 2312 bytes" 100 | var leak = 0 101 | for j in 0 ..< d: 102 | var start = quiesceMemory("starting memory:") 103 | let k = opens div d 104 | for n in 0 ..< k: 105 | db = waitfor golden.openDatabase(targets) 106 | db.close 107 | var occupied = quiesceMemory("ending memory:") 108 | # measure the first and second values 109 | if leak == 0 or k < 2: 110 | leak = occupied - start 111 | continue 112 | checkpoint "memory leak " & $leak & " for " & $k & " opens " & $(occupied - start) 113 | # to see if it's changing over iteration 114 | check occupied - start <= opens * sizeof(pointer) 115 | 116 | # recreate the file so we can confirm permissions 117 | db = waitfor golden.openDatabase(targets) 118 | -------------------------------------------------------------------------------- /src/golden/fsm.nim: -------------------------------------------------------------------------------- 1 | import tables, strutils, options 2 | 3 | type 4 | Callback = proc(): void 5 | StateEvent[S,E] = tuple[state: S, event: E] 6 | Transition[S] = tuple[nexState: S, action: Option[Callback]] 7 | 8 | Machine*[S,E] = ref object of RootObj 9 | initialState: S 10 | currentState: Option[S] 11 | transitions: TableRef[StateEvent[S,E], Transition[S]] 12 | transitionsAny: TableRef[S, Transition[S]] 13 | defaultTransition: Option[Transition[S]] 14 | 15 | TransitionNotFoundException* = object of Exception 16 | 17 | proc reset*(m: Machine) = 18 | m.currentState = some(m.initialState) 19 | 20 | proc setInitialState*[S,E](m: Machine[S,E], state: S) = 21 | m.initialState = state 22 | if m.currentState.isNone: 23 | m.reset() 24 | 25 | proc newMachine*[S,E](initialState: S): Machine[S,E] = 26 | result = new(Machine[S,E]) 27 | result.transitions = newTable[StateEvent[S,E], Transition[S]]() 28 | result.transitionsAny = newTable[S, Transition[S]]() 29 | result.setInitialState(initialState) 30 | 31 | proc addTransitionAny*[S,E](m: Machine[S,E], state: S, nextState: S) = 32 | m.transitionsAny[state] = (nextState, none(Callback)) 33 | 34 | proc addTransitionAny*[S,E](m: Machine[S,E], state, nextState: S, action: Callback) = 35 | m.transitionsAny[state] = (nextState, some(action)) 36 | 37 | proc addTransition*[S,E](m: Machine[S,E], state: S, event: E, nextState: S) = 38 | m.transitions[(state, event)] = (nextState, none(Callback)) 39 | 40 | proc addTransition*[S,E](m: Machine[S,E], state: S, event: E, nextState: S, action: Callback) = 41 | m.transitions[(state, event)] = (nextState, some(action)) 42 | 43 | proc setDefaultTransition*[S,E](m: Machine[S,E], state: S) = 44 | m.defaultTransition = some((state, none(Callback))) 45 | 46 | proc setDefaultTransition*[S,E](m: Machine[S,E], state: S, action: Callback) = 47 | m.defaultTransition = some((state, some(action))) 48 | 49 | proc getTransition*[S,E](m: Machine[S,E], event: E, state: S): Transition[S] = 50 | let map = (state, event) 51 | if m.transitions.hasKey(map): 52 | result = m.transitions[map] 53 | elif m.transitionsAny.hasKey(state): 54 | result = m.transitionsAny[state] 55 | elif m.defaultTransition.isSome: 56 | result = m.defaultTransition.get 57 | else: raise newException(TransitionNotFoundException, "Transition is not defined: Event($#) State($#)" % [$event, $state]) 58 | 59 | proc getCurrentState*(m: Machine): auto = 60 | m.currentState.get 61 | 62 | proc process*[S,E](m: Machine[S,E], event: E) = 63 | let transition = m.getTransition(event, m.currentState.get) 64 | if transition[1].isSome: 65 | get(transition[1])() 66 | m.currentState = some(transition[0]) 67 | #echo event, " ", m.currentState.get 68 | 69 | 70 | when isMainModule: 71 | proc cb() = 72 | echo "i'm evaporating" 73 | 74 | type 75 | State = enum 76 | SOLID 77 | LIQUID 78 | GAS 79 | PLASMA 80 | 81 | Event = enum 82 | MELT 83 | EVAPORATE 84 | SUBLIMATE 85 | IONIZE 86 | 87 | var m = newMachine[State, Event](LIQUID) 88 | #m.setDefaultTransition() 89 | m.addTransition(SOLID, MELT, LIQUID) 90 | m.addTransition(LIQUID, EVAPORATE, GAS, cb) 91 | m.addTransition(SOLID, SUBLIMATE, GAS) 92 | m.addTransition(GAS, IONIZE, PLASMA) 93 | m.addTransition(SOLID, MELT, LIQUID) 94 | 95 | assert m.getCurrentState() == LIQUID 96 | m.process(EVAPORATE) 97 | assert m.getCurrentState() == GAS 98 | m.process(IONIZE) 99 | assert m.getCurrentState() == PLASMA 100 | -------------------------------------------------------------------------------- /src/golden/compilation.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncfutures 3 | import asyncdispatch 4 | import strutils 5 | import sequtils 6 | 7 | import spec 8 | import invoke 9 | 10 | proc pathToCompilationTarget*(filename: string): string = 11 | ## calculate the path of a source file's compiled binary output 12 | assert filename.endsWith ".nim" 13 | var (head, tail) = filename.absolutePath.normalizedPath.splitPath 14 | tail.removeSuffix ".nim" 15 | result = head / tail 16 | 17 | proc sniffCompilerGitHash*(compiler: Gold): Future[string] {.async.} = 18 | ## determine the git hash of the compiler binary if possible; 19 | ## this should ideally compile a file to measure the version constants, too. 20 | const pattern = "git hash: " 21 | var binary = newFileDetailWithInfo(compiler.binary) 22 | let invocation = await invoke(binary, @["--version"]) 23 | if invocation.okay: 24 | for line in invocation.invokation.stdout.splitLines: 25 | if line.startsWith(pattern): 26 | let commit = line[pattern.len .. ^1] 27 | if commit.len == 40: 28 | result = commit 29 | break 30 | 31 | proc newCompiler*(hint: string = ""): Gold = 32 | var path: string 33 | if hint == "": 34 | path = getCurrentCompilerExe() 35 | else: 36 | path = hint 37 | # i know, it's expensive, but this is what we have right now 38 | result = newGold(aCompiler) 39 | var binary = newFileDetailWithInfo(path) 40 | result.binary = binary 41 | result.version = NimVersion 42 | result.major = NimMajor 43 | result.minor = NimMinor 44 | result.patch = NimPatch 45 | result.chash = waitfor result.sniffCompilerGitHash() 46 | 47 | proc newCompilation*(): Gold = 48 | var compiler = newGold(aCompiler) 49 | result = newGold(aCompilation) 50 | result.compilation = CompilationInfo() 51 | result.compiler = compiler 52 | 53 | proc newCompilation*(compiler: var Gold): Gold = 54 | result = newGold(aCompilation) 55 | result.compilation = CompilationInfo() 56 | result.compiler = compiler 57 | 58 | proc newCompilation*(compiler: var Gold; source: var Gold): Gold = 59 | let 60 | output = pathToCompilationTarget(source.file.path) 61 | 62 | result = newCompilation(compiler) 63 | var 64 | target = newFileDetail(output) 65 | result.source = source 66 | result.target = target 67 | 68 | proc newCompilation*(compiler: var Gold; filename: string): Gold = 69 | let 70 | target = pathToCompilationTarget(filename) 71 | 72 | result = newCompilation(compiler) 73 | var 74 | source = newFileDetailWithInfo(filename) 75 | binary = newFileDetail(target) 76 | result.source = source 77 | result.target = binary 78 | 79 | proc argumentsForCompilation*(compilation: var Gold; args: seq[string]): seq[string] = 80 | # support lazy folks 81 | if args.len == 0: 82 | result = @["c", "-d:danger"] 83 | elif args[0] notin ["c", "cpp", "js"]: 84 | result = @["c"].concat(args) 85 | else: 86 | result = args 87 | 88 | proc compileFile*(filename: string; arguments: seq[string] = @[]): Future[Gold] {.async.} = 89 | ## compile a source file and yield details of the event 90 | var 91 | compiler = newCompiler() 92 | gold = newCompilation(compiler, filename) 93 | # the compilation binary (the target output) is only partially built here 94 | # but at least the source detail is fully built 95 | args = gold.argumentsForCompilation(arguments) 96 | 97 | # add the source filenames to compilation arguments 98 | for source in gold.sources: 99 | args.add source.file.path 100 | 101 | # perform the compilation 102 | var 103 | binary = newFileDetailWithInfo(compiler.binary) 104 | invocation = await invoke(binary, args) 105 | gold.invocation = invocation 106 | if invocation.okay: 107 | # populate this partially-built file detail 108 | var target = newFileDetailWithInfo(gold.target) 109 | gold.target = target 110 | result = gold 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # golden 2 | 3 | A benchmarking tool that measures and records runtime of any executable and 4 | also happens to know how to compile Nim. 5 | 6 | - `cpp +/ nim-1.0` [![Build Status](https://travis-ci.org/disruptek/golden.svg?branch=master)](https://travis-ci.org/disruptek/golden) 7 | - `arc +/ cpp +/ nim-1.3` [![Build Status](https://travis-ci.org/disruptek/golden.svg?branch=devel)](https://travis-ci.org/disruptek/golden) 8 | 9 | The idea here is that we're gonna make a record of everything we run, 10 | everything we build, and be able to discover and pinpoint regressions 11 | automatically using native git-fu performed by the tool. Fire and forget! 12 | 13 | ## Installation 14 | 15 | ### Nimph 16 | 17 | ``` 18 | $ nimph clone golden 19 | ``` 20 | 21 | ### Nimble 22 | 23 | ``` 24 | $ nimble install golden 25 | ``` 26 | 27 | ## Usage 28 | 29 | If you pass it a binary, it'll run it a bunch of times and report some runtime 30 | statistics periodically. 31 | 32 | If you pass it some Nim source, it will compile it for you and report some 33 | compilation and runtime statistics periodically. 34 | 35 | By default, it will run until you interrupt it. 36 | 37 | ``` 38 | $ golden --truth=0.002 bench.nim 39 | compilations after 0s 40 | ┌────────┬──────────┬──────────┬──────────┬──────────┐ 41 | │ Builds │ Min │ Max │ Mean │ StdDev │ 42 | ├────────┼──────────┼──────────┼──────────┼──────────┤ 43 | │ 1 │ 0.396129 │ 0.396129 │ 0.396129 │ 0.000000 │ 44 | └────────┴──────────┴──────────┴──────────┴──────────┘ 45 | benchmark after 1s 46 | ┌────────┬──────────┬──────────┬──────────┬──────────┐ 47 | │ Runs │ Min │ Max │ Mean │ StdDev │ 48 | ├────────┼──────────┼──────────┼──────────┼──────────┤ 49 | │ 1 │ 1.959187 │ 1.959187 │ 1.959187 │ 0.000000 │ 50 | └────────┴──────────┴──────────┴──────────┴──────────┘ 51 | benchmark after 3s 52 | ┌────────┬──────────┬──────────┬──────────┬──────────┐ 53 | │ Runs │ Min │ Max │ Mean │ StdDev │ 54 | ├────────┼──────────┼──────────┼──────────┼──────────┤ 55 | │ 2 │ 1.958892 │ 1.959187 │ 1.959039 │ 0.000147 │ 56 | └────────┴──────────┴──────────┴──────────┴──────────┘ 57 | completed benchmark after 5s 58 | ┌────────┬──────────┬──────────┬──────────┬──────────┐ 59 | │ Runs │ Min │ Max │ Mean │ StdDev │ 60 | ├────────┼──────────┼──────────┼──────────┼──────────┤ 61 | │ 3 │ 1.958892 │ 1.961293 │ 1.959791 │ 0.001069 │ 62 | └────────┴──────────┴──────────┴──────────┴──────────┘ 63 | ``` 64 | 65 | Benchmarking the compilation of Nim itself: 66 | ``` 67 | $ cd ~/git/Nim 68 | $ golden koch -- boot -d:danger 69 | # ... 70 | ┌────────┬──────────┬──────────┬──────────┬──────────┐ 71 | │ # │ Min │ Max │ Mean │ StdDev │ 72 | ├────────┼──────────┼──────────┼──────────┼──────────┤ 73 | │ 12 │ 8.846606 │ 9.485832 │ 8.945023 │ 0.165638 │ 74 | └────────┴──────────┴──────────┴──────────┴──────────┘ 75 | ``` 76 | 77 | Benchmarking compilation of slow-to-compile Nim: 78 | 79 | ``` 80 | $ golden --compilation openapi.nim 81 | ┌────────┬───────────┬───────────┬───────────┬──────────┐ 82 | │ # │ Min │ Max │ Mean │ StdDev │ 83 | ├────────┼───────────┼───────────┼───────────┼──────────┤ 84 | │ 1 │ 91.946370 │ 91.946370 │ 91.946370 │ 0.000000 │ 85 | └────────┴───────────┴───────────┴───────────┴──────────┘ 86 | ┌────────┬───────────┬───────────┬───────────┬───────────┐ 87 | │ # │ Min │ Max │ Mean │ StdDev │ 88 | ├────────┼───────────┼───────────┼───────────┼───────────┤ 89 | │ 2 │ 29.271556 │ 91.946370 │ 60.608963 │ 31.337407 │ 90 | └────────┴───────────┴───────────┴───────────┴───────────┘ 91 | ``` 92 | 93 | ## Command Line Options 94 | 95 | - `truth` a float percentage indicating how much jitter you'll accept 96 | - `runtime` a float of seconds after which we should kill each invocation 97 | - `iterations` a number of invocations after which we should stop the benchmark 98 | - `storage` the path to a database file you wish to use; must end in `.golden-lmdb` 99 | - `interactive-forced` assume output friendly to humans 100 | - `json-output` assume output friendly to machines _(work in progress)_ 101 | - `color-forced` enable color output when not in `interactive` mode 102 | - `prune-outliers` throw out this percentage of aberrant invocations with long runtime in order to clean up the histogram 103 | - `dry-run` don't write any results to the database 104 | - `histogram-classes` the number of points in the histogram 105 | - `compilation-only` benchmark the Nim compiler on the given source(s) 106 | - `brief` only output the statistics at the completion of the benchmark 107 | - `never-output` never emit anything via stdout/stderr 108 | - `dump-output` always print the stdout/stderr of the benchmarked program 109 | - `--` the following arguments are passed to the compiler and runtime. Note that if you supply `-- cpp` for compilation via C++, you will need to supply your own defines such as `-d:danger`. 110 | 111 | ## License 112 | MIT 113 | -------------------------------------------------------------------------------- /src/golden/running.nim: -------------------------------------------------------------------------------- 1 | #[ 2 | 3 | stuff related to the RunningResult and statistics in a broad sense 4 | 5 | ]# 6 | import times 7 | import stats 8 | import lists 9 | import math 10 | import strformat 11 | import strutils 12 | 13 | import terminaltables 14 | import msgpack4nim 15 | 16 | import linkedlists 17 | 18 | export stats 19 | 20 | const 21 | billion* = 1_000_000_000 22 | export billion 23 | 24 | type 25 | StatValue* = float64 26 | ClassDimensions* = tuple 27 | count: int 28 | size: StatValue 29 | 30 | RunningResult*[T] = ref object 31 | list*: SinglyLinkedList[T] 32 | stat*: RunningStat 33 | 34 | proc toTerminalTable(running: RunningResult; name: string): ref TerminalTable = 35 | let 36 | stat = running.stat 37 | var 38 | row: seq[string] 39 | result = newUnicodeTable() 40 | result.setHeaders @[name, "Min", "Max", "Mean", "StdDev"] 41 | result.separateRows = false 42 | row.add fmt"{stat.n:>6d}" 43 | row.add fmt"{stat.min:>0.6f}" 44 | row.add fmt"{stat.max:>0.6f}" 45 | row.add fmt"{stat.mean:>0.6f}" 46 | row.add fmt"{stat.standardDeviation:>0.6f}" 47 | result.addRow row 48 | 49 | proc renderTable*(running: RunningResult; name: string): string = 50 | let table = running.toTerminalTable(name) 51 | result = table.render.strip 52 | 53 | proc `$`*(running: RunningResult): string = 54 | result = running.renderTable(" #") 55 | 56 | proc len*(running: RunningResult): int = 57 | result = running.stat.n 58 | 59 | proc isEmpty*(running: RunningResult): bool = 60 | result = running.list.isEmpty 61 | 62 | proc first*[T](running: RunningResult[T]): T = 63 | assert not running.isEmpty 64 | result = running.list.first 65 | 66 | converter toSeconds*(wall: Duration): StatValue = 67 | result = wall.inNanoSeconds.StatValue / billion 68 | 69 | proc standardScore*(stat: RunningStat; value: StatValue): StatValue = 70 | result = (value - stat.mean) / stat.standardDeviation 71 | 72 | proc classSize*(stat: RunningStat; count: int): StatValue = 73 | ## the step size for each element in the histogram 74 | let delta = stat.max - stat.min 75 | if delta > 0: 76 | result = delta / count.StatValue 77 | else: 78 | result = stat.max 79 | 80 | proc makeDimensions*(stat: RunningStat; maximum: int): ClassDimensions = 81 | ## the best dimensions for a histogram of <= maximum items 82 | if stat.n == 0: 83 | return (count: 0, size: 0.0) 84 | let count = min(maximum, stat.n) 85 | result = (count: count, size: stat.classSize(count)) 86 | 87 | proc crudeHistogram*(running: RunningResult; dims: ClassDimensions): seq[int] = 88 | ## make a simple histogram suitable for a text/image graph 89 | result = newSeqOfCap[int](dims.count) 90 | for i in 0 .. dims.count - 1: 91 | result.add 0 92 | let (smin, smax) = (running.stat.min, running.stat.max) 93 | for element in running.list.items: 94 | var n = 0 95 | let s = element.toStatValue 96 | if s == smin: 97 | n = 0 98 | elif s == smax: 99 | n = dims.count - 1 100 | else: 101 | assert dims.size > 0 102 | n = int((s - smin) / dims.size) 103 | result[n].inc 104 | 105 | proc prunePoint(stat: RunningStat; histogram: var seq[int]; dims: ClassDimensions; outlier: float): StatValue = 106 | ## find a good value above which we should prune outlier entries 107 | if stat.n <= 3: 108 | return 109 | var totalSum = sum(histogram).float 110 | while histogram[^1].float < totalSum * outlier: 111 | result = stat.min + (dims.size * histogram.high.StatValue) 112 | delete histogram, histogram.high 113 | totalSum = sum(histogram).float 114 | 115 | proc maybePrune*(running: var RunningResult; histogram: var seq[int]; 116 | dims: ClassDimensions; outlier: float): bool = 117 | ## maybe prune some outliers from the top of our histogram 118 | 119 | # first, see if we really want to prune anything 120 | let prunePoint = running.stat.prunePoint(histogram, dims, outlier) 121 | if prunePoint == 0: 122 | return 123 | 124 | # okay; turn seconds into a Duration and then prune anything bigger 125 | let pruneOffset = initDuration(nanoseconds = int(prunePoint * billion)) 126 | var head = running.list.head 127 | while true: 128 | while head.next != nil and head.next.value.toWallDuration > pruneOffset: 129 | head.removeNext 130 | if head.next == nil: 131 | break 132 | head = head.next 133 | # recompute stats 134 | running.reset 135 | # let the world know that we probably did something 136 | result = true 137 | 138 | iterator mitems*[T](running: RunningResult[T]): var T = 139 | for item in running.list.mitems: 140 | yield item 141 | 142 | iterator items*[T](running: RunningResult[T]): T = 143 | for item in running.list.items: 144 | yield item 145 | 146 | proc truthy*(running: RunningResult; honesty: float): bool = 147 | ## do we think we know enough about the running result to stop running? 148 | if running.len < 3: 149 | return 150 | if running.stat.mean * honesty > running.stat.standardDeviation: 151 | return true 152 | 153 | proc newRunningResult*[T](): RunningResult[T] = 154 | new result 155 | result.list = initSinglyLinkedList[T]() 156 | 157 | proc pack_type*[ByteStream](s: ByteStream; x: RunningResult) = 158 | s.pack_type(x.oid) 159 | s.pack_type(x.entry) 160 | s.pack_type(x.list) 161 | s.pack_type(x.stat) 162 | 163 | proc unpack_type*[ByteStream](s: ByteStream; x: var RunningResult) = 164 | s.unpack_type(x.oid) 165 | s.unpack_type(x.entry) 166 | s.unpack_type(x.list) 167 | s.unpack_type(x.stat) 168 | -------------------------------------------------------------------------------- /src/golden.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import options 3 | import asyncfutures 4 | import asyncdispatch 5 | import strutils 6 | 7 | import bump 8 | import cligen 9 | import gittyup 10 | 11 | import golden/spec 12 | import golden/benchmark 13 | import golden/compilation 14 | 15 | import golden/lm as dbImpl 16 | 17 | when false: 18 | proc shutdown(golden: Golden) {.async.} = 19 | gittyup.shutdown() 20 | 21 | proc `$`*(gold: Gold): string = 22 | case gold.kind: 23 | of aCompiler: 24 | result = $gold.compiler 25 | of anInvocation: 26 | result = $gold.invocation 27 | of aBenchmark: 28 | result = $gold.benchmark 29 | of aFile: 30 | result = $gold.file 31 | else: 32 | result = $gold.kind & $gold.oid & " entry " & $gold.created 33 | 34 | proc output*(golden: Golden; gold: Gold; desc: string = "") = 35 | if desc != "": 36 | gold.description = desc 37 | if Interactive in golden.options.flags: 38 | golden.output $gold 39 | if jsonOutput(golden): 40 | golden.output gold.toJson 41 | 42 | proc storageForTarget*(golden: Golden; target: string): string = 43 | if golden.options.storage != "": 44 | result = golden.options.storage 45 | elif target.endsWith ".nim": 46 | result = pathToCompilationTarget(target) 47 | else: 48 | result = target 49 | 50 | proc storageForTargets*(golden: Golden; targets: seq[string]): string = 51 | # see if we need to hint at a specific storage site 52 | if golden.options.storage != "": 53 | result = golden.options.storage 54 | elif targets.len == 1: 55 | result = golden.storageForTarget(targets[0]) 56 | else: 57 | quit "specify --storage to benchmark multiple programs" 58 | 59 | proc openDatabase*(golden: Golden; targets: seq[string]): Future[GoldenDatabase] {.async.} = 60 | ## load a database using a filename 61 | let storage = golden.storageForTargets(targets) 62 | result = await dbImpl.open(storage, golden.options.flags) 63 | 64 | proc removeDatabase*(db: var GoldenDatabase; flags: set[GoldenFlag]) = 65 | ## remove a database 66 | dbImpl.removeDatabase(db, flags) 67 | 68 | proc removeDatabase*(golden: Golden; targets: seq[string]) = 69 | ## remove a database without a database handle by opening it first 70 | var db = waitfor golden.openDatabase(targets) 71 | removeDatabase(db, golden.options.flags) 72 | 73 | iterator performBenchmarks(golden: Golden; targets: seq[string]): Future[Gold] = 74 | var 75 | db: GoldenDatabase 76 | 77 | db = waitfor golden.openDatabase(targets) 78 | # setup the db and prepare to close it down again 79 | defer: 80 | dbImpl.close(db) 81 | 82 | # compile-only mode, for benchmarking the compiler 83 | if CompileOnly in golden.options.flags: 84 | for filename in targets.items: 85 | if not filename.appearsToBeCompileableSource: 86 | quit filename & ": does not appear to be compileable Nim source" 87 | for filename in targets.items: 88 | yield golden.benchmarkCompiler(filename) 89 | 90 | # mostly-run mode, for benchmarking runtimes 91 | else: 92 | for filename in targets.items: 93 | if filename.appearsToBeCompileableSource: 94 | var bench = newBenchmarkResult() 95 | # compile it, then benchmark it 96 | for b in golden.benchmarkNim(bench, filename): 97 | # first is the compilations, next is binary benches 98 | yield b 99 | else: 100 | # just benchmark it; it's already executable, we hope 101 | yield golden.benchmark(filename, golden.options.arguments) 102 | 103 | proc golden(sources: seq[string]; brief = false; compilation_only = false; 104 | dump_output = false; iterations = 0; runtime = 0.0; 105 | never_output = false; color_forced = false; json_output = false; 106 | interactive_forced = false; prune_outliers = 0.0; 107 | histogram_classes = 10; truth = 0.0; dry_run = false; 108 | storage_path = "") {.used.} = 109 | ## Nim benchmarking tool; 110 | ## pass 1+ .nim source files to compile and benchmark 111 | var 112 | targets: seq[string] 113 | golden = newGolden() 114 | 115 | if not gittyup.init(): 116 | raise newException(OSError, "unable to init git") 117 | defer: 118 | if not gittyup.shutdown(): 119 | raise newException(OSError, "unable to shut git") 120 | 121 | if json_output: 122 | golden.options.flags.incl PipeOutput 123 | if interactive_forced: 124 | golden.options.flags.incl Interactive 125 | if Interactive in golden.options.flags and color_forced: 126 | golden.options.flags.incl ColorConsole 127 | when defined(plotGraphs): 128 | golden.options.flags.incl ConsoleGraphs 129 | if dry_run: 130 | golden.options.flags.incl DryRun 131 | if compilation_only: 132 | golden.options.flags.incl CompileOnly 133 | if dump_output: 134 | golden.options.flags.incl DumpOutput 135 | if never_output: 136 | golden.options.flags.incl NeverOutput 137 | if brief: 138 | golden.options.flags.incl Brief 139 | golden.output "in brief mode, you will only receive output at termination..." 140 | 141 | golden.options.honesty = truth 142 | golden.options.prune = prune_outliers 143 | golden.options.classes = histogram_classes 144 | golden.options.storage = storage_path 145 | if runtime != 0.0: 146 | golden.options.flags.incl TimeLimit 147 | golden.options.timeLimit = runtime 148 | if iterations != 0: 149 | golden.options.flags.incl RunLimit 150 | golden.options.runLimit = iterations 151 | 152 | # work around cligen --stopWords support 153 | for index in 1 .. paramCount(): 154 | targets.add paramStr(index) 155 | let dashdash = targets.find("--") 156 | if dashdash == -1: 157 | targets = sources 158 | else: 159 | golden.options.arguments = targets[dashdash + 1 .. ^1] 160 | targets = sources[0 ..< sources.len - golden.options.arguments.len] 161 | 162 | for filename in targets.items: 163 | if not filename.appearsBenchmarkable: 164 | quit "don't know how to benchmark `" & filename & "`" 165 | 166 | if targets.len == 0: 167 | quit "provide some files to benchmark, or\n" & paramStr(0) & " --help" 168 | 169 | # capture interrupts 170 | if Interactive in golden.options.flags: 171 | proc sigInt() {.noconv.} = 172 | raise newException(BenchmarkusInterruptus, "") 173 | setControlCHook(sigInt) 174 | 175 | for b in golden.performBenchmarks(targets): 176 | try: 177 | let mark = waitfor b 178 | if mark.terminated != Terminated.Success: 179 | quit(1) 180 | except BenchmarkusInterruptus: 181 | break 182 | except Exception as e: 183 | golden.output e.msg 184 | 185 | when isMainModule: 186 | import logging 187 | # log only warnings in release 188 | when defined(release) or defined(danger): 189 | let level = lvlWarn 190 | else: 191 | let level = lvlAll 192 | let logger = newConsoleLogger(useStderr=true, levelThreshold=level) 193 | addHandler(logger) 194 | 195 | const 196 | version = projectVersion() 197 | if version.isSome: 198 | clCfg.version = $version.get 199 | else: 200 | clCfg.version = "(unknown version)" 201 | 202 | dispatchCf(golden, cf = clCfg, stopWords = @["--"]) 203 | -------------------------------------------------------------------------------- /src/golden/invoke.nim: -------------------------------------------------------------------------------- 1 | import strutils 2 | import asyncdispatch 3 | import asyncfutures 4 | import streams 5 | import times 6 | import osproc 7 | import selectors 8 | 9 | import foreach 10 | 11 | import spec 12 | 13 | type 14 | Monitor = enum 15 | Output = "the process has some data for us on stdout" 16 | Errors = "the process has some data for us on stderr" 17 | Finished = "the process has finished" 18 | 19 | proc drainStreamInto(stream: Stream; output: var string) = 20 | while not stream.atEnd: 21 | output &= stream.readChar 22 | 23 | proc drain(ready: ReadyKey; stream: Stream; output: var string) = 24 | if Event.Read in ready.events: 25 | stream.drainStreamInto(output) 26 | elif {Event.Error} == ready.events: 27 | #stdmsg().writeLine "comms error: " & ready.errorCode.osErrorMsg 28 | discard 29 | else: 30 | assert ready.events.card == 0 31 | 32 | when declared(Tms): 33 | from posix import nil 34 | template cpuMark(gold: typed; invocation: typed) = 35 | discard posix.times(addr invocation.cpu) 36 | template cpuPreWait(gold: typed; invocation: typed) = 37 | var tms = Tms() 38 | discard posix.times(addr tms) 39 | invocation.cpu.tms_utime = tms.tms_utime - invocation.cpu.tms_utime 40 | invocation.cpu.tms_stime = tms.tms_stime - invocation.cpu.tms_stime 41 | invocation.cpu.tms_cutime = tms.tms_cutime - invocation.cpu.tms_cutime 42 | invocation.cpu.tms_cstime = tms.tms_cstime - invocation.cpu.tms_cstime 43 | template cpuPostWait(gold: typed; invocation: typed) = discard 44 | else: 45 | from posix import Rusage, Timeval, getrusage, RUSAGE_CHILDREN 46 | proc cpuMark(gold: Gold; invocation: var InvocationInfo) = 47 | invocation.cpu = Rusage() 48 | let ret = getrusage(RUSAGE_CHILDREN, addr invocation.cpu) 49 | assert ret == 0 50 | 51 | proc `$`*(t: Timeval): string = 52 | ## convenience 53 | result = $(t.tv_sec.float64 + (t.tv_usec.float64 / 1_000_000) ) 54 | 55 | proc sub(a: Timeval; b: Timeval): Timeval = 56 | result.tv_sec = posix.Time(a.tv_sec.int - b.tv_sec.int) 57 | result.tv_usec = a.tv_usec - b.tv_usec 58 | if result.tv_usec < 0: 59 | result.tv_sec.dec 60 | result.tv_usec.inc 1_000_000 61 | 62 | proc cpuPreWait(gold: Gold; invocation: var InvocationInfo) = 63 | discard 64 | 65 | proc cpuPostWait(gold: Gold; invocation: var InvocationInfo) = 66 | var 67 | ru = Rusage() 68 | let ret = getrusage(RUSAGE_CHILDREN, addr ru) 69 | assert ret == 0 70 | invocation.cpu.ru_utime = sub(ru.ru_utime, invocation.cpu.ru_utime) 71 | invocation.cpu.ru_stime = sub(ru.ru_stime, invocation.cpu.ru_utime) 72 | 73 | proc monitor(gold: var Gold; process: Process; deadline = -1.0) = 74 | ## keep a process's output streams empty, saving them into the 75 | ## invocation with other runtime details; deadline is an epochTime 76 | ## after which we should manually terminate the process 77 | var 78 | timeout = 1 # start with a timeout in the future 79 | clock = getTime() 80 | watcher = newSelector[Monitor]() 81 | invocation = gold.invokation 82 | 83 | # monitor whether the process has finished or produced output 84 | when defined(useProcessSignal): 85 | let signal = watcher.registerProcess(process.processId, Finished) 86 | watcher.registerHandle(process.outputHandle.int, {Event.Read}, Output) 87 | watcher.registerHandle(process.errorHandle.int, {Event.Read}, Errors) 88 | 89 | block running: 90 | try: 91 | while true: 92 | if deadline <= 0.0: 93 | timeout = -1 # wait forever if no deadline is specified 94 | # otherwise, reset the timeout if it hasn't passed 95 | elif timeout > 0: 96 | # cache the current time 97 | let rightNow = epochTime() 98 | block checktime: 99 | # we may break the checktime block before setting timeout to -1 100 | if rightNow < deadline: 101 | # the number of ms remaining until the deadline 102 | timeout = int( 1000 * (deadline - rightNow) ) 103 | # if there is time left, we're done here 104 | if timeout > 0: 105 | break checktime 106 | # otherwise, we'll fall through, setting the timeout to -1 107 | # which will cause us to kill the process... 108 | timeout = -1 109 | # if there's a deadline in place, see if we've passed it 110 | if deadline > 0.0 and timeout < 0: 111 | # the deadline has passed; kill the process 112 | process.terminate 113 | process.kill 114 | # wait for it to exit so that we pass through the loop below only one 115 | # additional time. 116 | # 117 | # if the process is wedged somehow, we will not continue to spawn more 118 | # invocations that will DoS the machine. 119 | invocation.code = process.waitForExit 120 | # make sure we catch any remaining output and 121 | # perform the expected measurements 122 | timeout = 0 123 | let events = watcher.select(timeout) 124 | foreach ready in events.items of ReadyKey: 125 | var kind: Monitor = watcher.getData(ready.fd) 126 | case kind: 127 | of Output: 128 | # keep the output stream from blocking 129 | ready.drain(process.outputStream, invocation.stdout) 130 | of Errors: 131 | # keep the errors stream from blocking 132 | ready.drain(process.errorStream, invocation.stderr) 133 | of Finished: 134 | # check the clock and cpu early 135 | cpuPreWait(gold, invocation) 136 | invocation.wall = getTime() - clock 137 | # drain any data in the streams 138 | process.outputStream.drainStreamInto invocation.stdout 139 | process.errorStream.drainStreamInto invocation.stderr 140 | break running 141 | when not defined(useProcessSignal): 142 | if process.peekExitCode != -1: 143 | # check the clock and cpu early 144 | cpuPreWait(gold, invocation) 145 | invocation.wall = getTime() - clock 146 | process.outputStream.drainStreamInto invocation.stdout 147 | process.errorStream.drainStreamInto invocation.stderr 148 | break 149 | if deadline >= 0: 150 | assert timeout > 0, "terminating process failed measurements" 151 | except IOSelectorsException as e: 152 | # merely report errors for database safety 153 | stdmsg().writeLine "error talkin' to process: " & e.msg 154 | 155 | try: 156 | # cleanup the selector 157 | when defined(useProcessSignal) and not defined(debugFdLeak): 158 | watcher.unregister signal 159 | watcher.close 160 | except Exception as e: 161 | # merely report errors for database safety 162 | stdmsg().writeLine e.msg 163 | 164 | # the process has exited, but this could be useful to Process 165 | # and in fact is needed for Rusage 166 | invocation.code = process.waitForExit 167 | cpuPostWait(gold, invocation) 168 | 169 | proc invoke*(exe: Gold; args: seq[string] = @[]; timeLimit = 0): Future[Gold] {.async.} = 170 | ## run a binary and yield info about its invocation; 171 | ## timeLimit is the number of ms to wait for the process to complete. 172 | ## a timeLimit of 0 means, "wait forever for completion." 173 | var 174 | gold = newInvocation(exe, args = nil) 175 | binary = gold.binary 176 | deadline = -1.0 177 | 178 | if timeLimit > 0: 179 | deadline = epochTime() + timeLimit.float / 1000 # timeLimit is in seconds 180 | 181 | # mark the current cpu time 182 | cpuMark(gold, gold.invokation) 183 | 184 | var 185 | process = startProcess(binary.file.path, args = args, options = {}) 186 | 187 | # watch the process to gather i/o and runtime details 188 | gold.monitor(process, deadline = deadline) 189 | # cleanup the process 190 | process.close 191 | 192 | result = gold 193 | 194 | proc invoke*(path: string; args: varargs[string, `$`]; timeLimit = -1): Future[Gold] = 195 | ## convenience invoke() 196 | var 197 | arguments: seq[string] 198 | binary = newFileDetailWithInfo(path) 199 | for a in args.items: 200 | arguments.add a 201 | result = binary.invoke(arguments, timeLimit = timeLimit) 202 | -------------------------------------------------------------------------------- /src/golden/db.nim: -------------------------------------------------------------------------------- 1 | #[ 2 | 3 | we're not gonna do much error checking, etc. 4 | 5 | ]# 6 | import os 7 | import times 8 | import asyncdispatch 9 | import asyncfutures 10 | import strutils 11 | 12 | import db_sqlite 13 | 14 | import fsm 15 | import spec 16 | import benchmark 17 | import running 18 | 19 | const ISO8601forDB* = initTimeFormat "yyyy-MM-dd\'T\'HH:mm:ss\'.\'fff" 20 | 21 | type 22 | SyncResult* = enum 23 | SyncOkay 24 | SyncRead 25 | SyncWrite 26 | 27 | DatabaseTables* = enum 28 | Meta 29 | Compilers = "CompilerInfo" 30 | Files = "FileDetail" 31 | 32 | DatabaseImpl* = ref object 33 | path: string 34 | store: FileInfo 35 | db: DbConn 36 | 37 | proc close*(self: DatabaseImpl) {.async.} = 38 | ## close the database 39 | self.db.close() 40 | 41 | type 42 | ModelVersion* = enum 43 | v0 = "(none)" 44 | v1 = "dragons; really alpha" 45 | 46 | Event = enum 47 | Upgrade 48 | Downgrade 49 | 50 | proc parseDuration(text: string): Duration = 51 | let f = text.parseFloat 52 | result = initDuration(nanoseconds = int64(billion * f)) 53 | 54 | proc getModelVersion(self: DatabaseImpl): ModelVersion = 55 | result = v0 56 | try: 57 | let row = self.db.getRow sql"""select value from Meta 58 | where name = "version"""" 59 | result = cast[ModelVersion](row[0].parseInt) 60 | except Exception: 61 | # blow up the database if we can't read a version from it 62 | discard 63 | 64 | proc setModelVersion(self: DatabaseImpl; version: ModelVersion) = 65 | self.db.exec sql"begin" 66 | self.db.exec sql"""delete from Meta where name = "version"""" 67 | self.db.exec sql"""insert into Meta (name, value) 68 | values (?, ?)""", "version", $ord(version) 69 | self.db.exec sql"commit" 70 | 71 | proc upgradeDatabase*(self: DatabaseImpl) = 72 | var currently = self.getModelVersion 73 | if currently == ModelVersion.high: 74 | return 75 | var mach = newMachine[ModelVersion, Event](currently) 76 | mach.addTransition v0, Upgrade, v1, proc () = 77 | for name in DatabaseTables.low .. DatabaseTables.high: 78 | self.db.exec sql""" 79 | drop table if exists ? 80 | """, $name 81 | self.db.exec sql""" 82 | create table Meta ( 83 | name varchar(100) not null, 84 | value varchar(100) not null ) 85 | """ 86 | self.db.exec sql""" 87 | create table FileDetail ( 88 | oid char(24), 89 | entry datetime, 90 | digest char(16), 91 | size int(4), 92 | path varchar(2048) 93 | ) 94 | """ 95 | self.db.exec sql""" 96 | create table CompilerInfo ( 97 | oid char(24), 98 | entry datetime, 99 | binary char(24), 100 | major int(4), 101 | minor int(4), 102 | patch int(4), 103 | chash char(40) 104 | ) 105 | """ 106 | self.db.exec sql""" 107 | create table CompilationInfo ( 108 | oid char(24), 109 | entry datetime, 110 | source char(24), 111 | compiler char(24), 112 | binary char(24), 113 | invocation char(24), 114 | runtime char(24) 115 | ) 116 | """ 117 | self.db.exec sql""" 118 | create table InvocationInfo ( 119 | oid char(24), 120 | entry datetime, 121 | binary char(24), 122 | runtime char(24) 123 | arguments varchar(2048) 124 | ) 125 | """ 126 | self.db.exec sql""" 127 | create table RuntimeInfo ( 128 | oid char(24), 129 | entry datetime, 130 | wall real(8), 131 | runtime char(24) 132 | arguments varchar(2048) 133 | ) 134 | """ 135 | self.setModelVersion(v1) 136 | 137 | while currently != ModelVersion.high: 138 | mach.process Upgrade 139 | currently = mach.getCurrentState 140 | 141 | method sync(self: DatabaseImpl; gold: var GoldObject): SyncResult {.base.} = 142 | raise newException(Defect, "sync not implemented") 143 | 144 | method renderTimestamp(gold: GoldObject): string {.base.} = 145 | ## turn a datetime into a string for the db 146 | gold.entry.inZone(utc()).format(ISO8601forDB) 147 | 148 | template loadTimestamp(gold: typed; datetime: string) = 149 | ## parse a db timestamp into a datetime 150 | gold.entry = datetime.parse(ISO8601forDB).inZone(local()) 151 | 152 | method sync(self: DatabaseImpl; detail: var FileDetail): SyncResult {.base.} = 153 | if not detail.dirty: 154 | return SyncOkay 155 | detail.dirty = false 156 | 157 | var row: Row 158 | row = self.db.getRow(sql"""select oid, entry, digest 159 | from FileDetail where digest = ?""", $detail.digest) 160 | if row[0] != "": 161 | result = SyncRead 162 | detail.oid = row[0].parseOid 163 | detail.loadTimestamp(row[1]) 164 | else: 165 | result = SyncWrite 166 | self.db.exec sql""" 167 | insert into FileDetail 168 | (oid, entry, digest, size, path) 169 | values 170 | (?, ?, ?, ?, ?) 171 | """, 172 | $detail.oid, 173 | detail.renderTimestamp, 174 | detail.digest, 175 | detail.size, 176 | detail.path 177 | 178 | method sync*(self: DatabaseImpl; compiler: var CompilerInfo): SyncResult {.base.} = 179 | discard self.sync(compiler.binary) 180 | if not compiler.dirty: 181 | return SyncOkay 182 | compiler.dirty = false 183 | 184 | var row: Row 185 | row = self.db.getRow(sql""" 186 | select oid, entry, major, minor, patch, chash 187 | from CompilerInfo where binary = ?""", compiler.binary.oid) 188 | 189 | if row[0] != "": 190 | result = SyncRead 191 | compiler.oid = row[0].parseOid 192 | compiler.loadTimestamp(row[1]) 193 | 194 | compiler.major = row[2].parseInt 195 | compiler.minor = row[3].parseInt 196 | compiler.patch = row[4].parseInt 197 | compiler.chash = row[5] 198 | else: 199 | result = SyncWrite 200 | self.db.exec sql""" 201 | insert into CompilerInfo 202 | (oid, entry, binary, major, minor, patch, chash) 203 | values 204 | (?, ?, ?, ?, ?, ?, ?) 205 | """, 206 | $compiler.oid, 207 | compiler.renderTimestamp, 208 | $compiler.binary.oid, 209 | $compiler.major, 210 | $compiler.minor, 211 | $compiler.patch, 212 | $compiler.chash 213 | 214 | method sync*(self: DatabaseImpl; invocation: var InvocationInfo): SyncResult {.base.} = 215 | ## binary, runningstats 216 | discard self.sync(invocation.binary) 217 | if not invocation.dirty: 218 | return SyncOkay 219 | invocation.dirty = false 220 | 221 | var row: Row 222 | row = self.db.getRow(sql""" 223 | select oid, entry, arguments, wall 224 | from InvocationInfo where oid = ?""", $invocation.oid) 225 | 226 | if row[0] != "": 227 | result = SyncRead 228 | invocation.oid = row[0].parseOid 229 | invocation.loadTimestamp(row[1]) 230 | invocation.arguments = row[2].split(" ") 231 | invocation.runtime.wall = row[3].parseDuration 232 | else: 233 | result = SyncWrite 234 | self.db.exec sql""" 235 | insert into InvocationInfo 236 | (oid, entry, arguments, wall) 237 | values 238 | (?, ?, ?, ?) 239 | """, 240 | $invocation.oid, 241 | invocation.renderTimestamp, 242 | invocation.arguments.join(" "), 243 | $invocation.runtime.wall 244 | 245 | method sync*(self: DatabaseImpl; compilation: var CompilationInfo): SyncResult {.base.} = 246 | if not compilation.dirty: 247 | return SyncOkay 248 | compilation.dirty = false 249 | 250 | discard self.sync(compilation.compiler) 251 | discard self.sync(compilation.source) 252 | discard self.sync(compilation.binary) 253 | discard self.sync(compilation.invocation) 254 | 255 | method sync*(self: DatabaseImpl; running: var RunningResult): SyncResult {.base.} = 256 | if running.len == 0: 257 | return 258 | if not running.dirty: 259 | return SyncOkay 260 | running.dirty = false 261 | 262 | for thing in running.mitems: 263 | discard self.sync(thing) 264 | 265 | method sync*(self: DatabaseImpl; benchmark: var BenchmarkResult): SyncResult {.base.} = 266 | discard self.sync(benchmark.binary) 267 | if not benchmark.dirty: 268 | return SyncOkay 269 | benchmark.dirty = false 270 | 271 | discard self.sync(benchmark.compilations) 272 | discard self.sync(benchmark.invocations) 273 | 274 | proc storagePath(filename: string): string = 275 | ## make up a good path for the database file 276 | var (head, tail) = filename.absolutePath.normalizedPath.splitPath 277 | if not filename.endsWith(".golden-db"): 278 | tail = "." & tail & ".golden-db" 279 | result = head / tail 280 | 281 | proc newDatabaseImpl*(filename: string): Future[DatabaseImpl] {.async.} = 282 | ## instantiate a database using the filename 283 | new result 284 | result.path = storagePath(filename) 285 | result.db = open(result.path, "", "", "") 286 | if not result.path.fileExists: 287 | waitfor result.close 288 | return await newDatabaseImpl(filename) 289 | result.store = getFileInfo(result.path) 290 | result.upgradeDatabase() 291 | -------------------------------------------------------------------------------- /src/golden/benchmark.nim: -------------------------------------------------------------------------------- 1 | #[ 2 | 3 | stuff related to the BenchmarkResult and benchmarking in a broad sense 4 | 5 | ]# 6 | import os 7 | import times 8 | import strutils 9 | import asyncdispatch 10 | import asyncfutures 11 | 12 | import spec 13 | import running 14 | import compilation 15 | import invoke 16 | 17 | when defined(plotGraphs): 18 | import osproc 19 | import plot 20 | 21 | proc `$`*(bench: BenchmarkResult): string = 22 | if bench.invocations.len > 0: 23 | let invocation = bench.invocations.first 24 | result = invocation.commandLine 25 | if bench.invocations.len > 0: 26 | result &= "\ninvocations:\n" & $bench.invocations 27 | 28 | proc newBenchmarkResult*(): Gold = 29 | result = newGold(aBenchmark) 30 | result.benchmark = BenchmarkResult() 31 | result.benchmark.invocations = newRunningResult[Gold]() 32 | 33 | proc output*(golden: Golden; benchmark: BenchmarkResult; started: Time; 34 | desc: string = "") = 35 | ## generally used to output a benchmark result periodically 36 | let since = getTime() - started 37 | golden.output desc & " after " & $since.inSeconds & "s" 38 | if benchmark.invocations.len > 0: 39 | var name: string 40 | let invocation = benchmark.invocations.first 41 | if invocation.kind == aCompilation: 42 | name = "Builds" 43 | else: 44 | name = "Runs" 45 | if not invocation.okay: 46 | golden.output invocation 47 | golden.output benchmark.invocations, name 48 | when defined(debug): 49 | goldenDebug() 50 | when defined(plotGraphs): 51 | while ConsoleGraphs in golden.options.flags: 52 | var 53 | dims = benchmark.invocations.stat.makeDimensions(golden.options.classes) 54 | histo = benchmark.invocations.crudeHistogram(dims) 55 | if benchmark.invocations.maybePrune(histo, dims, golden.options.prune): 56 | continue 57 | golden.output $histo 58 | # hangs if histo.len == 1 due to max-min == 0 59 | if histo.len <= 1: 60 | break 61 | let filename = plot.consolePlot(benchmark.invocations.stat, histo, dims) 62 | if os.getEnv("TERM", "") == "xterm-kitty": 63 | let kitty = "/usr/bin/kitty" 64 | if kitty.fileExists: 65 | var process = startProcess(kitty, args = @["+kitten", "icat", filename], options = {poInteractive, poParentStreams}) 66 | discard process.waitForExit 67 | break 68 | 69 | const 70 | executable = {fpUserExec, fpGroupExec, fpOthersExec} 71 | readable = {fpUserRead, fpGroupRead, fpOthersRead} 72 | 73 | proc appearsToBeReadable(path: string): bool = 74 | ## see if a file is readable 75 | var file: File 76 | result = file.open(path, mode = fmRead) 77 | if result: 78 | file.close 79 | 80 | proc veryLikelyRunnable(path: string; info: FileInfo): bool = 81 | ## weirdly, we don't seem to have a way to test if a file 82 | ## is going to be executable, so just estimate if we are 83 | ## likely able to execute the file 84 | let 85 | user = {fpUserExec, fpUserRead} * info.permissions 86 | group = {fpGroupExec, fpGroupRead} * info.permissions 87 | others = {fpOthersExec, fpOthersRead} * info.permissions 88 | r = readable * info.permissions 89 | x = executable * info.permissions 90 | 91 | if info.kind notin {pcFile, pcLinkToFile}: 92 | return false 93 | 94 | # if you can't read it, you can't run it 95 | if r.len == 0: 96 | return false 97 | # if you really can't read it, you can't run it 98 | if not path.appearsToBeReadable: 99 | return false 100 | 101 | # assume that something in readable is giving us read permissions 102 | # assume that we are in "Others" 103 | if fpOthersRead in r and fpOthersExec in x: 104 | return true 105 | 106 | # let's see if there's only one readable flag and it shares 107 | # a class with an executable flag... 108 | if r.len == 1: 109 | for r1 in r.items: 110 | for c in [user, group, others]: 111 | if r1 in c: 112 | if (x * c).len > 0: 113 | return true 114 | # okay, so it doesn't share the class, but if Others has Exec, 115 | # assume that we are in "Others" 116 | if fpOthersExec in x: 117 | return true 118 | 119 | # we might be able to execute it, but we might not! 120 | result = false 121 | 122 | proc appearsToBeCompileableSource*(path: string): bool = 123 | result = path.endsWith(".nim") and path.appearsToBeReadable 124 | 125 | proc appearsToBeExecutable*(path: string; info: FileInfo): bool = 126 | result = veryLikelyRunnable(path, info) 127 | result = result or (executable * info.permissions).len > 0 # lame 128 | 129 | proc appearsBenchmarkable*(path: string): bool = 130 | ## true if the path looks like something we can bench 131 | try: 132 | let info = getFileInfo(path) 133 | var detail = newFileDetail(path, info) 134 | if detail.file.kind notin {pcFile, pcLinkToFile}: 135 | return false 136 | if detail.file.path.appearsToBeCompileableSource: 137 | return true 138 | result = detail.file.path.appearsToBeExecutable(info) 139 | except OSError as e: 140 | stdmsg().writeLine(path & ": " & e.msg) 141 | return false 142 | 143 | proc benchmark*(golden: Golden; binary: Gold; 144 | arguments: seq[string]): Future[Gold] {.async.} = 145 | ## benchmark an arbitrary executable 146 | let 147 | wall = getTime() 148 | timeLimit = int( 1000 * golden.options.timeLimit ) 149 | var 150 | bench = newBenchmarkResult() 151 | invocation: Gold 152 | runs, outputs, fib = 0 153 | lastOutputTime = getTime() 154 | truthy = false 155 | secs: Duration 156 | 157 | bench.terminated = Terminated.Success 158 | try: 159 | while true: 160 | when defined(debugFdLeak): 161 | {.warning: "this build is for debugging fd leak".} 162 | invocation = await invoke("/usr/bin/lsof", "-p", getCurrentProcessId()) 163 | golden.output invocation.output.stdout 164 | invocation = await invoke(binary, arguments, timeLimit = timeLimit) 165 | runs.inc 166 | if invocation.okay: 167 | bench.invocations.add invocation 168 | if DumpOutput in golden.options.flags: 169 | golden.output invocation, "invocation", arguments = arguments 170 | else: 171 | golden.output invocation, "failed invocation", arguments = arguments 172 | secs = getTime() - wall 173 | truthy = bench.invocations.truthy(golden.options.honesty) 174 | if RunLimit in golden.options.flags: 175 | if runs >= golden.options.runLimit: 176 | truthy = true 177 | when false: 178 | # we use the time limit to limit runtime of each invocation now 179 | if TimeLimit in golden.options.flags: 180 | if secs.toSeconds >= golden.options.timeLimit: 181 | truthy = true 182 | when not defined(debug): 183 | secs = getTime() - lastOutputTime 184 | if not truthy and secs.inSeconds < fib: 185 | continue 186 | lastOutputTime = getTime() 187 | outputs.inc 188 | fib = fibonacci(outputs) 189 | if not invocation.okay: 190 | bench.terminated = Terminated.Failure 191 | break 192 | if truthy: 193 | break 194 | if Brief notin golden.options.flags: 195 | golden.output bench.benchmark, started = bench.created, "benchmark" 196 | except BenchmarkusInterruptus as e: 197 | bench.terminated = Terminated.Interrupt 198 | raise e 199 | except Exception as e: 200 | bench.terminated = Terminated.Failure 201 | raise e 202 | finally: 203 | result = bench 204 | var 205 | name = "execution" 206 | if not bench.invocations.isEmpty: 207 | var 208 | first = bench.invocations.first 209 | if first.kind == aCompilation: 210 | name = "compilation" 211 | else: 212 | name = "invocation" 213 | case bench.terminated: 214 | of Terminated.Interrupt: 215 | name &= " halted" 216 | of Terminated.Failure: 217 | name &= " failed" 218 | of Terminated.Success: 219 | name &= " complete" 220 | golden.output bench.benchmark, started = bench.created, name 221 | 222 | proc benchmark*(golden: Golden; filename: string; 223 | arguments: seq[string]): Future[Gold] {.async.} = 224 | result = await golden.benchmark(newFileDetailWithInfo(filename), arguments) 225 | 226 | proc benchmarkCompiler*(golden: Golden; 227 | filename: string): Future[Gold] {.async.} = 228 | assert CompileOnly in golden.options.flags 229 | var 230 | compiler = newCompiler() 231 | gold = newCompilation(compiler, filename) 232 | args = gold.argumentsForCompilation(golden.options.arguments) 233 | # add the source filename to compilation arguments 234 | for source in gold.sources: 235 | args.add source.file.path 236 | result = await golden.benchmark(compiler.binary, args) 237 | 238 | iterator benchmarkNim*(golden: Golden; gold: var Gold; 239 | filename: string): Future[Gold] = 240 | ## benchmark a source file 241 | assert CompileOnly notin golden.options.flags 242 | var 243 | future = newFuture[Gold]() 244 | compilation = waitfor compileFile(filename, golden.options.arguments) 245 | bench = gold.benchmark 246 | 247 | # the compilation is pretty solid; let's add it to the benchmark 248 | bench.invocations.add compilation 249 | # and yield it so the user can see the compilation result 250 | future.complete(gold) 251 | yield future 252 | 253 | # if the compilation was successful, 254 | # we go on to yield a benchmark of the executable we just built 255 | if compilation.okay: 256 | yield golden.benchmark(compilation.target, golden.options.arguments) 257 | -------------------------------------------------------------------------------- /src/golden/lm.nim: -------------------------------------------------------------------------------- 1 | import os 2 | import times 3 | import asyncdispatch 4 | import asyncfutures 5 | import strutils 6 | import options 7 | 8 | import msgpack4nim 9 | import lmdb 10 | 11 | import fsm 12 | import spec 13 | #import benchmark 14 | #import running 15 | import linkedlists 16 | 17 | const 18 | ISO8601forDB* = initTimeFormat "yyyy-MM-dd\'T\'HH:mm:ss\'.\'fff" 19 | # we probably only need one, but 20 | # we might need as many as two for a migration 21 | MAXDBS = 2 22 | 23 | when defined(LongWayHome): 24 | import posix 25 | when defined(Heapster): 26 | {.warning: "heapster".} 27 | 28 | ##[ 29 | 30 | Here's how the database works: 31 | 32 | There are three types of objects which have three unique forms of key. 33 | 34 | 1) Gold objects 35 | - every Gold has an oid 36 | - the stringified Oid is used as the key in this key/value store 37 | - this is a 24-char string 38 | - use msgpack to unpack the value into the Gold object 39 | 40 | 2) File Checksums 41 | - every FileDetail has a digest representing its contents in SHA or MD5 42 | - this digest is used as the key in this key/value store 43 | - this is a 20-(or 16)-char string 44 | - the value retrieved is the Oid of the FileDetail object 45 | 46 | FileDetail is associated with 47 | - Compilations (source or binary) 48 | - Benchmarks (binary) 49 | - Compilers (binary) 50 | - Invocations (binary) -- currently NOT saved in the database 51 | 52 | 3) Git References 53 | - this is a 40-char string 54 | 55 | To get data from the database, you start with any of these three keys, which 56 | gives you your entry. This seems like an extra step, but the fact is, the data 57 | is useless if you cannot associate it to something you have in-hand, and so you 58 | are simply going to end up gathering that content anyway in order to confirm 59 | that it matches a value in the database. 60 | 61 | ]## 62 | 63 | type 64 | GoldenDatabase* = ref object 65 | path: string 66 | store: FileInfo 67 | version: ModelVersion 68 | db: ptr MDBEnv 69 | flags: set[GoldenFlag] 70 | when defined(Heapster): 71 | ## this won't help 72 | #env: ref Env 73 | #txn: ref Txn 74 | 75 | # these should only be used for assertions; not for export 76 | proc isOpen(self: GoldenDatabase): bool {.inline.} = self.db != nil 77 | proc isClosed(self: GoldenDatabase): bool {.inline.} = self.db == nil 78 | 79 | proc close*(self: var GoldenDatabase) = 80 | ## close the database 81 | if self != nil: 82 | if self.isOpen: 83 | when defined(LongWayHome): 84 | goldenDebug() 85 | when defined(debug): 86 | echo "OLD CLOSE" 87 | mdb_env_close(self.db) 88 | when defined(debug): 89 | echo "OLD CLOSE complete" 90 | else: 91 | when defined(debug): 92 | echo "NO CLOSE" 93 | when defined(Heapster): 94 | when compiles(self.env): 95 | when defined(debug): 96 | echo "HEAPSTER env to nil" 97 | self.env = nil 98 | # XXX: without setting this to nil, it crashes after a few 99 | # iterations... but why? 100 | self.db = nil 101 | 102 | proc removeStorage(path: string) = 103 | if existsDir(path): 104 | removeDir(path) 105 | assert not existsDir(path) 106 | 107 | proc createStorage(path: string) = 108 | if existsDir(path): 109 | return 110 | createDir(path) 111 | assert existsDir(path) 112 | 113 | proc removeDatabase*(self: var GoldenDatabase; flags: set[GoldenFlag]) = 114 | ## remove the database from the filesystem 115 | self.close 116 | assert self.isClosed 117 | if DryRun notin flags: 118 | removeStorage(self.path) 119 | 120 | when defined(LongWayHome): 121 | proc umaskFriendlyPerms*(executable: bool): Mode = 122 | ## compute permissions for new files which are sensitive to umask 123 | 124 | # set it to 0 but read the last value 125 | result = umask(0) 126 | # set it to that value and discard zero 127 | discard umask(result) 128 | 129 | if executable: 130 | result = S_IWUSR.Mode or S_IRUSR.Mode or S_IXUSR.Mode or (result xor 0o777) 131 | else: 132 | result = S_IWUSR.Mode or S_IRUSR.Mode or (result xor 0o666) 133 | 134 | proc open(self: var GoldenDatabase; path: string) = 135 | ## open the database 136 | assert self.isClosed 137 | when defined(LongWayHome): 138 | var 139 | flags: cuint = 0 140 | else: 141 | var 142 | flags: int = 0 143 | 144 | if DryRun in self.flags: 145 | flags = MDB_RdOnly 146 | else: 147 | flags = 0 148 | createStorage(path) 149 | when defined(LongWayHome): 150 | when defined(debug): 151 | echo "OPEN DB" 152 | let mode = umaskFriendlyPerms(executable = false) 153 | when defined(Heapster): 154 | proc heapEnv(): ref Env = 155 | when defined(debug): 156 | echo "HEAPSTER heap env" 157 | new result 158 | when compiles(self.env): 159 | self.env = heapEnv() 160 | GC_ref(self.env) 161 | self.db = addr self.env[] 162 | else: 163 | var env = heapEnv() 164 | GC_ref(env) 165 | self.db = addr env[] 166 | else: 167 | var e: ptr MDBEnv 168 | self.db = e 169 | if mdb_env_create(addr self.db) != 0: 170 | raise newException(IOError, "unable to instantiate db") 171 | if mdb_env_set_maxdbs(self.db, MAXDBS) != 0: 172 | raise newException(IOError, "unable to set max dbs") 173 | if mdb_env_open(self.db, path.cstring, flags, mode) != 0: 174 | raise newException(IOError, "unable to open db") 175 | else: 176 | self.db = newLMDBEnv(path, maxdbs = MAXDBS, openflags = flags) 177 | goldenDebug() 178 | assert self.isOpen 179 | 180 | proc newTransaction(self: GoldenDatabase): ptr MDBTxn = 181 | assert self.isOpen 182 | var flags: cuint 183 | if DryRun in self.flags: 184 | flags = MDB_RdOnly 185 | else: 186 | flags = 0 187 | # 188 | # i'm labelling this with the `when` simply so i can mark this as another 189 | # area where a memory change for LongWayHome may end up being relevant 190 | when true or defined(LongWayHome): 191 | var 192 | parent: ptr MDBTxn 193 | 194 | when defined(Heapster): 195 | proc heapTxn(): ref Txn = 196 | when defined(debug): 197 | echo "HEAPSTER heap txn" 198 | new result 199 | 200 | when compiles(self.txn): 201 | self.txn = heapTxn() 202 | GC_ref(self.txn) 203 | result = addr self.txn[] 204 | else: 205 | var txn = heapTxn() 206 | GC_ref(txn) 207 | result = addr txn[] 208 | else: 209 | var txn: ptr MDB_Txn 210 | result = txn 211 | assert parent == nil 212 | if mdb_txn_begin(self.db, parent, flags = flags, addr result) != 0: 213 | raise newException(IOError, "unable to begin transaction") 214 | else: 215 | # but, we cannot use this with DryRun, so it's an error to try 216 | {.error: "this build doesn't work with DryRun".} 217 | result = newTxn(self.db) 218 | assert result != nil 219 | 220 | proc newHandle(self: GoldenDatabase; transaction: ptr MDBTxn; 221 | version: ModelVersion): MDBDbi = 222 | assert self.isOpen 223 | var flags: cuint 224 | if DryRun in self.flags: 225 | flags = 0 226 | else: 227 | flags = MDB_Create 228 | if mdb_dbi_open(transaction, $ord(version), flags, addr result) != 0: 229 | raise newException(IOError, "unable to create db handle") 230 | 231 | proc newHandle(self: GoldenDatabase; transaction: ptr MDBTxn): MDBDbi = 232 | result = self.newHandle(transaction, self.version) 233 | 234 | proc getModelVersion(self: GoldenDatabase): ModelVersion = 235 | assert self.isOpen 236 | result = ModelVersion.low 237 | let 238 | transaction = self.newTransaction 239 | defer: 240 | mdb_txn_abort(transaction) 241 | 242 | for version in countDown(ModelVersion.high, ModelVersion.low): 243 | try: 244 | # just try to open all the known versions 245 | discard self.newHandle(transaction, version) 246 | # if we were successful, that's our version 247 | result = version 248 | break 249 | except Exception: 250 | discard 251 | 252 | proc setModelVersion(self: GoldenDatabase; version: ModelVersion) = 253 | ## noop; the version is set by any write 254 | assert self.isOpen 255 | 256 | proc upgradeDatabase*(self: GoldenDatabase): ModelVersion = 257 | result = self.getModelVersion 258 | if result == ModelVersion.high: 259 | return 260 | var mach = newMachine[ModelVersion, ModelEvent](result) 261 | mach.addTransition v0, Upgrade, v1, proc () = 262 | self.setModelVersion(v1) 263 | 264 | while result != ModelVersion.high: 265 | mach.process Upgrade 266 | result = mach.getCurrentState 267 | 268 | when false: 269 | proc parseDuration(text: string): Duration = 270 | let f = text.parseFloat 271 | result = initDuration(nanoseconds = int64(billion * f)) 272 | 273 | proc utcTzInfo(time: Time): ZonedTime = 274 | result = ZonedTime(utcOffset: 0 * 3600, isDst: false, time: time) 275 | 276 | let tzUTC* = newTimezone("Somewhere/UTC", utcTzInfo, utcTzInfo) 277 | 278 | proc get(transaction: ptr MDBTxn; handle: MDBDbi; key: string): string = 279 | var 280 | key = MDBVal(mvSize: key.len.uint, mvData: key.cstring) 281 | value: MDBVal 282 | 283 | if mdb_get(transaction, handle, addr(key), addr(value)) != 0: 284 | raise newException(IOError, "unable to get value for key") 285 | 286 | result = newStringOfCap(value.mvSize) 287 | result.setLen(value.mvSize) 288 | copyMem(cast[pointer](result.cstring), cast[pointer](value.mvData), 289 | value.mvSize) 290 | assert result.len == value.mvSize.int 291 | 292 | proc put(transaction: ptr MDBTxn; handle: MDBDbi; 293 | key: string; value: string; flags = 0) = 294 | var 295 | key = MDBVal(mvSize: key.len.uint, mvData: key.cstring) 296 | value = MDBVal(mvSize: value.len.uint, mvData: value.cstring) 297 | 298 | if mdb_put(transaction, handle, addr key, addr value, flags.cuint) != 0: 299 | raise newException(IOError, "unable to put value for key") 300 | 301 | proc fetchViaOid(transaction: ptr MDBTxn; 302 | handle: MDBDbi; oid: Oid): Option[string] = 303 | defer: 304 | mdb_txn_abort(transaction) 305 | 306 | result = get(transaction, handle, $oid).some 307 | 308 | proc fetchVia*(self: GoldenDatabase; oid: Oid): Option[string] = 309 | assert self.isOpen 310 | let 311 | transaction = self.newTransaction 312 | handle = self.newHandle(transaction) 313 | result = fetchViaOid(transaction, handle, oid) 314 | 315 | proc read*(self: GoldenDatabase; gold: var Gold) = 316 | assert self.isOpen 317 | assert not gold.dirty 318 | var 319 | transaction = self.newTransaction 320 | handle = self.newHandle(transaction) 321 | defer: 322 | mdb_txn_abort(transaction) 323 | let existing = transaction.get(handle, $gold.oid) 324 | unpack(existing, gold) 325 | gold.dirty = false 326 | 327 | proc write*(self: GoldenDatabase; gold: var Gold) = 328 | assert self.isOpen 329 | assert gold.dirty 330 | let 331 | transaction = self.newTransaction 332 | handle = self.newHandle(transaction) 333 | try: 334 | transaction.put(handle, $gold.oid, pack(gold), MDB_NoOverWrite) 335 | if mdb_txn_commit(transaction) != 0: 336 | raise newException(IOError, "unable to commit transaction") 337 | gold.dirty = false 338 | except Exception as e: 339 | mdb_txn_abort(transaction) 340 | raise e 341 | 342 | proc storagePath(filename: string): string = 343 | ## make up a good path for the database file 344 | var (head, tail) = filename.absolutePath.normalizedPath.splitPath 345 | # we're gonna assume that if you are pointing to a .golden-lmdb, 346 | # and you named/renamed it, that you might not want the leading `.` 347 | if not filename.endsWith(".golden-lmdb"): 348 | tail = "." & tail & ".golden-lmdb" 349 | result = head / tail 350 | 351 | proc open*(filename: string; flags: set[GoldenFlag]): Future[GoldenDatabase] {.async.} = 352 | ## instantiate a database using the filename 353 | new result 354 | result.db = nil 355 | result.path = storagePath(filename) 356 | result.flags = flags 357 | result.open(result.path) 358 | result.version = result.upgradeDatabase() 359 | result.store = getFileInfo(result.path) 360 | -------------------------------------------------------------------------------- /src/golden/spec.nim: -------------------------------------------------------------------------------- 1 | #[ 2 | 3 | basic types and operations likely shared by all modules 4 | 5 | ]# 6 | import os 7 | import times 8 | import oids 9 | import strutils 10 | import terminal 11 | import stats 12 | import lists 13 | import json 14 | import tables 15 | import sequtils 16 | import strformat 17 | import terminal 18 | 19 | #from posix import Tms 20 | 21 | when defined(useSHA): 22 | import std/sha1 23 | else: 24 | import md5 25 | 26 | import msgpack4nim 27 | 28 | import running 29 | 30 | export oids 31 | export times 32 | export terminal 33 | 34 | const 35 | ISO8601noTZ* = initTimeFormat "yyyy-MM-dd\'T\'HH:mm:ss\'.\'fff\'Z\'" 36 | billion* = 1_000_000_000 37 | 38 | when declared(Tms): 39 | export Tms 40 | type CpuDuration* = Tms 41 | else: 42 | from posix import Rusage 43 | type CpuDuration* = Rusage 44 | 45 | type 46 | BenchmarkusInterruptus* = IOError 47 | 48 | ModelVersion* = enum 49 | v0 = "(none)" 50 | v1 = "dragons; really alpha" 51 | 52 | ModelEvent* = enum Upgrade, Downgrade 53 | 54 | # ordinal value is used as a magic number! 55 | GoldKind* = enum 56 | aFile = "📂" 57 | #aRuntime = "⏱️" 58 | #anOutput = "📢" 59 | aCompiler = "🧰" 60 | aCompilation = "🎯" 61 | anInvocation = "🎽" 62 | aBenchmark = "🏁" 63 | aGolden = "👑" 64 | #aPackageOfCourse = "📦" 65 | #aTestHaHa = "🧪" 66 | #aRocketDuh = "🚀" 67 | #aLinkerGetIt = "🔗" 68 | 69 | Gold* = ref object 70 | oid*: Oid 71 | description*: string 72 | links: GoldLinks 73 | when defined(StoreEntry): 74 | entry*: DateTime 75 | dirty*: bool 76 | case kind*: GoldKind 77 | of aFile: 78 | file*: FileDetail 79 | of aCompiler: 80 | version*: string 81 | major*: int 82 | minor*: int 83 | patch*: int 84 | chash*: string 85 | of aCompilation: 86 | compilation: CompilationInfo 87 | of anInvocation: 88 | invokation*: InvocationInfo 89 | of aBenchmark: 90 | benchmark*: BenchmarkResult 91 | terminated*: Terminated 92 | of aGolden: 93 | options*: GoldenOptions 94 | 95 | LinkFlag = enum 96 | Incoming 97 | Outgoing 98 | Unique 99 | Directory 100 | Binary 101 | Source 102 | Stdout 103 | Stderr 104 | Stdin 105 | Input 106 | Output 107 | 108 | LinkTarget = ref object 109 | oid: Oid 110 | kind: GoldKind 111 | 112 | Link = ref object 113 | flags: set[LinkFlag] 114 | source: LinkTarget 115 | target: LinkTarget 116 | entry: Time 117 | dirty: bool 118 | 119 | GoldLinks = ref object 120 | dad: Gold 121 | ins: seq[Link] 122 | outs: seq[Link] 123 | group: GoldGroup 124 | flags: TableRef[Oid, set[LinkFlag]] 125 | dirty: bool 126 | 127 | GoldGroup* = ref object 128 | cache: TableRef[Oid, Gold] 129 | 130 | LinkPairs = tuple[flags: set[LinkFlag], gold: Gold] 131 | 132 | FileSize* = BiggestInt 133 | FileDetail* = ref object 134 | kind*: PathComponent 135 | digest*: string 136 | size*: BiggestInt 137 | path*: string 138 | mtime*: Time 139 | 140 | WallDuration* = Duration 141 | MemorySize* = BiggestInt 142 | 143 | InvocationInfo* = ref object 144 | wall*: WallDuration 145 | cpu*: CpuDuration 146 | memory*: MemorySize 147 | arguments*: ref seq[string] 148 | stdout*: string 149 | stderr*: string 150 | code*: int 151 | 152 | CompilationInfo* = ref object 153 | compiler*: Gold 154 | invocation*: Gold 155 | source*: Gold 156 | binary*: Gold 157 | 158 | Terminated* {.pure.} = enum 159 | Success 160 | Failure 161 | Interrupt 162 | 163 | BenchmarkResult* = ref object 164 | binary*: Gold 165 | compilations* {.deprecated.}: RunningResult[Gold] 166 | invocations*: RunningResult[Gold] 167 | 168 | GoldenFlag* = enum 169 | Interactive 170 | PipeOutput 171 | ColorConsole 172 | ConsoleGraphs 173 | DryRun 174 | CompileOnly 175 | TimeLimit 176 | RunLimit 177 | DumpOutput 178 | NeverOutput 179 | Brief 180 | 181 | GoldenOptions* = object 182 | flags*: set[GoldenFlag] 183 | arguments*: seq[string] 184 | honesty*: float 185 | prune*: float 186 | classes*: int 187 | storage*: string 188 | timeLimit*: float 189 | runLimit*: int 190 | 191 | Golden* = object 192 | options*: GoldenOptions 193 | 194 | proc created*(gold: Gold): Time {.inline.} = 195 | ## when the object was originally created 196 | gold.oid.generatedTime 197 | 198 | proc `==`(a, b: LinkTarget): bool {.inline.} = a.oid == b.oid 199 | 200 | proc newGoldGroup(): GoldGroup = 201 | result = GoldGroup(cache: newTable[Oid, Gold]()) 202 | 203 | proc newGoldLinks(gold: Gold): GoldLinks = 204 | result = GoldLinks(dad: gold, ins: @[], outs: @[]) 205 | result.group = newGoldGroup() 206 | result.flags = newTable[Oid, set[LinkFlag]]() 207 | 208 | proc init(gold: var Gold) = 209 | gold.links = newGoldLinks(gold) 210 | 211 | proc newGold*(kind: GoldKind): Gold = 212 | ## create a new instance that we may want to save in the database 213 | when defined(StoreEntry): 214 | result = Gold(kind: kind, oid: genOid(), entry: now(), dirty: true) 215 | else: 216 | result = Gold(kind: kind, oid: genOid(), dirty: true) 217 | result.init 218 | 219 | when defined(StoreEntry): 220 | proc newGold(kind: GoldKind; oid: Oid; entry: DateTime): Gold = 221 | ## prep a new Gold object for database paint 222 | result = Gold(kind: kind, oid: oid, entry: entry, dirty: false) 223 | result.init 224 | else: 225 | proc newGold(kind: GoldKind; oid: Oid): Gold = 226 | ## prep a new Gold object for database paint 227 | result = Gold(kind: kind, oid: oid, dirty: false) 228 | result.init 229 | 230 | proc digestOf*(content: string): string = 231 | ## calculate the digest of a string 232 | when defined(useSHA): 233 | result = $secureHash(content) 234 | else: 235 | result = $toMD5(content) 236 | 237 | proc digestOfFileContents(path: string): string = 238 | ## calculate the digest of a file 239 | assert path.fileExists 240 | when defined(useSHA): 241 | result = $secureHashFile(path) 242 | else: 243 | result = digestOf(readFile(path)) 244 | 245 | proc newFileDetail*(path: string): Gold = 246 | result = newGold(aFile) 247 | result.file = FileDetail(path: path) 248 | 249 | proc newFileDetail*(path: string; size: FileSize; digest: string): Gold = 250 | result = newFileDetail(path) 251 | result.file.size = size 252 | result.file.digest = digest 253 | 254 | proc newFileDetail*(path: string; info: FileInfo): Gold = 255 | let normal = path.absolutePath.normalizedPath 256 | result = newFileDetail(normal, info.size, digestOfFileContents(normal)) 257 | result.file.mtime = info.lastWriteTime 258 | result.file.kind = info.kind 259 | 260 | proc newFileDetailWithInfo*(path: string): Gold = 261 | assert path.fileExists, "path `" & path & "` does not exist" 262 | result = newFileDetail(path, getFileInfo(path)) 263 | 264 | proc newFileDetailWithInfo*(gold: Gold): Gold = 265 | result = newFileDetailWithInfo(gold.file.path) 266 | 267 | when defined(GoldenGold): 268 | proc newGolden*(): Gold = 269 | result = newGold(aGolden) 270 | if stdmsg().isatty: 271 | result.options.flags.incl Interactive 272 | result.options.flags.incl ColorConsole 273 | else: 274 | result.options.flags.incl PipeOutput 275 | else: 276 | proc newGolden*(): Golden = 277 | if stdmsg().isatty: 278 | result.options.flags.incl Interactive 279 | result.options.flags.incl ColorConsole 280 | else: 281 | result.options.flags.incl PipeOutput 282 | 283 | proc linkTarget(gold: Gold): LinkTarget = 284 | result = LinkTarget(oid: gold.oid, kind: gold.kind) 285 | 286 | proc newLink(source: Gold; flags: set[LinkFlag]; target: Gold; 287 | dirty = true): Link = 288 | result = Link(flags: flags, entry: getTime(), dirty: dirty, 289 | source: source.linkTarget, target: target.linkTarget) 290 | 291 | iterator values(group: GoldGroup): Gold = 292 | for gold in group.cache.values: 293 | yield gold 294 | 295 | proc `[]`(group: GoldGroup; key: Oid): Gold = 296 | result = group.cache[key] 297 | 298 | iterator pairs(links: GoldLinks): LinkPairs = 299 | for oid, flags in links.flags.pairs: 300 | yield (flags: flags, gold: links.group[oid]) 301 | 302 | iterator `[]`(links: GoldLinks; kind: GoldKind): Gold = 303 | for gold in links.group.values: 304 | if gold.kind == kind: 305 | yield gold 306 | 307 | proc `[]`(links: GoldLinks; kind: GoldKind): Gold = 308 | for gold in links.group.values: 309 | if gold.kind == kind: 310 | return gold 311 | 312 | proc `[]`*(gold: Gold; kind: GoldKind): Gold = 313 | result = gold.links[kind] 314 | 315 | iterator `{}`*(links: GoldLinks; flag: LinkFlag): Gold = 316 | if flag == Incoming: 317 | for link in links.ins: 318 | yield links.group[link.target.oid] 319 | elif flag == Outgoing: 320 | for link in links.outs: 321 | yield links.group[link.target.oid] 322 | else: 323 | for flags, gold in links.pairs: 324 | if flag in flags: 325 | yield gold 326 | 327 | iterator `{}`*(links: GoldLinks; flags: set[LinkFlag]): Gold = 328 | for tags, gold in links.pairs: 329 | if (flags - tags).len == 0: 330 | yield gold 331 | 332 | iterator `{}`*(links: GoldLinks; flags: varargs[LinkFlag]): Gold = 333 | var tags: set[LinkFlag] 334 | for flag in flags: 335 | tags.incl flag 336 | for gold in links{tags}: 337 | yield gold 338 | 339 | proc contains(group: GoldGroup; oid: Oid): bool = 340 | result = oid in group.cache 341 | 342 | proc contains(group: GoldGroup; gold: Gold): bool = 343 | result = gold.oid in group 344 | 345 | proc contains(links: GoldLinks; kind: GoldKind): bool = 346 | for gold in links.group.values: 347 | if gold.kind == kind: 348 | return true 349 | 350 | proc excl(group: GoldGroup; oid: Oid) = 351 | ## exclude an oid from the group 352 | group.cache.del oid 353 | 354 | proc `==`(a, b: Link): bool {.inline.} = 355 | ## two links are equal if they refer to the same endpoints 356 | result = a.source == b.source and a.target == b.target 357 | 358 | proc incl(group: GoldGroup; gold: Gold) = 359 | ## add gold to a group; no duplicate entries 360 | if gold in group: 361 | return 362 | group.cache[gold.oid] = gold 363 | 364 | proc excl(links: var GoldLinks; link: Link) = 365 | ## remove a link 366 | if link.target.oid notin links.flags: 367 | return 368 | if Outgoing in link.flags: 369 | links.outs = links.outs.filterIt it != link 370 | if Incoming in link.flags: 371 | links.ins = links.ins.filterIt it != link 372 | links.flags.del link.target.oid 373 | links.group.excl link.target.oid 374 | 375 | proc excl(links: var GoldLinks; target: Gold) = 376 | ## remove a link to the given target 377 | # construct a mask that matches Incoming and Outgoing 378 | let mask = newLink(links.dad, {Incoming, Outgoing}, target) 379 | # and exclude it 380 | links.excl mask 381 | 382 | proc incl(links: var GoldLinks; link: Link; target: Gold) = 383 | ## link to a target with the given, uh, link 384 | var existing: set[LinkFlag] 385 | if target.oid in links.flags: 386 | existing = links.flags[target.oid] 387 | if Outgoing in link.flags and Outgoing notin existing: 388 | links.outs.add link 389 | if Incoming in link.flags and Incoming notin existing: 390 | links.ins.add link 391 | links.flags[target.oid] = existing + link.flags 392 | links.group.incl target 393 | 394 | proc rotateLinkFlags(flags: set[LinkFlag]): set[LinkFlag] = 395 | result = flags 396 | if Incoming in result and Outgoing in result: 397 | discard 398 | elif Incoming in result: 399 | result.incl Outgoing 400 | result.excl Incoming 401 | elif Outgoing in result: 402 | result.incl Incoming 403 | result.excl Outgoing 404 | 405 | proc createLink(links: var GoldLinks; flags: set[LinkFlag]; target: var Gold) = 406 | ## create a link by specifying the flags and the target 407 | var 408 | future: set[LinkFlag] 409 | existing: set[LinkFlag] 410 | if target.oid in links.flags: 411 | existing = links.flags[target.oid] 412 | future = flags + existing 413 | if future.len == existing.len: 414 | return 415 | var tags = flags 416 | case target.kind: 417 | of aFile: 418 | discard 419 | of aCompiler: 420 | tags.incl Unique 421 | of aCompilation: 422 | discard 423 | of anInvocation: 424 | discard 425 | else: 426 | raise newException(Defect, 427 | &"{links.dad.kind} doesn't link to {target.kind}") 428 | let link = newLink(links.dad, tags, target) 429 | if Unique in tags: 430 | if target.kind in links: 431 | links.excl links[target.kind] 432 | links.incl link, target 433 | tags = rotateLinkFlags(tags) 434 | createLink(target.links, tags, links.dad) 435 | 436 | proc `{}=`(links: var GoldLinks; flags: varargs[LinkFlag]; target: var Gold) = 437 | ## create a link by specifying the flags and the target 438 | var tags: set[LinkFlag] 439 | for flag in flags: 440 | tags.incl flag 441 | links.createLink(tags, target) 442 | 443 | proc compiler*(gold: Gold): Gold = 444 | result = gold.links[aCompiler] 445 | 446 | proc `compiler=`*(gold: var Gold; compiler: var Gold) = 447 | ## link to a compiler 448 | assert compiler.kind == aCompiler 449 | var flags: set[LinkFlag] 450 | case gold.kind: 451 | of aFile: 452 | flags = {Outgoing} 453 | of aCompilation: 454 | flags = {Incoming} 455 | else: 456 | raise newException(Defect, "inconceivable!") 457 | gold.links.createLink(flags, compiler) 458 | 459 | proc binary*(gold: Gold): Gold = 460 | assert gold != nil 461 | case gold.kind: 462 | of aCompiler, aCompilation, anInvocation: 463 | for file in gold.links{Binary}: 464 | return file 465 | raise newException(Defect, "unable to find binary for " & $gold.kind) 466 | else: 467 | raise newException(Defect, "inconceivable!") 468 | 469 | iterator sources*(gold: Gold): Gold = 470 | for file in gold.links{Source}: 471 | yield file 472 | 473 | proc source*(gold: Gold): Gold {.deprecated.} = 474 | for file in gold.sources: 475 | return file 476 | 477 | proc target*(gold: Gold): Gold = 478 | assert gold.kind == aCompilation 479 | for file in gold.links{Output,Binary}: 480 | return file 481 | 482 | proc compilation*(gold: Gold): Gold = 483 | for compilation in gold.links{Incoming}: 484 | if compilation.kind == aCompilation: 485 | return compilation 486 | 487 | proc invocation*(gold: Gold): Gold = 488 | assert gold.kind != anInvocation 489 | for invocation in gold.links{Incoming}: 490 | if invocation.kind == anInvocation: 491 | return invocation 492 | 493 | proc invocations*(gold: var Gold): RunningResult[Gold] = 494 | result = gold.benchmark.invocations 495 | 496 | proc compilations*(gold: var Gold): RunningResult[Gold] = 497 | result = gold.compilations 498 | 499 | proc `source=`*(gold: var Gold; source: var Gold) = 500 | ## link to a source file 501 | assert gold.kind == aCompilation 502 | assert source.kind == aFile 503 | source.links{Outgoing, Incoming, Input, Source} = gold 504 | 505 | proc `binary=`*(gold: var Gold; binary: var Gold) = 506 | ## link to a binary (executable) file 507 | assert gold.kind in [anInvocation, aCompiler] 508 | assert binary.kind == aFile 509 | binary.links{Outgoing, Input, Binary} = gold 510 | 511 | proc `invocation=`*(gold: var Gold; invocation: var Gold) = 512 | ## link a compilation to its invocation 513 | assert gold.kind == aCompilation 514 | assert invocation.kind == anInvocation 515 | invocation.links{Outgoing} = gold 516 | 517 | proc `target=`*(gold: var Gold; target: var Gold) = 518 | ## link a compilation to its target (binary) 519 | assert gold.kind == aCompilation 520 | assert target.kind == aFile 521 | target.links{Outgoing, Output, Binary} = gold 522 | 523 | proc `compilation=`*(gold: var Gold; compilation: CompilationInfo) = 524 | assert gold.kind == aCompilation 525 | gold.compilation = compilation 526 | 527 | proc commandLine*(invocation: Gold): string = 528 | ## compose the full commandLine for the given invocation 529 | result = invocation.binary.file.path 530 | if invocation.invokation.arguments != nil: 531 | if invocation.invokation.arguments[].len > 0: 532 | result &= " " & invocation.invokation.arguments[].join(" ") 533 | 534 | proc init*(gold: var Gold; binary: var Gold; args: ref seq[string]) = 535 | assert gold.kind == anInvocation 536 | assert binary.kind == aFile 537 | gold.invokation = InvocationInfo() 538 | gold.links{Incoming,Outgoing,Binary,Input} = binary 539 | gold.invokation.arguments = args 540 | 541 | proc okay*(gold: Gold): bool = 542 | ## measure the output code of a completed process 543 | case gold.kind: 544 | of aCompilation: 545 | result = gold.invocation.okay 546 | of anInvocation: 547 | result = gold.invokation.code == 0 548 | else: 549 | raise newException(Defect, "inconceivable!") 550 | 551 | proc newInvocation*(): Gold = 552 | result = newGold(anInvocation) 553 | result.invokation = InvocationInfo() 554 | 555 | proc newInvocation*(file: Gold; args: ref seq[string]): Gold = 556 | var binary = newFileDetailWithInfo(file.file.path) 557 | result = newInvocation() 558 | result.init(binary, args = args) 559 | 560 | proc fibonacci*(x: int): int = 561 | result = if x <= 2: 1 562 | else: fibonacci(x - 1) + fibonacci(x - 2) 563 | 564 | proc pack_type*[ByteStream](s: ByteStream; x: GoldKind) = 565 | let v = cast[uint8](ord(x)) 566 | s.pack(v) 567 | 568 | proc unpack_type*[ByteStream](s: ByteStream; x: var GoldKind) = 569 | var v: uint8 570 | s.unpack_type(v) 571 | x = cast[GoldKind](v) 572 | 573 | proc pack_type*[ByteStream](s: ByteStream; x: Timezone) {.deprecated.} = 574 | s.pack(x.name) 575 | 576 | proc unpack_type*[ByteStream](s: ByteStream; x: var Timezone) {.deprecated.} = 577 | s.unpack_type(x.name) 578 | case x.name: 579 | of "LOCAL": 580 | x = local() 581 | of "UTC": 582 | x = utc() 583 | else: 584 | raise newException(Defect, "dunno how to unpack timezone `" & x.name & "`") 585 | 586 | proc pack_type*[ByteStream](s: ByteStream; x: NanosecondRange) = 587 | s.pack(cast[int32](x)) 588 | 589 | proc unpack_type*[ByteStream](s: ByteStream; x: var NanosecondRange) = 590 | var y: int32 591 | s.unpack_type(y) 592 | x = y 593 | 594 | proc pack_type*[ByteStream](s: ByteStream; x: Time) = 595 | s.pack(x.toUnix) 596 | s.pack(x.nanosecond) 597 | 598 | proc unpack_type*[ByteStream](s: ByteStream; x: var Time) = 599 | var 600 | unix: int64 601 | nanos: NanosecondRange 602 | s.unpack_type(unix) 603 | s.unpack_type(nanos) 604 | x = initTime(unix, nanos) 605 | 606 | proc pack_type*[ByteStream](s: ByteStream; x: DateTime) = 607 | s.pack(x.toTime) 608 | 609 | proc unpack_type*[ByteStream](s: ByteStream; x: var DateTime) = 610 | var t: Time 611 | s.unpack_type(t) 612 | x = t.inZone(local()) 613 | 614 | proc pack_type*[ByteStream](s: ByteStream; x: Oid) = 615 | s.pack($x) 616 | 617 | proc unpack_type*[ByteStream](s: ByteStream; x: var Oid) = 618 | var oid: string 619 | s.unpack_type(oid) 620 | x = parseOid(oid) 621 | 622 | proc pack_type*[ByteStream](s: ByteStream; x: FileDetail) = 623 | s.pack(x.digest) 624 | s.pack(x.size) 625 | s.pack(x.path) 626 | s.pack(x.kind) 627 | s.pack(x.mtime) 628 | 629 | proc unpack_type*[ByteStream](s: ByteStream; x: var FileDetail) = 630 | s.unpack_type(x.digest) 631 | s.unpack_type(x.size) 632 | s.unpack_type(x.path) 633 | s.unpack_type(x.kind) 634 | s.unpack_type(x.mtime) 635 | 636 | proc pack_type*[ByteStream](s: ByteStream; x: Gold) = 637 | s.pack(x.kind) 638 | s.pack(x.oid) 639 | #s.pack(x.entry) 640 | case x.kind: 641 | of aFile: 642 | s.pack(x.file) 643 | else: 644 | raise newException(Defect, "inconceivable!") 645 | 646 | proc unpack_type*[ByteStream](s: ByteStream; gold: var Gold) = 647 | var 648 | oid: string 649 | kind: GoldKind 650 | s.unpack_type(kind) 651 | s.unpack_type(oid) 652 | when defined(StoreEntry): 653 | var entry: DateTime 654 | s.unpack_type(entry) 655 | gold = newGold(kind, oid = parseOid(oid), entry = entry) 656 | else: 657 | gold = newGold(kind, oid = parseOid(oid)) 658 | case kind: 659 | of aFile: 660 | new gold.file 661 | s.unpack_type(gold.file) 662 | else: 663 | raise newException(Defect, "inconceivable!") 664 | 665 | proc toJson*(entry: DateTime): JsonNode = 666 | result = newJString entry.format(ISO8601noTZ) 667 | 668 | proc toJson*(gold: Gold): JsonNode = 669 | result = %* { 670 | "oid": newJString $gold.oid, 671 | "description": newJString gold.description, 672 | } 673 | when defined(StoreEntry): 674 | result["entry"] = gold.entry.toJson 675 | 676 | proc jsonOutput*(golden: Golden): bool = 677 | let flags = golden.options.flags 678 | result = PipeOutput in flags or Interactive notin flags 679 | 680 | proc add*[T: InvocationInfo](running: RunningResult[T]; value: T) = 681 | ## for stats, pull out the invocation duration from invocation info 682 | let seconds = value.wall.toSeconds 683 | running.list.append value 684 | running.stat.push seconds 685 | 686 | proc add*[T: Gold](running: RunningResult[T]; value: T) = 687 | ## for stats, pull out the invocation duration from invocation info 688 | var v: StatValue 689 | case value.kind: 690 | of aCompilation: 691 | v = value.invocation.invokation.wall.toSeconds 692 | of anInvocation: 693 | v = value.invokation.wall.toSeconds 694 | else: 695 | raise newException(Defect, "inconceivable!") 696 | running.list.append value 697 | running.stat.push v 698 | 699 | proc reset*[T: InvocationInfo](running: RunningResult[T]) {.deprecated.} = 700 | running.stat.clear 701 | var stat: seq[StatValue] 702 | for invocation in running.list.items: 703 | stat.add invocation.runtime.stat.toSeconds 704 | running.stat.push stat 705 | 706 | proc add*[T: CompilationInfo](running: RunningResult[T]; value: T) {.deprecated.} = 707 | ## for stats, pull out the invocation duration from compilation info 708 | running.list.append value 709 | running.stat.push value.invocation.wall.toSeconds 710 | 711 | proc quiesceMemory*(message: string): int {.inline.} = 712 | GC_fullCollect() 713 | when defined(debug): 714 | stdmsg().writeLine GC_getStatistics() 715 | result = getOccupiedMem() 716 | 717 | template goldenDebug*() = 718 | when defined(debug): 719 | when defined(nimTypeNames): 720 | dumpNumberOfInstances() 721 | stdmsg().writeLine "total: " & $getTotalMem() 722 | stdmsg().writeLine " free: " & $getFreeMem() 723 | stdmsg().writeLine "owned: " & $getOccupiedMem() 724 | stdmsg().writeLine " max: " & $getMaxMem() 725 | 726 | proc render*(d: Duration): string {.raises: [].} = 727 | ## cast a duration to a nice string 728 | let 729 | n = d.inNanoseconds 730 | ss = (n div 1_000_000_000) mod 1_000 731 | ms = (n div 1_000_000) mod 1_000 732 | us = (n div 1_000) mod 1_000 733 | ns = (n div 1) mod 1_000 734 | try: 735 | return fmt"{ss:>3}s {ms:>3}ms {us:>3}μs {ns:>3}ns" 736 | except: 737 | return [$ss, $ms, $us, $ns].join(" ") 738 | 739 | proc `$`*(detail: FileDetail): string = 740 | result = detail.path 741 | 742 | proc toJson(invokation: InvocationInfo): JsonNode = 743 | result = newJObject() 744 | result["stdout"] = newJString invokation.stdout 745 | result["stderr"] = newJString invokation.stderr 746 | result["code"] = newJInt invokation.code 747 | 748 | proc output*(golden: Golden; content: string; style: set[terminal.Style] = {}; fg: ForegroundColor = fgDefault; bg: BackgroundColor = bgDefault) = 749 | let 750 | flags = golden.options.flags 751 | fh = stdmsg() 752 | if NeverOutput in golden.options.flags: 753 | return 754 | if ColorConsole in flags or PipeOutput notin flags: 755 | fh.setStyle style 756 | fh.setForegroundColor fg 757 | fh.setBackgroundColor bg 758 | fh.writeLine content 759 | 760 | proc output*(golden: Golden; content: JsonNode) = 761 | var ugly: string 762 | if NeverOutput in golden.options.flags: 763 | return 764 | ugly.toUgly(content) 765 | stdout.writeLine ugly 766 | 767 | proc output*(golden: Golden; invocation: Gold; desc: string = ""; 768 | arguments: seq[string] = @[]) = 769 | ## generally used to output a failed invocation 770 | let invokation = invocation.invokation 771 | if ColorConsole in golden.options.flags: 772 | if invokation.stdout.len > 0: 773 | golden.output invokation.stdout, fg = fgCyan 774 | if invokation.stderr.len > 0: 775 | golden.output invokation.stderr, fg = fgRed 776 | if invokation.code != 0: 777 | golden.output "exit code: " & $invokation.code 778 | if jsonOutput(golden): 779 | golden.output invokation.toJson 780 | if invokation.code != 0: 781 | new invokation.arguments 782 | for n in arguments: 783 | invokation.arguments[].add n 784 | golden.output "command-line:\n " & invocation.commandLine 785 | invokation.arguments = nil 786 | 787 | proc toWallDuration*(gold: Gold): Duration = 788 | case gold.kind: 789 | of anInvocation: 790 | result = gold.invokation.wall 791 | of aCompilation: 792 | result = gold.compilation.invocation.toWallDuration 793 | else: 794 | raise newException(Defect, "inconceivable!") 795 | 796 | proc toStatValue*(gold: Gold): StatValue = 797 | case gold.kind: 798 | of anInvocation: 799 | result = gold.toWallDuration.toSeconds 800 | of aCompilation: 801 | result = gold.toWallDuration.toSeconds 802 | else: 803 | raise newException(Defect, "inconceivable!") 804 | 805 | proc output*(golden: Golden; running: RunningResult; desc: string = "") = 806 | golden.output running.renderTable(desc) 807 | --------------------------------------------------------------------------------