├── src ├── website.nim.cfg ├── builder.nim.cfg ├── index.html ├── irclog.nim ├── types.nim ├── irclogrender.nim ├── htmlhelp.nim ├── db.nim ├── github.nim ├── ircbot.nim ├── builder.nim └── website.nim ├── public ├── images │ ├── error.png │ ├── icons.png │ ├── tick.png │ ├── download.png │ └── progress.gif └── css │ ├── log.css │ ├── boilerplate.css │ └── style.css ├── todo.markdown ├── readme.markdown ├── tests └── dummyhub.nim └── structure.markdown /src/website.nim.cfg: -------------------------------------------------------------------------------- 1 | -d:ssl -------------------------------------------------------------------------------- /public/images/error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dom96/nimbuild/HEAD/public/images/error.png -------------------------------------------------------------------------------- /public/images/icons.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dom96/nimbuild/HEAD/public/images/icons.png -------------------------------------------------------------------------------- /public/images/tick.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dom96/nimbuild/HEAD/public/images/tick.png -------------------------------------------------------------------------------- /public/images/download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dom96/nimbuild/HEAD/public/images/download.png -------------------------------------------------------------------------------- /public/images/progress.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dom96/nimbuild/HEAD/public/images/progress.gif -------------------------------------------------------------------------------- /src/builder.nim.cfg: -------------------------------------------------------------------------------- 1 | threads:on 2 | -d:debug 3 | @if windows: 4 | tlsEmulation:on 5 | @end 6 | @if bsd: 7 | tlsEmulation:on 8 | -d:useFork 9 | @end 10 | -------------------------------------------------------------------------------- /public/css/log.css: -------------------------------------------------------------------------------- 1 | 2 | body { 3 | background-color: #2d2d2d; 4 | color: #ffffff; 5 | font-size: 12pt; 6 | } 7 | 8 | table { 9 | margin-top: 15pt; 10 | 11 | } 12 | 13 | table td, table th { 14 | padding: 0.3em 0.4em; 15 | } 16 | 17 | td.nick { 18 | border-left: 1px solid #CC880A; 19 | border-right: 1px solid #CC880A; 20 | text-align: right; 21 | } 22 | 23 | tr.join td.msg, tr.join td.nick { 24 | color: #33DB09; 25 | } 26 | 27 | tr.quit td.msg, tr.quit td.nick { 28 | color: #DB2109; 29 | } 30 | 31 | tr.nick td.msg, tr.nick td.nick { 32 | color: #098BDB; 33 | } 34 | 35 | tr.part td.msg, tr.part td.nick { 36 | color: #DB9909; 37 | } 38 | 39 | tr.action td.msg, tr.action td.nick { 40 | color: #C609DB; 41 | } 42 | 43 | a:link { 44 | color: #F2A20C; 45 | 46 | } 47 | 48 | a:hover { 49 | color: #CC880A; 50 | } 51 | 52 | #controls { 53 | font-size: 16pt; 54 | text-align: center; 55 | } 56 | 57 | hr { 58 | background-color: #CC880A; 59 | border:0; 60 | height: 2px; 61 | } -------------------------------------------------------------------------------- /todo.markdown: -------------------------------------------------------------------------------- 1 | * Move from redis to mongodb or sqlite(?) 2 | * Convert redis data to suit the current website layout. 3 | * Hub should only have the FTP info, builders should query it for this data. 4 | * Download table is broken. 5 | * Keep only the latest version of Nimrod for download. 6 | * grep "TODO" 7 | * Fix nimbot 8 | * Integration with Github status API 9 | * When a pull request is made nimbuild should be intelligent on what it compiles 10 | I.e. If the file edited is in the compiler/ directory then nimrod should be bootstrapped 11 | if the file edited is a test, the test should be ran 12 | if the file edited is a module in the stdlib it should just be compiled 13 | * This should then be reported to github... 14 | 15 | * Nimbuild should be smart, as described above, this should be replicated to 16 | all pushes. Single file change: build only that file, compiler changes do whole bootstrap + test suite. 17 | etc. 18 | * If the newest build only built one file, the website should show the last known test results (perhaps with a warning). 19 | * DB needs to know about this. 20 | 21 | * Inspect diff. 22 | * If all lines that were changed start with (\s+)## then no need to bootstrap, 23 | just ened to rebuild the docs. 24 | 25 | * Change the left side color of each platform box to show the current builder status. 26 | * Pulsates blue when building 27 | * Red when disconnected 28 | * Blue when connected? 29 | 30 | * .deb gen -------------------------------------------------------------------------------- /readme.markdown: -------------------------------------------------------------------------------- 1 | # Nimbuild 2 | 3 | This is the Nim build farm. It is separated into multiple components; main one being 4 | the website (website.nim); which acts as a ``hub`` for all the other components 5 | - they all connect to it. It also acts as the front end and is available at 6 | http://build.nim-lang.org/. 7 | 8 | The other components: 9 | 10 | ### Github.nim 11 | This component waits for connections from github, it acts as a POST receive hook. 12 | It waits for a POST request from github containing a payload informing it 13 | of the file commited to the Nim repo, it then sends this information on 14 | to the website. 15 | 16 | ### Builder.nim 17 | This component does the actual building, multiple instances of it run on different 18 | platforms. It pulls the latest version of the compiler from github, bootstraps 19 | it, zips the binary then uploads the zip to nimbuild. It then finally runs 20 | the test suite. It also does some other tasks which are optional, like generating 21 | c sources and the documentation. 22 | 23 | ### ircbot.nim 24 | This component is an IRC bot which idles in the #nim channel on freenode, it 25 | has some features already. It's main purpose is to announce a commit in the 26 | channel, but it also has a !seen command. More features are planned for later. 27 | 28 | ## Contributing 29 | Pull requests are always welcome. If you are not much of a programmer you can 30 | always donate a machine, we are always looking for new machines, especially 31 | Windows ones to run nimbuild on, if you have one please contact me on Github 32 | or on freenode (i'm dom96). 33 | -------------------------------------------------------------------------------- /tests/dummyhub.nim: -------------------------------------------------------------------------------- 1 | import asyncio, jester, sockets, json 2 | 3 | var currentClient: PAsyncSocket 4 | 5 | proc clientRead(s: PAsyncSocket) = 6 | var line = "" 7 | if s.readLine(line): 8 | if line == "": 9 | echo("Client disconnected") 10 | currentClient.close() 11 | currentClient = nil 12 | else: 13 | echo("Recv: ", line) 14 | var json = parseJson(line) 15 | if json.hasKey("ping"): 16 | json["pong"] = json["ping"] 17 | json.delete("ping") 18 | s.send($json & "\c\L") 19 | elif json.hasKey("name"): 20 | s.send($(%{ "reply": %"OK" }) & "\c\L") 21 | 22 | when isMainModule: 23 | var disp = newDispatcher() 24 | var hubSock = AsyncSocket() 25 | hubSock.bindAddr(TPort(5123)) 26 | hubSock.listen() 27 | 28 | hubSock.handleAccept = 29 | proc (s: PAsyncSocket) = 30 | if currentClient != nil: currentClient.close() 31 | new(currentClient) 32 | s.accept(currentClient) 33 | currentClient.handleRead = clientRead 34 | disp.register(currentClient) 35 | disp.register(hubSock) 36 | 37 | get "/": 38 | if currentClient == nil: resp "No client." 39 | else: resp "OK" 40 | 41 | get "/boot": 42 | if currentClient == nil: halt "No client! :(" 43 | var reply = newJObject() 44 | reply["payload"] = newJObject() 45 | reply["payload"]["after"] = newJString("HEAD") 46 | reply["payload"]["ref"] = newJString("refs/heads/master") 47 | reply["payload"]["commits"] = newJArray() 48 | reply["rebuild"] = newJBool(true) 49 | currentClient.send($reply & "\c\L") 50 | resp "We are now bootstrapping." 51 | 52 | get "/stop": 53 | if currentClient == nil: halt "No client! :(" 54 | var reply = %{"do": %"stop"} 55 | currentClient.send($reply & "\c\L") 56 | resp "Stopped." 57 | 58 | disp.register() 59 | 60 | while true: 61 | doAssert disp.poll() 62 | -------------------------------------------------------------------------------- /src/index.html: -------------------------------------------------------------------------------- 1 | #! stdtmpl | standard 2 | #proc genHtml(state: PState): string = 3 | # result = "" 4 | 5 | 6 | 7 | NimBuild 8 | 9 | 10 | 11 | 12 |
13 | #var platforms: seq[string] = @[] # Every platform, from every commit. 14 | #var entries = getCommits(state.database, platforms) 15 | #platforms.sort(cmpPlatforms, Descending) 16 | 20 |
21 | 22 | ${genDownloadTable(state.req, entries, platforms)} 23 | #if platforms.len() > 0: 24 |
25 | ${genBuildResults(state, platforms, entries)} 26 |
27 | #else: 28 |

No commits found

29 | #end if 30 | 31 |

Platforms

32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | #for m in items(state.modules): 40 | #if m.name == "builder": 41 | #var platf = state.platforms[m.platform] 42 | 43 | 44 | 45 | 46 | 47 | 48 | #end if 49 | #end for 50 | 51 |
PlatformLagStatusDescription
${m.platform}${formatFloat(m.ping)}${platf}${platf.desc}
52 |
53 |
54 |
55 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /src/irclog.nim: -------------------------------------------------------------------------------- 1 | import htmlgen, times, irc, streams, strutils, os, json, parseutils, marshal 2 | from xmltree import escape 3 | 4 | type 5 | TLogger* = object of TObject # Items get erased when new day starts. 6 | startTime*: TTimeInfo 7 | logFilepath*: string 8 | logFile*: TFile 9 | PLogger* = ref TLogger 10 | 11 | const 12 | webFP = {fpUserRead, fpUserWrite, fpUserExec, 13 | fpGroupRead, fpGroupExec, fpOthersRead, fpOthersExec} 14 | 15 | proc loadLogger*(f: string): PLogger = 16 | new(result) 17 | let logs = readFile(f) 18 | let lines = logs.splitLines() 19 | # Line 1: Start time 20 | result.startTime = fromSeconds(to[float](lines[0])).getGMTime() 21 | 22 | doAssert open(result.logFile, f, fmAppend) 23 | result.logFilepath = f.splitFile.dir 24 | 25 | proc writeFlush(file: TFile, s: string) = 26 | file.write(s) 27 | file.flushFile() 28 | 29 | proc newLogger*(logFilepath: string): PLogger = 30 | let startTime = getTime().getGMTime() 31 | let log = logFilepath / startTime.format("dd'-'MM'-'yyyy'.logs'") 32 | if existsFile(log): 33 | result = loadLogger(log) 34 | else: 35 | new(result) 36 | result.startTime = startTime 37 | result.logFilepath = logFilepath 38 | doAssert open(result.logFile, log, fmAppend) 39 | # Write start time 40 | result.logFile.writeFlush($$epochTime() & "\n") 41 | 42 | proc `$`(s: seq[string]): string = 43 | var escaped = system.map(s) do (x: string) -> string: 44 | strutils.escape(x) 45 | result = "[" & join(escaped, ",") & "]" 46 | 47 | proc writeLog(logger: PLogger, msg: TIRCEvent) = 48 | logger.logFile.writeFlush($$(time: getTime(), msg: msg) & "\n") 49 | 50 | proc log*(logger: PLogger, msg: TIRCEvent) = 51 | if msg.origin != "#nimrod" and msg.cmd notin {MQuit, MNick}: return 52 | if getTime().getGMTime().yearday != logger.startTime.yearday: 53 | # It's time to cycle to next day. 54 | # Reset logger. 55 | logger.logFile.close() 56 | logger.startTime = getTime().getGMTime() 57 | let log = logger.logFilepath / logger.startTime.format("dd'-'MM'-'yyyy'.logs'") 58 | doAssert open(logger.logFile, log, fmAppend) 59 | # Write start time 60 | logger.logFile.writeFlush($epochTime() & "\n") 61 | 62 | case msg.cmd 63 | of MPrivMsg, MJoin, MPart, MNick, MQuit: # TODO: MTopic? MKick? 64 | #logger.items.add((getTime(), msg)) 65 | #logger.save(logger.logFilepath / logger.startTime.format("dd'-'MM'-'yyyy'.json'")) 66 | writeLog(logger, msg) 67 | else: nil 68 | 69 | proc log*(logger: PLogger, nick, msg, chan: string) = 70 | var m: TIRCEvent 71 | m.typ = EvMsg 72 | m.cmd = MPrivMsg 73 | m.params = @[chan, msg] 74 | m.origin = chan 75 | m.nick = nick 76 | logger.log(m) 77 | 78 | when isMainModule: 79 | var logger = newLogger("testing/logstest") 80 | logger.log("dom96", "Hello!", "#nimrod") 81 | logger.log("dom96", "Hello\r, testingí, \"\"", "#nimrod") 82 | #logger = loadLogger("testing/logstest/26-05-2013.logs") 83 | echo repr(logger) -------------------------------------------------------------------------------- /src/types.nim: -------------------------------------------------------------------------------- 1 | import os, tables, hashes 2 | # TODO: Rename this module to ``utils`` 3 | type 4 | TBuilderJob* = enum 5 | jBuild, jTest, jDocGen, jCSrcGen, jInnoSetup 6 | 7 | TProgress* = enum 8 | jUnknown, jFail, jInProgress, jSuccess 9 | 10 | TStatus* = object 11 | isInProgress*: bool 12 | desc*: string 13 | hash*: string 14 | branch*: string 15 | jobs*: TTable[TBuilderJob, TProgress] 16 | cmd*: string 17 | args*: string 18 | FTPSpeed*: float 19 | 20 | TBuilderEventType* = enum 21 | bProcessStart, bProcessLine, bProcessExit, bFTPUploadSpeed, bEnd, bStart 22 | 23 | proc hash*[T: enum](x: T): THash = ord(x) 24 | 25 | proc initStatus*(): TStatus = 26 | result.isInProgress = false 27 | result.jobs = initTable[TBuilderJob, TProgress]() 28 | result.desc = "" 29 | result.hash = "" 30 | result.cmd = "" 31 | result.args = "" 32 | result.FTPSpeed = -1.0 33 | 34 | proc jobInProgress*(s: TStatus): TBuilderJob = 35 | assert s.isInProgress 36 | for j, p in s.jobs: 37 | if p == jInProgress: 38 | return j 39 | raise newException(EInvalidValue, "No job could be found that is in progress.") 40 | 41 | proc findLatestJob*(s: TStatus, job: var TBuilderJob): bool = 42 | for i in TBuilderJob: 43 | if s.jobs[i] == jFail or s.jobs[i] == jSuccess: 44 | job = i 45 | return true 46 | 47 | proc `$`*(s: TStatus): string = 48 | if s.isInProgress: 49 | let job = jobInProgress(s) 50 | case job 51 | of jBuild: 52 | result = "Bootstrapping" 53 | of jTest: 54 | result = "Testing" 55 | of jDocGen: 56 | result = "Generating docs" 57 | of jCSrcGen: 58 | result = "Generating C Sources" 59 | of jInnoSetup: 60 | result = "Generating Inno setup file" 61 | else: 62 | var job: TBuilderJob 63 | result = "Unknown" 64 | if findLatestJob(s, job): 65 | case job 66 | of jBuild: 67 | if s.jobs[job] == jSuccess: 68 | result = "Bootstrapped successfully" 69 | elif s.jobs[job] == jFail: 70 | result = "Bootstrapping failed" 71 | of jTest: 72 | if s.jobs[job] == jSuccess: 73 | result = "Tested successfully" 74 | elif s.jobs[job] == jFail: 75 | result = "Testing failed" 76 | of jDocGen: 77 | if s.jobs[job] == jSuccess: 78 | result = "Doc generation succeeded" 79 | elif s.jobs[job] == jFail: 80 | result = "Doc generation failed" 81 | of jCSrcGen: 82 | if s.jobs[job] == jSuccess: 83 | result = "C source generation succeeded" 84 | elif s.jobs[job] == jFail: 85 | result = "C source generation failed" 86 | of jInnoSetup: 87 | if s.jobs[job] == jSuccess: 88 | result = "Inno setup generation succeeded" 89 | elif s.jobs[job] == jFail: 90 | result = "Inno setup generation failed" 91 | 92 | proc makeCommitPath*(platform, hash: string): string = 93 | return platform / hash.substr(0, 11) # 11 Chars. 94 | 95 | proc makeZipPath*(platform, hash: string): string = 96 | return platform / "nimrod_" & hash.substr(0, 11) 97 | 98 | proc makeInnoSetupPath*(hash: string): string = 99 | return ("nimrod_" & hash.substr(0, 11)) & ".exe" 100 | -------------------------------------------------------------------------------- /src/irclogrender.nim: -------------------------------------------------------------------------------- 1 | import irc, htmlgen, times, strutils, marshal, os, xmltree 2 | from jester import TRequest, makeUri 3 | import irclog 4 | 5 | type 6 | TLogRenderer = object of TLogger 7 | items*: seq[tuple[time: TTime, msg: TIRCEvent]] ## Only used for HTML gen 8 | PLogRenderer* = ref TLogRenderer 9 | 10 | proc loadRenderer*(f: string): PLogRenderer = 11 | new(result) 12 | result.items = @[] 13 | let logs = readFile(f) 14 | let lines = logs.splitLines() 15 | var i = 1 16 | # Line 1: Start time 17 | result.startTime = fromSeconds(to[float](lines[0])).getGMTime() 18 | 19 | result.logFilepath = f.splitFile.dir 20 | while i < lines.len: 21 | if lines[i] != "": 22 | result.items.add(to[tuple[time: TTime, msg: TIRCEvent]](lines[i])) 23 | inc i 24 | 25 | proc renderItems(logger: PLogRenderer): string = 26 | result = "" 27 | for i in logger.items: 28 | var c = "" 29 | case i.msg.cmd 30 | of MJoin: 31 | c = "join" 32 | of MPart: 33 | c = "part" 34 | of MNick: 35 | c = "nick" 36 | of MQuit: 37 | c = "quit" 38 | else: 39 | nil 40 | var message = i.msg.params[i.msg.params.len-1] 41 | if message.startswith("\x01ACTION "): 42 | c = "action" 43 | message = message[8 .. -2] 44 | 45 | if c == "": 46 | result.add(tr(td(i.time.getGMTime().format("HH':'mm':'ss")), 47 | td(class="nick", xmltree.escape(i.msg.nick)), 48 | td(class="msg", xmltree.escape(message)))) 49 | else: 50 | case c 51 | of "join": 52 | message = i.msg.nick & " joined " & i.msg.origin 53 | of "part": 54 | message = i.msg.nick & " left " & i.msg.origin & " (" & message & ")" 55 | of "nick": 56 | message = i.msg.nick & " is now known as " & message 57 | of "quit": 58 | message = i.msg.nick & " quit (" & message & ")" 59 | of "action": 60 | message = i.msg.nick & " " & message 61 | else: assert(false) 62 | result.add(tr(class=c, 63 | td(i.time.getGMTime().format("HH':'mm':'ss")), 64 | td(class="nick", "*"), 65 | td(class="msg", xmltree.escape(message)))) 66 | 67 | proc renderHtml*(logger: PLogRenderer, req: jester.TRequest): string = 68 | let today = getTime().getGMTime() 69 | let isToday = logger.startTime.monthday == today.monthday and 70 | logger.startTime.month == today.month and 71 | logger.startTime.year == today.year 72 | let previousDay = logger.startTime - (initInterval(days=1)) 73 | let prevUrl = req.makeUri("irclogs/" & 74 | previousDay.format("dd'-'MM'-'yyyy'.html'"), 75 | absolute = false) 76 | let nextDay = logger.startTime + (initInterval(days=1)) 77 | let nextUrl = 78 | if isToday: "" 79 | else: req.makeUri("irclogs/" & 80 | nextDay.format("dd'-'MM'-'yyyy'.html'"), absolute = false) 81 | result = 82 | html( 83 | head(title("#nimrod logs for " & logger.startTime.format("dd'-'MM'-'yyyy")), 84 | meta(content="text/html; charset=UTF-8", `http-equiv` = "Content-Type"), 85 | link(rel="stylesheet", href=req.makeUri("css/boilerplate.css", absolute = false)), 86 | link(rel="stylesheet", href=req.makeUri("css/log.css", absolute = false)) 87 | ), 88 | body( 89 | htmlgen.`div`(id="controls", 90 | a(href=prevUrl, "<<"), 91 | span(logger.startTime.format("dd'-'MM'-'yyyy")), 92 | (if nextUrl == "": span(">>") else: a(href=nextUrl, ">>")) 93 | ), 94 | hr(), 95 | table( 96 | renderItems(logger) 97 | ) 98 | ) 99 | ) -------------------------------------------------------------------------------- /src/htmlhelp.nim: -------------------------------------------------------------------------------- 1 | import htmlgen, strtabs, strutils 2 | type 3 | THtmlTable* = object 4 | rows: seq[TRow] 5 | 6 | TRow* = seq[PColumn] 7 | PColumn* = ref TColumn 8 | TColumn* = tuple[header: bool, attrs: PStringTable, text: string] 9 | 10 | proc initTable*(): THtmlTable = 11 | result.rows = @[] 12 | 13 | proc addRow*(table: var THtmlTable, count = 1) = 14 | ## Adds `count` many rows to `table`. 15 | for i in 0..count-1: 16 | table.rows.add(@[]) 17 | 18 | proc addCol*(row: var TRow, text: string, isHeader = false, 19 | attrs: seq[tuple[name, content: string]] = @[]) = 20 | ## Adds column with the name of `text` to `row`. 21 | var c: PColumn 22 | new(c) 23 | c.header = isHeader 24 | c.attrs = newStringTable(modeCaseInsensitive) 25 | for key, val in items(attrs): 26 | c.attrs[key] = val 27 | c.text = text 28 | row.add(c) 29 | 30 | proc insertCol*(row: var TRow, i: int, text: string, isHeader = false, 31 | attrs: seq[tuple[name, content: string]] = @[]) = 32 | ## Inserts column with the name of `text` to `row` at index `i`. 33 | var c: PColumn 34 | new(c) 35 | c.header = isHeader 36 | c.attrs = newStringTable(modeCaseInsensitive) 37 | for key, val in items(attrs): 38 | c.attrs[key] = val 39 | c.text = text 40 | 41 | row.insert(c, i) 42 | 43 | proc `[]`*(table: var THtmlTable, i: int): var TRow = 44 | ## Retrieves row at `i` 45 | return table.rows[i] 46 | 47 | proc findCols*(row: var TRow, text: string): seq[PColumn] = 48 | ## Finds and returns columns with the name of `text`. 49 | result = @[] 50 | for c in row: 51 | if c.text == text: 52 | result.add(c) 53 | 54 | proc contains*(row: var TRow, text: string): bool = 55 | ## Returns whether `row` contains column by the name of `text`. 56 | result = false 57 | for c in row: 58 | if c.text == text: 59 | return true 60 | 61 | iterator items*(table: THtmlTable): TRow = 62 | var i = 0 63 | while i < table.rows.len: 64 | yield table.rows[i] 65 | i.inc() 66 | 67 | iterator items*(row: TRow): PColumn = 68 | var i = 0 69 | while i < row.len: 70 | yield row[i] 71 | i.inc() 72 | 73 | proc len*(table: var THtmlTable): int = 74 | ## Returns the number of rows. 75 | return table.rows.len() 76 | 77 | proc len*(row: TRow): int = 78 | ## Returns the number of columns in a row. 79 | return system.len(row) 80 | # Solely because using only `len` would cause a recursive loop. 81 | 82 | proc toPretty(table: THtmlTable): string = 83 | # Returns an ASCII representation of the table. 84 | # TODO: Make this nicer, or just get rid of it. 85 | result = "" 86 | for cols in table.rows: 87 | result.add("| ") 88 | for i in cols: 89 | result.add(i.text & " | ") 90 | result.add("\n----------------------------\n") 91 | 92 | proc toHtml*(table: THtmlTable, attrs=""): string = 93 | result = "" 94 | var htmlRows: string = "" 95 | for row in table.rows: 96 | var htmlCols = "" 97 | for col in row: 98 | var htmlAttrs = "" 99 | for name, text in col.attrs: 100 | htmlAttrs.add(" " & name & "=\"" & text & "\"") 101 | 102 | if col.header: 103 | htmlCols.add("\n$2\n" % [htmlAttrs, col.text]) 104 | else: 105 | htmlCols.add("\n$2\n" % [htmlAttrs, col.text]) 106 | htmlRows.add(tr(htmlCols) & "\n") 107 | 108 | result = "\n" % [attrs] & htmlRows & "
" 109 | 110 | when isMainModule: 111 | var tab = initTable() 112 | tab.addRow() 113 | tab.addRow() 114 | tab[0].addCol("Col 1", true, @[("class", "something"), ("blah", "something")]) 115 | tab[0].addCol("Col 2", true) 116 | tab[2].addCol("Hello") 117 | tab[2].addCol("I SHOULD BE IN SCHOOL") 118 | tab[2].addCol("With R.") 119 | tab[2].addCol("And be k....") 120 | echo tab.toPretty 121 | 122 | echo(" ") 123 | for r in tab: 124 | for c in r: 125 | echo(c.text) 126 | echo("----") 127 | 128 | echo tab.toHtml() 129 | -------------------------------------------------------------------------------- /structure.markdown: -------------------------------------------------------------------------------- 1 | # Structure 2 | 3 | Github 4 | \ 5 | \ 6 | Hub --------- Builder A, B, C 7 | \ 8 | \ 9 | NimBot 10 | 11 | ## The Hub a.k.a "the website" 12 | 13 | This serves the website, it also acts as a hub, all the "modules" connect to it. 14 | 15 | ## Builder 16 | 17 | There are many of these connected at once to the Hub. Each runs on a different 18 | platform. 19 | 20 | Its job is to wait for a new commit notification from the hub. When this event 21 | occurs, it will update the local Nimrod git repository and begin the build 22 | process. 23 | 24 | The build process consists of the following: 25 | 26 | * Downloading the current C sources and building from them if they changed. 27 | * Bootstrapping the Nimrod compiler in debug and release mode. 28 | * Running the test suite. 29 | * Building the C sources (one builder only) 30 | * Building the documentation (one builder only) 31 | 32 | While doing so the builder keeps the hub updated with the progress of the build. 33 | The build is separated into jobs which include ``JBuild``, ``JTest``, 34 | ``JCSrcGen``, ``JDocGen`` and ``JInnoGen``. The jobs are not run in parallel, 35 | and if one fails the rest fail. 36 | 37 | ## Github 38 | 39 | This module waits for a request from Github which notifies it that there is a 40 | new commit in Nimrod's repo. It passes this information on to the hub. 41 | 42 | The hub uses this information to do multiple things including: 43 | 44 | * Sending the information to NimBot so that it announces the commit in the IRC channel. 45 | * Notifying the connected builders that a new commit is ready to be built. 46 | 47 | ## NimBot 48 | 49 | This is the IRC bot that resides in #nimrod on Freenode. It announces new commits 50 | to the Nimrod repo (and other nimrod-related repos) in that channel. It also 51 | announces build info in the #nimbuild channel. 52 | 53 | Other features include: 54 | 55 | * !seen feature 56 | 57 | ## Communication 58 | 59 | The hub communicates with the modules that are connected to it using JSON and 60 | vice versa; the modules do the same. 61 | 62 | ### Hub 63 | 64 | The hub currently supports the following messages: 65 | 66 | #### ``{ "job": types.TBuilderJob.ord }`` 67 | 68 | This marks the start of a new build job. 69 | 70 | **Sent by:** builder. 71 | 72 | #### ``{ "result": system.TResult.ord }`` 73 | 74 | Finishes the current builder's job with ``result`` (either success or failure). 75 | 76 | Other params: 77 | 78 | * ``detail`` - When ``result`` is ``Failure`` this contains the reason as to the 79 | failure. 80 | * ``total``/``passed``/``skipped``/``failed`` - When the current job is ``JTest`` 81 | and ``result`` is ``Success`` these fields contain the total tests, and also 82 | the amount of passed, skipped and failed tests. 83 | 84 | **Sent by:** builder. 85 | 86 | #### ``{ "eventType": TBuilderEventType.ord }`` 87 | 88 | Updates the hub on the status of a build in progress. 89 | 90 | **Sent by:** builder. 91 | 92 | #### ``{ "payload": { ... } }`` 93 | 94 | Tells the hub about a new commit or multiple commits made to a repo. 95 | Not necessarily the Nimrod repo. 96 | 97 | **Sent by:** github 98 | 99 | #### ``{ "latestCommit": true }`` 100 | 101 | Requests info about the latest commit from the hub. 102 | 103 | **Sent by:** builder. 104 | 105 | #### ``{ "ping": TimeSinceUnixEpoch }`` 106 | 107 | A module is verifying that it's still connected. 108 | 109 | **Sent by:** All modules. 110 | 111 | #### ``{ "pong": TimeSinceUnixEpoch }`` 112 | 113 | A module is replying to a "ping" message from the hub. 114 | 115 | **Sent by:** All modules. 116 | 117 | #### ``{ "do": "request" }`` 118 | 119 | A module is asking for info. 120 | 121 | ##### ``redisinfo`` 122 | 123 | Sends redis db info to the module requesting it. 124 | 125 | **Sent by:** NimBot. 126 | 127 | # Database 128 | 129 | The current database that is being used is redis. 130 | 131 | ## Structure 132 | 133 | Here is a brief description of the current database which is to be deprecated. 134 | 135 | ### A list 136 | 137 | This will be called ``commits``. Each commit hash will be LPUSH-ed onto this list for easy iteration from the latest to the oldest commits with LRANGE. 138 | 139 | ### Keys 140 | 141 | Information about each commit will be saved in a hash by the name of ``commit_hash``. 142 | The fields will be: 143 | * commitMsg 144 | * date 145 | * username 146 | * branch 147 | 148 | Specific information about a build can be retrieved by accessing ``platform:commit_hash``, where ``platform`` can be for example "linux-x86". 149 | The fields that these will contain will be: 150 | * buildResult -> db.TBuildResult ( bUnknown, bFail, bSuccess ) 151 | * testResult -> db.TTestResult ( tUnknown, tFail, tSuccess ) 152 | * total -> total tests 153 | * passed -> passed tests 154 | * skipped -> skipped tests 155 | * failed -> failed tests 156 | * csources -> whether the csources have been built for this platform/commit combination ("t" or "f" (?)) 157 | * timeBuild -> Time taken to build TODO 158 | * timeTest -> Time taken to test 159 | 160 | -------------------------------------------------------------------------------- /public/css/boilerplate.css: -------------------------------------------------------------------------------- 1 | 2 | /* ==== Scroll down to find where to put your styles :) ==== */ 3 | 4 | /* HTML5 ✰ Boilerplate */ 5 | 6 | html, body, div, span, object, iframe, 7 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 8 | abbr, address, cite, code, del, dfn, em, img, ins, kbd, q, samp, 9 | small, strong, sub, sup, var, b, i, dl, dt, dd, ol, ul, li, 10 | fieldset, form, label, legend, 11 | table, caption, tbody, tfoot, thead, tr, th, td, 12 | article, aside, canvas, details, figcaption, figure, 13 | footer, header, hgroup, menu, nav, section, summary, 14 | time, mark, audio, video { 15 | margin: 0; 16 | padding: 0; 17 | border: 0; 18 | font-size: 100%; 19 | font: inherit; 20 | vertical-align: baseline; 21 | } 22 | 23 | article, aside, details, figcaption, figure, 24 | footer, header, hgroup, menu, nav, section { 25 | display: block; 26 | } 27 | 28 | blockquote, q { quotes: none; } 29 | blockquote:before, blockquote:after, 30 | q:before, q:after { content: ''; content: none; } 31 | ins { background-color: #ff9; color: #000; text-decoration: none; } 32 | mark { background-color: #ff9; color: #000; font-style: italic; font-weight: bold; } 33 | del { text-decoration: line-through; } 34 | abbr[title], dfn[title] { border-bottom: 1px dotted; cursor: help; } 35 | table { border-collapse: collapse; border-spacing: 0; } 36 | hr { display: block; height: 1px; border: 0; border-top: 1px solid #ccc; margin: 1em 0; padding: 0; } 37 | input, select { vertical-align: middle; } 38 | 39 | body { font:13px/1.231 sans-serif; *font-size:small; } 40 | select, input, textarea, button { font:99% sans-serif; } 41 | pre, code, kbd, samp { font-family: monospace, sans-serif; } 42 | 43 | html { overflow-y: scroll; } 44 | a:hover, a:active { outline: none; } 45 | ul, ol { margin-left: 2em; } 46 | ol { list-style-type: decimal; } 47 | nav ul, nav li { margin: 0; list-style:none; list-style-image: none; } 48 | small { font-size: 85%; } 49 | strong, th { font-weight: bold; } 50 | td { vertical-align: top; } 51 | 52 | sub, sup { font-size: 75%; line-height: 0; position: relative; } 53 | sup { top: -0.5em; } 54 | sub { bottom: -0.25em; } 55 | 56 | pre { white-space: pre; white-space: pre-wrap; word-wrap: break-word; padding: 15px; } 57 | textarea { overflow: auto; } 58 | .ie6 legend, .ie7 legend { margin-left: -7px; } 59 | input[type="radio"] { vertical-align: text-bottom; } 60 | input[type="checkbox"] { vertical-align: bottom; } 61 | .ie7 input[type="checkbox"] { vertical-align: baseline; } 62 | .ie6 input { vertical-align: text-bottom; } 63 | label, input[type="button"], input[type="submit"], input[type="image"], button { cursor: pointer; } 64 | button, input, select, textarea { margin: 0; } 65 | input:valid, textarea:valid { } 66 | input:invalid, textarea:invalid { border-radius: 1px; -moz-box-shadow: 0px 0px 5px red; -webkit-box-shadow: 0px 0px 5px red; box-shadow: 0px 0px 5px red; } 67 | .no-boxshadow input:invalid, .no-boxshadow textarea:invalid { background-color: #f0dddd; } 68 | 69 | a:link { -webkit-tap-highlight-color: #FF5E99; } 70 | 71 | button { width: auto; overflow: visible; } 72 | .ie7 img { -ms-interpolation-mode: bicubic; } 73 | 74 | body, select, input, textarea { color: #444; } 75 | h1, h2, h3, h4, h5, h6 { font-weight: bold; } 76 | a, a:active, a:visited { color: #607890; } 77 | a:hover { color: #036; } 78 | 79 | /* 80 | // ========================================== \\ 81 | || || 82 | || Your styles ! || 83 | || || 84 | \\ ========================================== // 85 | */ 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | .ir { display: block; text-indent: -999em; overflow: hidden; background-repeat: no-repeat; text-align: left; direction: ltr; } 102 | .hidden { display: none; visibility: hidden; } 103 | .visuallyhidden { border: 0; clip: rect(0 0 0 0); height: 1px; margin: -1px; overflow: hidden; padding: 0; position: absolute; width: 1px; } 104 | .visuallyhidden.focusable:active, 105 | .visuallyhidden.focusable:focus { clip: auto; height: auto; margin: 0; overflow: visible; position: static; width: auto; } 106 | .invisible { visibility: hidden; } 107 | .clearfix:before, .clearfix:after { content: "\0020"; display: block; height: 0; overflow: hidden; } 108 | .clearfix:after { clear: both; } 109 | .clearfix { zoom: 1; } 110 | 111 | 112 | @media all and (orientation:portrait) { 113 | 114 | } 115 | 116 | @media all and (orientation:landscape) { 117 | 118 | } 119 | 120 | @media screen and (max-device-width: 480px) { 121 | 122 | /* html { -webkit-text-size-adjust:none; -ms-text-size-adjust:none; } */ 123 | } 124 | 125 | 126 | @media print { 127 | * { background: transparent !important; color: black !important; text-shadow: none !important; filter:none !important; 128 | -ms-filter: none !important; } 129 | a, a:visited { color: #444 !important; text-decoration: underline; } 130 | a[href]:after { content: " (" attr(href) ")"; } 131 | abbr[title]:after { content: " (" attr(title) ")"; } 132 | .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { content: ""; } 133 | pre, blockquote { border: 1px solid #999; page-break-inside: avoid; } 134 | thead { display: table-header-group; } 135 | tr, img { page-break-inside: avoid; } 136 | @page { margin: 0.5cm; } 137 | p, h2, h3 { orphans: 3; widows: 3; } 138 | h2, h3{ page-break-after: avoid; } 139 | } 140 | -------------------------------------------------------------------------------- /src/db.nim: -------------------------------------------------------------------------------- 1 | # This module is used by the website. 2 | import redis, times, strutils 3 | from sockets import TPort 4 | 5 | type 6 | TDb* = object 7 | r*: TRedis 8 | lastPing: float 9 | 10 | TBuildResult* = enum 11 | bUnknown, bFail, bSuccess 12 | 13 | TTestResult* = enum 14 | tUnknown, tFail, tSuccess 15 | 16 | TEntry* = tuple[c: TCommit, p: seq[TPlatform]] 17 | 18 | TCommit* = object 19 | commitMsg*, username*, hash*, branch*: string 20 | date*: TTime 21 | 22 | # TODO: rename to TBuild? 23 | TPlatform* = object 24 | buildResult*: TBuildResult 25 | testResult*: TTestResult 26 | failReason*, platform*: string 27 | total*, passed*, skipped*, failed*: BiggestInt 28 | csources*: bool 29 | docs*: bool 30 | 31 | const 32 | listName = "commits" 33 | failOnExisting = false 34 | 35 | proc open*(host = "localhost", port: TPort): TDb = 36 | result.r = redis.open(host, port) 37 | result.lastPing = epochTime() 38 | 39 | proc customHSet(database: TDb, name, field, value: string) = 40 | if database.r.hSet(name, field, value).int == 0: 41 | if failOnExisting: 42 | assert(false) 43 | else: 44 | echo("[Warning:REDIS] ", field, " already exists in ", name) 45 | 46 | proc updateProperty*(database: TDb, commitHash, platform, property, 47 | value: string) = 48 | var name = platform & ":" & commitHash 49 | if database.r.hSet(name, property, value).int == 0: 50 | echo("[INFO:REDIS] '$1' field updated in hash" % [property]) 51 | else: 52 | echo("[INFO:REDIS] '$1' new field added to hash" % [property]) 53 | 54 | proc globalProperty*(database: TDb, commitHash, property, value: string) = 55 | if database.r.hSet(commitHash, property, value).int == 0: 56 | echo("[INFO:REDIS] '$1' field updated in hash" % [property]) 57 | else: 58 | echo("[INFO:REDIS] '$1' new field added to hash" % [property]) 59 | 60 | proc addCommit*(database: TDb, commitHash, commitMsg, user, branch: string) = 61 | # Add the commit hash to the `commits` list. 62 | discard database.r.lPush(listName, commitHash) 63 | # Add the commit message, current date and username as a property 64 | globalProperty(database, commitHash, "commitMsg", commitMsg) 65 | globalProperty(database, commitHash, "date", $int(getTime())) 66 | globalProperty(database, commitHash, "username", user) 67 | globalProperty(database, commitHash, "branch", branch) 68 | 69 | proc keepAlive*(database: var TDb) = 70 | ## Keep the connection alive. Ping redis in this case. This functions does 71 | ## not guarantee that redis will be pinged. 72 | var t = epochTime() 73 | if t - database.lastPing >= 60.0: 74 | echo("PING -> redis") 75 | assert(database.r.ping() == "PONG") 76 | database.lastPing = t 77 | 78 | proc getCommits*(database: TDb, 79 | plStr: var seq[string]): seq[TEntry] = 80 | result = @[] 81 | var commitsRaw = database.r.lrange("commits", 0, -1) 82 | for c in items(commitsRaw): 83 | var commit: TCommit 84 | commit.hash = c 85 | for key, value in database.r.hPairs(c): 86 | case normalize(key) 87 | of "commitmsg": commit.commitMsg = value 88 | of "date": commit.date = TTime(parseInt(value)) 89 | of "username": commit.username = value 90 | of "branch": commit.branch = value 91 | else: 92 | echo("[redis] Key not found: ", key) 93 | assert(false) 94 | 95 | var platformsRaw = database.r.lrange(c & ":platforms", 0, -1) 96 | var platforms: seq[TPlatform] = @[] 97 | for p in items(platformsRaw): 98 | var platform: TPlatform 99 | for key, value in database.r.hPairs(p & ":" & c): 100 | case normalize(key) 101 | of "buildresult": 102 | platform.buildResult = parseInt(value).TBuildResult 103 | of "testresult": 104 | platform.testResult = parseInt(value).TTestResult 105 | of "failreason": 106 | platform.failReason = value 107 | of "total": 108 | platform.total = parseBiggestInt(value) 109 | of "passed": 110 | platform.passed = parseBiggestInt(value) 111 | of "skipped": 112 | platform.skipped = parseBiggestInt(value) 113 | of "failed": 114 | platform.failed = parseBiggestInt(value) 115 | of "csources": 116 | platform.csources = if value == "t": true else: false 117 | of "docs": 118 | platform.docs = if value == "t": true else: false 119 | else: 120 | echo("[redis] platf key not found: " & normalize(key)) 121 | assert(false) 122 | 123 | platform.platform = p 124 | 125 | platforms.add(platform) 126 | if p notin plStr: 127 | plStr.add(p) 128 | result.add((commit, platforms)) 129 | 130 | proc commitExists*(database: TDb, commit: string, starts = false): bool = 131 | # TODO: Consider making the 'commits' list a set. 132 | for c in items(database.r.lrange("commits", 0, -1)): 133 | if starts: 134 | if c.startsWith(commit): return true 135 | else: 136 | if c == commit: return true 137 | return false 138 | 139 | proc platformExists*(database: TDb, commit: string, platform: string): bool = 140 | for p in items(database.r.lrange(commit & ":" & "platforms", 0, -1)): 141 | if p == platform: return true 142 | 143 | proc expandHash*(database: TDb, commit: string): string = 144 | for c in items(database.r.lrange("commits", 0, -1)): 145 | if c.startsWith(commit): return c 146 | assert false 147 | 148 | proc isNewest*(database: TDb, commit: string): bool = 149 | return database.r.lIndex("commits", 0) == commit 150 | 151 | proc getNewest*(database: TDb): string = 152 | return database.r.lIndex("commits", 0) 153 | 154 | proc getBranch*(database: TDb, commit: string): string = 155 | if database.r.hExists(commit, "branch"): 156 | return database.r.hGet(commit, "branch") 157 | else: 158 | return "master" 159 | 160 | proc addPlatform*(database: TDb, commit: string, platform: string) = 161 | assert database.commitExists(commit) 162 | assert (not database.platformExists(commit, platform)) 163 | var name = platform & ":" & commit 164 | if database.r.exists(name): 165 | if failOnExisting: quit("[FAIL] " & name & " already exists!", 1) 166 | else: echo("[Warning] " & name & " already exists!") 167 | 168 | discard database.r.lPush(commit & ":" & "platforms", platform) 169 | 170 | proc `[]`*(p: seq[TPlatform], name: string): TPlatform = 171 | for platform in items(p): 172 | if platform.platform == name: 173 | return platform 174 | raise newException(EInvalidValue, name & " platforms not found in commits.") 175 | 176 | proc contains*(p: seq[TPlatform], s: string): bool = 177 | for i in items(p): 178 | if i.platform == s: 179 | return true 180 | 181 | -------------------------------------------------------------------------------- /src/github.nim: -------------------------------------------------------------------------------- 1 | import strtabs, sockets, scgi, strutils, os, json, 2 | osproc, streams, times, parseopt, parseutils 3 | 4 | from cgi import URLDecode 5 | from httpclient import get # httpclient.post conflicts with jester.post 6 | from net import nil 7 | 8 | import asyncio 9 | import asyncdispatch except Port, newDispatcher 10 | 11 | import jester 12 | import types 13 | 14 | 15 | type 16 | TSubnet = object 17 | cidr: range[8 .. 32] 18 | a, b, c, d: int 19 | 20 | PState = ref TState 21 | TState = object of TObject 22 | dispatcher: PDispatcher 23 | sock: PAsyncSocket 24 | scgi: PAsyncScgiState 25 | platform: string 26 | 27 | hubPort: TPort 28 | scgiPort: net.Port 29 | 30 | timeReconnected: float 31 | 32 | subnets: seq[TSubnet] # TODO: Separate into a TGithubAPI object. 33 | apiETag: string 34 | lastAPIAccess: float 35 | 36 | when not defined(ssl): 37 | {.error: "Need SSL support to get Github's IPs, compile with -d:ssl.".} 38 | 39 | # Command line reading 40 | proc getCommandArgs(state: PState) = 41 | for kind, key, value in getOpt(): 42 | case kind 43 | of cmdArgument: 44 | quit("Syntax: ./github --hp:hubPort --sp:scgiPort") 45 | of cmdLongOption, cmdShortOption: 46 | if value == "": 47 | quit("Syntax: ./github --hp:hubPort --sp:scgiPort") 48 | case key 49 | of "hubPort", "hp": 50 | state.hubPort = TPort(parseInt(value)) 51 | of "scgiPort", "sp": 52 | state.scgiPort = net.Port(parseInt(value)) 53 | else: quit("Syntax: ./github -hp hubPort -sp scgiPort") 54 | of cmdEnd: assert false 55 | 56 | # Github specific 57 | 58 | # -- subnets 59 | 60 | proc invalidSubnet(msg: string = "Invalid subnet") = 61 | raise newException(EInvalidValue, msg) 62 | 63 | proc parseSubnet(subnet: string): TSubnet = 64 | var i = 0 65 | 66 | template parsePart(letter: expr, dot: bool) = 67 | var j = parseInt(subnet, letter, i) 68 | if j <= 0: invalidSubnet() 69 | inc(i, j) 70 | if dot: 71 | if subnet[i] == '.': inc(i) 72 | else: invalidSubnet("Invalid subnet, expected '.'.") 73 | 74 | parsePart(result.a, true) 75 | parsePart(result.b, true) 76 | parsePart(result.c, true) 77 | parsePart(result.d, false) 78 | # Parse CIDR 79 | if subnet[i] != '/': invalidSubnet("Invalid subnet, expected '/'.") 80 | inc(i) 81 | var cidr = 0 82 | let j = parseInt(subnet, cidr, i) 83 | if j <= 0: invalidSubnet("Invalid subnet, expected int after '/'.") 84 | inc(i, j) 85 | if subnet[i] != '\0': invalidSubnet("Invalid subnet, expected \0.") 86 | result.cidr = cidr 87 | 88 | proc calcSubmask(cidr: range[8 .. 32]): int = 89 | for i in 0 .. int(cidr)-1: 90 | result = 1 shl (i+(32-cidr)) or result 91 | 92 | proc contains(subnet: TSubnet, ip: string): bool = 93 | # http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_blocks 94 | let submask = (not calcSubmask(subnet.cidr)) and 0xFFFFFFFF # Mask to 32bits 95 | let subnetIP = subnet.a shl 24 or subnet.b shl 16 or 96 | subnet.c shl 8 or subnet.d 97 | let ipmask = parseIP4(ip) 98 | result = (subnetIP or submask) == (ipmask or submask) 99 | 100 | proc getHookSubnets(state: PState, timeout = 3000) = 101 | ## Gets the allowed IP addresses using Github's API. 102 | except: echo(" [Warning] Getting hookSubnets failed: ", getCurrentExceptionMsg()) 103 | 104 | if epochTime() - state.lastAPIAccess < 5.0: 105 | return 106 | 107 | var extraHeaders = 108 | if state.apiETag != "": "If-None-Match: \"" & state.apiETag & "\"\c\L" 109 | else: "" 110 | let resp = httpclient.get("https://api.github.com/meta", extraHeaders) 111 | state.lastAPIAccess = epochTime() 112 | if resp.status[0] in {'4', '5'}: 113 | echo(" [Warning] HookSubnets won't change. Status code was: ", resp.status) 114 | return 115 | elif resp.status[0 .. 2] == "304": 116 | # Nothing changed. 117 | return 118 | 119 | let j = parseJSON(resp.body) 120 | if j.existsKey("hooks"): 121 | for ip in j["hooks"]: state.subnets.add(parseSubnet(ip.str)) 122 | 123 | state.apiETag = resp.headers["ETag"] 124 | 125 | # Communication 126 | 127 | proc parseReply(line: string, expect: string): bool = 128 | var jsonDoc = parseJson(line) 129 | return jsonDoc["reply"].str == expect 130 | 131 | proc hubConnect(state: PState) 132 | proc handleConnect(s: PAsyncSocket, state: PState) = 133 | try: 134 | # Send greeting 135 | var obj = newJObject() 136 | obj["name"] = newJString("github") 137 | obj["platform"] = newJString(state.platform) 138 | obj["version"] = %"1" 139 | state.sock.send($obj & "\c\L") 140 | # Wait for reply. 141 | var line = "" 142 | sleep(1500) 143 | if state.sock.readLine(line): 144 | assert(line != "") 145 | doAssert parseReply(line, "OK") 146 | echo("The hub accepted me!") 147 | else: 148 | raise newException(EInvalidValue, 149 | "Hub didn't accept me. Waited 1.5 seconds.") 150 | except EOS: 151 | echo(getCurrentExceptionMsg()) 152 | s.close() 153 | echo("Waiting 5 seconds.") 154 | sleep(5000) 155 | state.hubConnect() 156 | 157 | proc handleMessage(state: PState, line: string) = 158 | echo("Got message from hub: ", line) 159 | 160 | proc handleModuleMessage(s: PAsyncSocket, state: PState) = 161 | var line = "" 162 | if not state.sock.readLine(line): return # Didn't receive a full line. 163 | if line != "": 164 | state.handleMessage(line) 165 | else: 166 | state.sock.close() 167 | echo("Disconnected from hub: ", osErrorMsg()) 168 | echo("Reconnecting...") 169 | state.hubConnect() 170 | 171 | proc hubConnect(state: PState) = 172 | state.sock = asyncSocket() 173 | state.sock.connect("127.0.0.1", state.hubPort) 174 | state.sock.handleConnect = 175 | proc (s: PAsyncSocket) {.gcsafe.} = handleConnect(s, state) 176 | state.sock.handleRead = 177 | proc (s: PAsyncSocket) {.gcsafe.} = handleModuleMessage(s, state) 178 | state.dispatcher.register(state.sock) 179 | 180 | state.platform = "linux-x86" 181 | state.timeReconnected = -1.0 182 | 183 | proc open(port: TPort = TPort(9321), 184 | scgiPort: net.Port = net.Port(9323)): PState = 185 | new(result) 186 | 187 | result.dispatcher = newDispatcher() 188 | 189 | result.hubPort = port 190 | result.scgiPort = scgiPort 191 | result.subnets = @[] 192 | 193 | result.getCommandArgs() 194 | 195 | result.hubConnect() 196 | 197 | result.apiETag = "" 198 | getHookSubnets(result, timeout = -1) # Get initial set of subnets 199 | 200 | 201 | proc sendBuild(sock: PAsyncSocket, payload: PJsonNode) = 202 | var obj = newJObject() 203 | obj["payload"] = payload 204 | sock.send($obj & "\c\L") 205 | 206 | proc contains(subnets: seq[TSubnet], ip: string): bool = 207 | for subnet in subnets: 208 | if ip in subnet: 209 | return true 210 | 211 | proc isAuthorized(state: PState, ip: string): bool = 212 | result = ip in state.subnets 213 | if result == false: 214 | # Update subnet list 215 | getHookSubnets(state) 216 | result = ip in state.subnets 217 | 218 | when isMainModule: 219 | var state = open() 220 | 221 | settings: 222 | port = state.scgiPort 223 | 224 | routes: 225 | post "/": 226 | let realIP = 227 | if request.ip == "127.0.0.1": 228 | request.headers["X-Real-IP"] 229 | else: 230 | request.ip 231 | echo("[POST] ", realIP) 232 | var hostname = "" 233 | try: 234 | hostname = getHostByAddr(realIP).name 235 | except: 236 | hostname = getCurrentExceptionMsg() 237 | echo(" ", hostname) 238 | let authorized = state.isAuthorized(realIP) 239 | echo(" ", if authorized: "Authorized." else: "Denied.") 240 | cond authorized 241 | let payload = @"payload" 242 | 243 | echo(" Payload:") 244 | for line in splitLines(payload): 245 | echo(" ", line) 246 | 247 | var json = parseJSON(payload) 248 | if json.hasKey("after"): 249 | sendBuild(state.sock, json) 250 | echo(" ", json["after"].str) 251 | resp "Cheers, Github." 252 | 253 | while true: 254 | asyncdispatch.poll() 255 | discard state.dispatcher.poll() 256 | 257 | -------------------------------------------------------------------------------- /public/css/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-size: medium; 3 | } 4 | 5 | body,html { 6 | height: 100%; 7 | } 8 | 9 | a { 10 | /* text-decoration: none; */ 11 | /* That looks interesting */ 12 | } 13 | 14 | #wrapper { 15 | height: auto !important; 16 | margin: 0 auto -21pt; /* For footer */ 17 | min-height: 100%; 18 | } 19 | 20 | div#header { 21 | font-size: 2em; 22 | background-color: #3d3d3d; 23 | border-bottom: solid 2px #000000; 24 | padding: 0.25em; 25 | color: #ffffff; 26 | } 27 | 28 | div#content { 29 | margin: 0.5em; 30 | } 31 | 32 | table { 33 | text-align: left; 34 | margin-top: 1em; 35 | margin-bottom: 0.5em; 36 | font-size: 9pt; 37 | border-collapse: separate; /* Fighting with boilerplate.css here. */ 38 | } 39 | 40 | table td, table th { 41 | padding: 0.45em 0.4em; 42 | } 43 | 44 | td { 45 | border-right: 1px solid #E0E0E0; 46 | } 47 | 48 | th { 49 | background-color: #5D5D5D; 50 | background: -moz-linear-gradient(top, #5D5D5D, #4D4D4D); 51 | background: -webkit-linear-gradient(top, #5D5D5D, #4D4D4D); 52 | background: -o-linear-gradient(top, #5D5D5D, #4D4D4D); 53 | color: #FFFFFF; 54 | text-align: center; 55 | 56 | border-right: 1px solid #3d3d3d; 57 | border-bottom: 1px solid #3d3d3d; 58 | } 59 | 60 | tr:nth-child(even) { 61 | background-color: #eee; 62 | } 63 | 64 | table#downloads td.green { 65 | background: -moz-linear-gradient(top, #00B40C, #03A90E); 66 | background: -webkit-linear-gradient(top, #00B40C, #03A90E); 67 | background: -o-linear-gradient(top, #00B40C, #03A90E); 68 | 69 | border-right: 1px solid #148420; 70 | border-bottom: 1px solid #148420; 71 | 72 | color: #ffffff; 73 | } 74 | 75 | table#downloads td.orange { 76 | background: -moz-linear-gradient(top, #DE9116, #CC8512); 77 | background: -webkit-linear-gradient(top, #DE9116, #CC8512); 78 | background: -o-linear-gradient(top, #DE9116, #CC8512); 79 | 80 | border-right: 1px solid #A86E0F; 81 | border-bottom: 1px solid #A86E0F; 82 | 83 | color: #ffffff; 84 | } 85 | 86 | table#downloads td:hover { 87 | background: -moz-linear-gradient(top, #0099c7, #0294C1); 88 | background: -webkit-linear-gradient(top, #0099c7, #0294C1); 89 | background: -o-linear-gradient(top, #0099c7, #0294C1); 90 | 91 | border-right: solid 1px #077A9C; 92 | border-bottom: solid 1px #077A9C; 93 | 94 | cursor: pointer; 95 | } 96 | 97 | table#downloads td a { 98 | color: #ffffff; 99 | text-decoration: none; 100 | } 101 | 102 | 103 | /* Awesome buttons :P */ 104 | 105 | a.button { 106 | border-radius: 2px 2px 2px 2px; 107 | background: -moz-linear-gradient(top, #f7f7f7, #ebebeb); 108 | background: -webkit-linear-gradient(top, #f7f7f7, #ebebeb); 109 | background: -o-linear-gradient(top, #f7f7f7, #ebebeb); 110 | text-decoration: none; 111 | color: #3d3d3d; 112 | padding: 5px; 113 | border: solid 1px #9d9d9d; 114 | display: inline-block; 115 | position: relative; 116 | text-align: center; 117 | font-size: small; 118 | } 119 | 120 | a.button.active { 121 | background: -moz-linear-gradient(top, #00B40C, #03A90E); 122 | background: -webkit-linear-gradient(top, #00B40C, #03A90E); 123 | background: -o-linear-gradient(top, #00B40C, #03A90E); 124 | border: solid 1px #148420; 125 | color: #ffffff; 126 | } 127 | 128 | a.button.warning { 129 | background: -moz-linear-gradient(top, #DE9116, #CC8512); 130 | background: -webkit-linear-gradient(top, #DE9116, #CC8512); 131 | background: -o-linear-gradient(top, #DE9116, #CC8512); 132 | border: solid 1px #A86E0F; 133 | color: #ffffff; 134 | } 135 | 136 | a.button.left { 137 | border-top-right-radius: 0; 138 | border-bottom-right-radius: 0; 139 | } 140 | 141 | a.button.middle { 142 | border-radius: 0; 143 | border-left: 0; 144 | } 145 | 146 | a.button.right { 147 | border-top-left-radius: 0; 148 | border-bottom-left-radius: 0; 149 | border-left: 0; 150 | } 151 | 152 | a.button:hover { 153 | background: -moz-linear-gradient(top, #0099c7, #0294C1); 154 | background: -webkit-linear-gradient(top, #0099c7, #0294C1); 155 | background: -o-linear-gradient(top, #0099c7, #0294C1); 156 | border: solid 1px #077A9C; 157 | color: #ffffff; 158 | } 159 | 160 | a.button.middle:hover, a.button.right:hover { 161 | border-left: 0; 162 | } 163 | 164 | a.button span.download { 165 | background-image: url("../images/icons.png"); 166 | background-repeat: no-repeat; 167 | display: inline-block; 168 | margin: auto 3px auto auto; 169 | height: 15px; 170 | width: 14px; 171 | position: relative; 172 | background-position: 0 -30px; 173 | top: 3px; 174 | } 175 | 176 | a.button span.book { 177 | background-image: url("../images/icons.png"); 178 | background-repeat: no-repeat; 179 | display: inline-block; 180 | margin: auto 3px auto auto; 181 | height: 15px; 182 | width: 14px; 183 | position: relative; 184 | background-position: 0 0; 185 | top: 3px; 186 | } 187 | 188 | a.button.active span.download, a.button.warning span.download, a.button:hover span.download { 189 | background-position: 0 -45px; 190 | } 191 | 192 | a.button.active span.book, a.button.warning span.book, a.button:hover span.book { 193 | background-position: 0 -15px; 194 | } 195 | 196 | div#header a.button { 197 | float: right; 198 | margin-top: 5px; 199 | } 200 | 201 | div#footer { 202 | background-color: #5D5D5D; 203 | background: -moz-linear-gradient(top, #5D5D5D, #4D4D4D); 204 | background: -webkit-linear-gradient(top, #5D5D5D, #4D4D4D); 205 | background: -o-linear-gradient(top, #5D5D5D, #4D4D4D); 206 | color: #ffffff; 207 | text-align: center; 208 | height: 18pt; 209 | padding-top: 3.5pt; 210 | } 211 | 212 | #footerPush { 213 | height: 21pt; 214 | } 215 | 216 | div#footer a, div#footer a:visited, div#footer a:link { 217 | color: #ffffff; 218 | } 219 | 220 | div#footer a:hover { 221 | text-decoration: none; 222 | } 223 | 224 | /* The new platform build result divs */ 225 | 226 | div.platfBuildResult { 227 | width: 32%; 228 | min-width: 350px; 229 | border-radius: 2px 0px 0px 2px; 230 | float: left; 231 | margin-right: 6pt; 232 | margin-bottom: 6pt; 233 | border-right: 2px solid #1d1d1d; 234 | } 235 | 236 | div.platfBuildResult p { 237 | padding-left: 1%; 238 | } 239 | 240 | div.platfBuildResult p.commitMsg { 241 | text-overflow: ellipsis; 242 | white-space: nowrap; 243 | overflow: hidden; 244 | } 245 | 246 | div.platfBuildResult div.header { 247 | background-color: #5D5D5D; 248 | background: -moz-linear-gradient(top, #5D5D5D, #4D4D4D); 249 | background: -webkit-linear-gradient(top, #5D5D5D, #4D4D4D); 250 | background: -o-linear-gradient(top, #5D5D5D, #4D4D4D); 251 | color: white; 252 | padding-left: 5px; 253 | font-weight: bold; 254 | font-size: 10pt; 255 | padding-top: 4px; 256 | padding-bottom: 3px; 257 | border-bottom: 1px solid #3d3d3d; 258 | border-top: 2px solid #3d3d3d; 259 | } 260 | 261 | div.platfBuildResult div.lastResults div.half, div.platfBuildResult div.lastResults div.branch { 262 | margin-bottom: 2px; 263 | } 264 | 265 | div.platfSuccess { 266 | border-left: 7px solid #43B800; 267 | } 268 | 269 | div.platfWarning { 270 | border-left: 7px solid #E87E15; 271 | } 272 | 273 | div.platfProgress { 274 | border-left: 7px solid #108BDE; 275 | } 276 | 277 | div.platfFailure { 278 | border-left: 7px solid #B50707; 279 | } 280 | 281 | div.indivSuccess { 282 | border-top: 3px solid #43B800; 283 | } 284 | 285 | div.indivWarning { 286 | border-top: 3px solid #E87E15; 287 | } 288 | 289 | div.indivFailure { 290 | border-top: 3px solid #B50707; 291 | } 292 | 293 | div.indivUnknown { 294 | border-top: 3px solid #8A8A8A; 295 | } 296 | 297 | 298 | div.platfBuildResult div.half { 299 | width: 22%; 300 | float: left; 301 | padding-left: 1%; 302 | } 303 | 304 | div.platfBuildResult .lastResults { 305 | padding-bottom: 2px; 306 | } 307 | 308 | div.platfBuildResult .lastResults .branch { 309 | width: 53%; 310 | float: left; 311 | padding-left: 1%; 312 | border-top: 3px solid #8A8A8A; 313 | } 314 | 315 | div.platfBuildResult .lastResults .branch span, span.branch { 316 | border-radius: 2px 2px 2px 2px; 317 | color: white; 318 | font-size: 10pt; 319 | padding: 1px 4px 2px 4px; /* Top right bottom left */ 320 | background-color: #0070BF; 321 | cursor: help; 322 | } 323 | 324 | div.platfBuildResult .lastResults .master span, span.master { 325 | background-color: #21A607; 326 | } 327 | 328 | div.platfBuildResult .buildInfo { 329 | padding-top: 5px; 330 | padding-bottom: 5px; 331 | padding-right: 5px; 332 | border-bottom: 2px solid #1d1d1d; 333 | } 334 | 335 | div#resultsWrapper { 336 | width: 100%; 337 | float: left; 338 | } -------------------------------------------------------------------------------- /src/ircbot.nim: -------------------------------------------------------------------------------- 1 | import irc, sockets, asyncio, json, os, strutils, db, times, redis, irclog, marshal, streams, parseopt 2 | 3 | 4 | type 5 | PState = ref TState 6 | TState = object of TObject 7 | dispatcher: PDispatcher 8 | sock: PAsyncSocket 9 | ircClient: PAsyncIRC 10 | hubPort: TPort 11 | ircServerAddr: string 12 | database: TDb 13 | dbConnected: bool 14 | logger: PLogger 15 | irclogsFilename: string 16 | settings: TSettings 17 | birthdayWish: bool ## Did we wish a happy birthday? :) 18 | 19 | TSettings = object 20 | trustedUsers: seq[tuple[nick: string, host: string]] 21 | announceRepos: seq[string] 22 | announceChans: seq[string] 23 | announceNicks: seq[string] 24 | 25 | TSeenType = enum 26 | PSeenJoin, PSeenPart, PSeenMsg, PSeenNick, PSeenQuit 27 | 28 | TSeen = object 29 | nick: string 30 | channel: string 31 | timestamp: TTime 32 | case kind*: TSeenType 33 | of PSeenJoin: nil 34 | of PSeenPart, PSeenQuit, PSeenMsg: 35 | msg: string 36 | of PSeenNick: 37 | newNick: string 38 | 39 | const 40 | ircServer = "irc.freenode.net" 41 | joinChans = @["#nimrod"] 42 | botNickname = "NimBot" 43 | 44 | proc getCommandArgs(state: PState) = 45 | for kind, key, value in getOpt(): 46 | case kind 47 | of cmdArgument: 48 | quit("Syntax: ./ircbot [--hp hubPort] [--sa serverAddr] --il irclogsPath") 49 | of cmdLongOption, cmdShortOption: 50 | if value == "": 51 | quit("Syntax: ./ircbot [--hp hubPort] --il irclogsPath") 52 | case key 53 | of "serverAddr", "sa": 54 | state.ircServerAddr = value 55 | of "hubPort", "hp": 56 | state.hubPort = TPort(parseInt(value)) 57 | of "irclogs", "il": 58 | state.irclogsFilename = value 59 | else: quit("Syntax: ./ircbot [--hp hubPort] --il irclogsPath") 60 | of cmdEnd: assert false 61 | 62 | proc initSettings(settings: var TSettings) = 63 | settings.trustedUsers = @[(nick: "dom96", host: "unaffiliated/dom96")] 64 | settings.announceRepos = @["Araq/Nimrod"] 65 | settings.announceChans = @["#nimbuild"] 66 | settings.announceNicks = @["dom96"] 67 | 68 | proc saveSettings(state: PState) = 69 | store(newFileStream("nimbot.json", fmWrite), state.settings) 70 | 71 | proc setSeen(d: TDb, s: TSeen) = 72 | #if d.r.isNil: 73 | # echo("[Warning] Redis db nil") 74 | # return 75 | discard d.r.del("seen:" & s.nick) 76 | 77 | var hashToSet = @[("type", $s.kind.int), ("channel", s.channel), 78 | ("timestamp", $s.timestamp.int)] 79 | case s.kind 80 | of PSeenJoin: nil 81 | of PSeenPart, PSeenMsg, PSeenQuit: 82 | hashToSet.add(("msg", s.msg)) 83 | of PSeenNick: 84 | hashToSet.add(("newnick", s.newNick)) 85 | 86 | d.r.hMSet("seen:" & s.nick, hashToSet) 87 | 88 | proc getSeen(d: TDb, nick: string, s: var TSeen): bool = 89 | #if d.r.isNil: 90 | # echo("[Warning] Redis db nil") 91 | # return 92 | if d.r.exists("seen:" & nick): 93 | result = true 94 | s.nick = nick 95 | # Get the type first 96 | s.kind = d.r.hGet("seen:" & nick, "type").parseInt.TSeenType 97 | 98 | for key, value in d.r.hPairs("seen:" & nick): 99 | case normalize(key) 100 | of "type": 101 | # Type is retrieved before this. 102 | of "channel": 103 | s.channel = value 104 | of "timestamp": 105 | s.timestamp = TTime(value.parseInt) 106 | of "msg": 107 | s.msg = value 108 | of "newnick": 109 | s.newNick = value 110 | 111 | template createSeen(typ: TSeenType, n, c: string): stmt {.immediate, dirty.} = 112 | var seenNick: TSeen 113 | seenNick.kind = typ 114 | seenNick.nick = n 115 | seenNick.channel = c 116 | seenNick.timestamp = getTime() 117 | 118 | proc parseReply(line: string, expect: string): Bool = 119 | var jsonDoc = parseJson(line) 120 | return jsonDoc["reply"].str == expect 121 | 122 | proc limitCommitMsg(m: string): string = 123 | ## Limits the message to 300 chars and adds ellipsis. 124 | ## Also gets rid of \n, uses only the first line. 125 | var m1 = m 126 | if NewLines in m1: 127 | m1 = m1.splitLines()[0] 128 | 129 | if m1.len >= 300: 130 | m1 = m1[0..300] 131 | 132 | if m1.len >= 300 or NewLines in m: m1.add("... ") 133 | 134 | if NewLines in m: m1.add($(m.splitLines().len-1) & " more lines") 135 | 136 | return m1 137 | 138 | template pm(chan, msg: string): stmt = 139 | state.ircClient.privmsg(chan, msg) 140 | state.logger.log("NimBot", msg, chan) 141 | 142 | proc announce(state: PState, msg: string, important: bool) = 143 | var newMsg = "" 144 | if important: 145 | newMsg.add(join(state.settings.announceNicks, ",")) 146 | newMsg.add(": ") 147 | newMsg.add(msg) 148 | for i in state.settings.announceChans: 149 | pm(i, newMsg) 150 | 151 | proc isRepoAnnounced(state: PState, url: string): bool = 152 | result = false 153 | for repo in state.settings.announceRepos: 154 | if url.ToLower().endswith(repo.ToLower()): 155 | return true 156 | 157 | proc getBranch(theRef: string): string = 158 | if theRef.startswith("refs/heads/"): 159 | result = theRef[11 .. -1] 160 | else: 161 | result = theRef 162 | 163 | proc handleWebMessage(state: PState, line: string) = 164 | echo("Got message from hub: " & line) 165 | var json = parseJson(line) 166 | if json.existsKey("payload"): 167 | if isRepoAnnounced(state, json["payload"]["repository"]["url"].str): 168 | let commitsToAnnounce = min(4, json["payload"]["commits"].len) 169 | if commitsToAnnounce != 0: 170 | for i in 0..commitsToAnnounce-1: 171 | var commit = json["payload"]["commits"][i] 172 | # Create the message 173 | var message = "" 174 | message.add(json["payload"]["repository"]["owner"]["name"].str & "/" & 175 | json["payload"]["repository"]["name"].str & " ") 176 | message.add(json["payload"]["ref"].str.getBranch() & " ") 177 | message.add(commit["id"].str[0..6] & " ") 178 | message.add(commit["author"]["name"].str & " ") 179 | message.add("[+" & $commit["added"].len & " ") 180 | message.add("±" & $commit["modified"].len & " ") 181 | message.add("-" & $commit["removed"].len & "]: ") 182 | message.add(limitCommitMsg(commit["message"].str)) 183 | 184 | # Send message to #nimrod. 185 | pm(joinChans[0], message) 186 | if commitsToAnnounce != json["payload"]["commits"].len: 187 | let unannounced = json["payload"]["commits"].len-commitsToAnnounce 188 | pm(joinChans[0], $unannounced & " more commits.") 189 | else: 190 | # New branch 191 | var message = "" 192 | message.add(json["payload"]["repository"]["owner"]["name"].str & "/" & 193 | json["payload"]["repository"]["name"].str & " ") 194 | let theRef = json["payload"]["ref"].str.getBranch() 195 | if existsKey(json["payload"], "base_ref"): 196 | let baseRef = json["payload"]["base_ref"].str.getBranch() 197 | message.add("New branch: " & baseRef & " -> " & theRef) 198 | else: 199 | message.add("New branch: " & theRef) 200 | 201 | message.add(" by " & json["payload"]["pusher"]["name"].str) 202 | 203 | elif json.existsKey("redisinfo"): 204 | assert json["redisinfo"].existsKey("port") 205 | let redisPort = json["redisinfo"]["port"].num 206 | state.database = db.open(port = TPort(redisPort)) 207 | state.dbConnected = true 208 | elif json.existsKey("announce"): 209 | announce(state, json["announce"].str, json["important"].bval) 210 | 211 | proc hubConnect(state: PState) 212 | proc handleConnect(s: PAsyncSocket, state: PState) = 213 | try: 214 | # Send greeting 215 | var obj = newJObject() 216 | obj["name"] = newJString("irc") 217 | obj["platform"] = newJString("?") 218 | obj["version"] = %"1" 219 | state.sock.send($obj & "\c\L") 220 | 221 | # Wait for reply. 222 | var line = "" 223 | sleep(1500) 224 | if state.sock.recvLine(line): 225 | assert(line != "") 226 | doAssert parseReply(line, "OK") 227 | echo("The hub accepted me!") 228 | else: 229 | raise newException(EInvalidValue, 230 | "Hub didn't accept me. Waited 1.5 seconds.") 231 | 232 | # ask for the redis info 233 | var riobj = newJObject() 234 | riobj["do"] = newJString("redisinfo") 235 | state.sock.send($riobj & "\c\L") 236 | 237 | except EOS, EInvalidValue, EAssertionFailed: 238 | echo(getCurrentExceptionMsg()) 239 | s.close() 240 | echo("Waiting 5 seconds...") 241 | sleep(5000) 242 | state.hubConnect() 243 | 244 | proc handleRead(s: PAsyncSocket, state: PState) = 245 | var line = "" 246 | if state.sock.recvLine(line): 247 | if line != "": 248 | # Handle the message 249 | state.handleWebMessage(line) 250 | else: 251 | echo("Disconnected from hub: ", OSErrorMsg()) 252 | announce(state, "Got disconnected from hub! " & OSErrorMsg(), true) 253 | state.sock.close() 254 | echo("Reconnecting...") 255 | state.hubConnect() 256 | else: 257 | echo(OSErrorMsg()) 258 | 259 | proc hubConnect(state: PState) = 260 | state.sock = AsyncSocket() 261 | state.sock.connect("127.0.0.1", state.hubPort) 262 | state.sock.handleConnect = proc (s: PAsyncSocket) = handleConnect(s, state) 263 | state.sock.handleRead = proc (s: PAsyncSocket) = handleRead(s, state) 264 | 265 | state.dispatcher.register(state.sock) 266 | 267 | proc isUserTrusted(state: PState, nick, host: string): bool = 268 | for i in state.settings.trustedUsers: 269 | if i.nick == nick and i.host == host: 270 | return true 271 | return false 272 | 273 | proc addDup[T](s: var seq[T], v: T) = 274 | ## Adds only if it doesn't already exist in seq. 275 | if v notin s: 276 | s.add(v) 277 | 278 | proc delTrust(s: var seq[tuple[nick: string, host: string]], nick, host: string): bool = 279 | for i in 0..s.len-1: 280 | if s[i].nick == nick and s[i].host == host: 281 | s.del(i) 282 | return true 283 | return false 284 | 285 | proc del[T](s: var seq[T], v: T): bool = 286 | for i in 0..s.len-1: 287 | if s[i] == v: 288 | s.del(i) 289 | return true 290 | return false 291 | 292 | proc `$`(s: seq[tuple[nick: string, host: string]]): string = 293 | result = "" 294 | for i in s: 295 | result.add(i.nick & "@" & i.host & ", ") 296 | result = result[0 .. -3] 297 | 298 | proc isFilwitBirthday(): bool = 299 | result = false 300 | let t = getTime().getGMTime() 301 | if t.month == mSep: 302 | if t.monthday == 10 and t.hour >= 19: 303 | return true 304 | if t.monthday == 11 and t.hour <= 8: 305 | return true 306 | 307 | proc handleIrc(irc: PAsyncIRC, event: TIRCEvent, state: PState) = 308 | case event.typ 309 | of EvConnected: nil 310 | of EvDisconnected: 311 | echo("Disconnected from server.") 312 | state.ircClient.reconnect() 313 | of EvMsg: 314 | echo("< ", event.raw) 315 | # Logs: 316 | state.logger.log(event) 317 | template pmOrig(msg: string) = 318 | pm(event.origin, msg) 319 | case event.cmd 320 | of MPrivMsg: 321 | let msg = event.params[event.params.len-1] 322 | let words = msg.split(' ') 323 | case words[0] 324 | of "!ping": pmOrig("pong") 325 | of "!lag": 326 | if state.ircClient.getLag != -1.0: 327 | var lag = state.ircClient.getLag 328 | lag = lag * 1000.0 329 | pmOrig($int(lag) & "ms between me and the server.") 330 | else: 331 | pmOrig("Unknown.") 332 | of "!seen": 333 | if words.len > 1: 334 | let nick = words[1] 335 | if nick == botNickname: 336 | pmOrig("Yes, I see myself.") 337 | var seenInfo: TSeen 338 | if state.database.getSeen(nick, seenInfo): 339 | case seenInfo.kind 340 | of PSeenMsg: 341 | pmOrig("$1 was last seen on $2 in $3 saying: $4" % 342 | [seenInfo.nick, $seenInfo.timestamp, 343 | seenInfo.channel, seenInfo.msg]) 344 | of PSeenJoin: 345 | pmOrig("$1 was last seen on $2 joining $3" % 346 | [seenInfo.nick, $seenInfo.timestamp, seenInfo.channel]) 347 | of PSeenPart: 348 | pmOrig("$1 was last seen on $2 leaving $3 with message: $4" % 349 | [seenInfo.nick, $seenInfo.timestamp, seenInfo.channel, 350 | seenInfo.msg]) 351 | of PSeenQuit: 352 | pmOrig("$1 was last seen on $2 quitting with message: $3" % 353 | [seenInfo.nick, $seenInfo.timestamp, seenInfo.msg]) 354 | of PSeenNick: 355 | pmOrig("$1 was last seen on $2 changing nick to $3" % 356 | [seenInfo.nick, $seenInfo.timestamp, seenInfo.newNick]) 357 | 358 | else: 359 | pmOrig("I have not seen " & nick) 360 | else: 361 | pmOrig("Syntax: !seen ") 362 | of "!addtrust": 363 | if words.len > 2: 364 | if isUserTrusted(state, event.nick, event.host): 365 | state.settings.trustedUsers.addDup((words[1], words[2])) 366 | saveSettings(state) 367 | pmOrig("Done.") 368 | else: 369 | pmOrig("Access denied.") 370 | else: 371 | pmOrig("Syntax: !addtrust ") 372 | of "!remtrust": 373 | if words.len > 2: 374 | if isUserTrusted(state, event.nick, event.host): 375 | if state.settings.trustedUsers.delTrust(words[1], words[2]): 376 | saveSettings(state) 377 | pmOrig("Done.") 378 | else: 379 | pmOrig("Could not find user") 380 | else: 381 | pmOrig("Access denied.") 382 | else: 383 | pmOrig("Syntax: !remtrust ") 384 | of "!trusted": 385 | pmOrig("Trusted users: " & $state.settings.trustedUsers) 386 | of "!addrepo": 387 | if words.len > 2: 388 | if isUserTrusted(state, event.nick, event.host): 389 | state.settings.announceRepos.addDup(words[1] & "/" & words[2]) 390 | saveSettings(state) 391 | pmOrig("Done.") 392 | else: 393 | pmOrig("Access denied.") 394 | else: 395 | pmOrig("Syntax: !addrepo ") 396 | of "!remrepo": 397 | if words.len > 2: 398 | if isUserTrusted(state, event.nick, event.host): 399 | if state.settings.announceRepos.del(words[1] & "/" & words[2]): 400 | saveSettings(state) 401 | pmOrig("Done.") 402 | else: 403 | pmOrig("Repo not found.") 404 | else: 405 | pmOrig("Access denied.") 406 | else: 407 | pmOrig("Syntax: !remrepo ") 408 | of "!repos": 409 | pmOrig("Announced repos: " & state.settings.announceRepos.join(", ")) 410 | of "!addnick": 411 | if words.len > 1: 412 | if isUserTrusted(state, event.nick, event.host): 413 | state.settings.announceNicks.addDup(words[1]) 414 | saveSettings(state) 415 | pmOrig("Done.") 416 | else: 417 | pmOrig("Access denied.") 418 | else: 419 | pmOrig("Syntax: !addnick ") 420 | of "!remnick": 421 | if words.len > 1: 422 | if isUserTrusted(state, event.nick, event.host): 423 | if state.settings.announceNicks.del(words[1]): 424 | saveSettings(state) 425 | pmOrig("Done.") 426 | else: 427 | pmOrig("Nick not found.") 428 | else: 429 | pmOrig("Access denied.") 430 | else: 431 | pmOrig("Syntax: !remnick ") 432 | of "!nicks": 433 | pmOrig("Announce nicks: " & state.settings.announceNicks.join(", ")) 434 | 435 | if words[0].startswith("!kirbyrape"): 436 | pmOrig("(>^(>O_O)>") 437 | 438 | # TODO: ... commands 439 | 440 | # -- Seen 441 | # Log this as activity. 442 | createSeen(PSeenMsg, event.nick, event.origin) 443 | seenNick.msg = msg 444 | state.database.setSeen(seenNick) 445 | of MJoin: 446 | createSeen(PSeenJoin, event.nick, event.origin) 447 | state.database.setSeen(seenNick) 448 | if event.nick == "filwit" and isFilwitBirthday() and (not state.birthdayWish): 449 | pmOrig("Happy birthday to you, happy birthday to you! Happy BIRTHDAY " & 450 | "dear filwit! happy birthday to you!!!") 451 | state.birthdayWish = true 452 | of MPart: 453 | createSeen(PSeenPart, event.nick, event.origin) 454 | let msg = event.params[event.params.high] 455 | seenNick.msg = msg 456 | state.database.setSeen(seenNick) 457 | of MQuit: 458 | createSeen(PSeenQuit, event.nick, event.origin) 459 | let msg = event.params[event.params.high] 460 | seenNick.msg = msg 461 | state.database.setSeen(seenNick) 462 | of MNick: 463 | createSeen(PSeenNick, event.nick, "#nimrod") 464 | seenNick.newNick = event.params[0] 465 | state.database.setSeen(seenNick) 466 | of MNumeric: 467 | if event.numeric == "433": 468 | # Nickname already in use. 469 | irc.send("NICK " & irc.getNick() & "_") 470 | else: 471 | nil # TODO: ? 472 | 473 | proc open(port: TPort = TPort(5123)): PState = 474 | var cres: PState 475 | new(cres) 476 | cres.dispatcher = newDispatcher() 477 | cres.settings.initSettings() 478 | if existsFile("nimbot.json"): 479 | load(newFileStream("nimbot.json", fmRead), cres.settings) 480 | 481 | cres.hubPort = port 482 | cres.irclogsFilename = "" 483 | cres.ircServerAddr = ircServer 484 | cres.getCommandArgs() 485 | 486 | if cres.irclogsFilename == "": 487 | quit("You need to specify the irclogs filename.") 488 | 489 | cres.hubConnect() 490 | 491 | # Connect to the irc server. 492 | let ie = proc (irc: PAsyncIRC, event: TIRCEvent) = 493 | handleIrc(irc, event, cres) 494 | var joinChannels = joinChans 495 | joinChannels.add(cres.settings.announceChans) 496 | cres.ircClient = AsyncIrc(cres.ircServerAddr, nick = botNickname, 497 | user = botNickname, joinChans = joinChannels, ircEvent = ie) 498 | cres.ircClient.connect() 499 | cres.dispatcher.register(cres.ircClient) 500 | 501 | cres.dbConnected = false 502 | 503 | cres.logger = newLogger(cres.irclogsFilename) 504 | result = cres 505 | 506 | proc isBDFLsBirthday(): bool = 507 | result = false 508 | let t = getTime().getGMTime() 509 | if t.month == mJun: 510 | if t.monthday == 16 and t.hour >= 22: 511 | return true 512 | if t.monthday == 17 and t.hour <= 21: 513 | return true 514 | 515 | var state = ircbot.open() # Connect to the website and the IRC server. 516 | 517 | while state.dispatcher.poll(): 518 | if state.dbConnected: 519 | state.database.keepAlive() 520 | 521 | if isBDFLsBirthday() and not state.birthdayWish: 522 | pm("#nimrod", "It's Araq's birthday today! Everybody wish our great BDFL a happy birthday!!!") 523 | state.birthdayWish = true 524 | -------------------------------------------------------------------------------- /src/builder.nim: -------------------------------------------------------------------------------- 1 | # This will build nimrod using the specified settings. 2 | import 3 | osproc, json, sockets, asyncio, os, streams, parsecfg, parseopt, strutils, 4 | ftpclient, times, strtabs 5 | import types 6 | 7 | const 8 | builderVer = "0.2" 9 | buildReadme = """ 10 | This is a minimal distribution of the Nimrod compiler. Full source code can be 11 | found at http://github.com/Araq/Nimrod 12 | """ 13 | webFP = {fpUserRead, fpUserWrite, fpUserExec, 14 | fpGroupRead, fpGroupExec, fpOthersRead, fpOthersExec} 15 | 16 | type 17 | TJob = object 18 | payload: PJsonNode 19 | p: PProcess ## Current process that is running. 20 | cmd: string 21 | 22 | TCfg = object 23 | nimLoc: string ## Location of the nimrod repo 24 | websiteLoc: string ## Location of the website. 25 | logLoc: string ## Location of the logs for this module. 26 | zipLoc: string ## Location of where to copy the files for zipping. 27 | docgen: bool ## Determines whether to generate docs. 28 | csourceGen: bool ## Determines whether to generate csources. 29 | csourceExtraBuildArgs: string 30 | innoSetupGen: bool 31 | platform: string 32 | hubAddr: string 33 | hubPort: int 34 | hubPass: string 35 | 36 | ftpUser: string 37 | ftpPass: string 38 | ftpPort: TPort 39 | ftpUploadDir: string 40 | 41 | requestNewest: bool 42 | deleteOutgoing: bool 43 | 44 | TState = object of TObject 45 | dispatcher: PDispatcher 46 | sock: PAsyncSocket 47 | building: bool 48 | buildJob: TJob ## Current build 49 | skipCSource: bool ## Skip the process of building csources 50 | logFile: TFile 51 | cfg: TCfg 52 | lastMsgTime: float ## The last time a message was received from the hub. 53 | pinged: float 54 | reconnecting: bool 55 | buildThread: TThread[int] # TODO: Change to void when bug is fixed. 56 | 57 | PState = ref TState 58 | 59 | TBuildProgressType = enum 60 | ProcessStart, ProcessExit, HubMsg, BuildEnd 61 | 62 | TBuildProgress = object ## This object gets sent to the main thread, by the builder thread. 63 | case kind: TBuildProgressType 64 | of ProcessStart: 65 | p: PProcess 66 | of ProcessExit, BuildEnd: nil 67 | of HubMsg: 68 | msg: string 69 | 70 | TThreadCommandType = enum 71 | BuildTerminate, BuildStart 72 | 73 | TThreadCommand = object 74 | case kind: TThreadCommandType 75 | of BuildTerminate: nil 76 | of BuildStart: 77 | payload: PJsonNode 78 | cfg: TCfg 79 | 80 | EBuildEnd = object of ESynch 81 | 82 | var 83 | hubChan: TChannel[TBuildProgress] 84 | threadCommandChan: TChannel[TThreadCommand] 85 | 86 | hubChan.open() 87 | threadCommandChan.open() 88 | 89 | # Configuration 90 | proc parseConfig(state: PState, path: string) = 91 | var f = newFileStream(path, fmRead) 92 | if f != nil: 93 | var p: TCfgParser 94 | open(p, f, path) 95 | var count = 0 96 | while true: 97 | var n = next(p) 98 | case n.kind 99 | of cfgEof: 100 | break 101 | of cfgSectionStart: 102 | raise newException(EInvalidValue, "Unknown section: " & n.section) 103 | of cfgKeyValuePair, cfgOption: 104 | case normalize(n.key) 105 | of "platform": 106 | state.cfg.platform = n.value 107 | inc(count) 108 | if ':' in state.cfg.platform: quit("No ':' allowed in the platform name.") 109 | of "nimgitpath": 110 | state.cfg.nimLoc = n.value 111 | inc(count) 112 | of "websitepath": 113 | state.cfg.websiteLoc = n.value 114 | inc(count) 115 | of "logfilepath": 116 | state.cfg.logLoc = n.value 117 | inc(count) 118 | of "archivepath": 119 | state.cfg.zipLoc = n.value 120 | inc(count) 121 | of "docgen": 122 | state.cfg.docgen = if normalize(n.value) == "true": true else: false 123 | of "csourcegen": 124 | state.cfg.csourceGen = if normalize(n.value) == "true": true else: false 125 | of "innogen": 126 | state.cfg.innoSetupGen = if normalize(n.value) == "true": true else: false 127 | of "csourceextrabuildargs": 128 | state.cfg.csourceExtraBuildArgs = n.value 129 | of "hubaddr": 130 | state.cfg.hubAddr = n.value 131 | inc(count) 132 | of "hubport": 133 | state.cfg.hubPort = parseInt(n.value) 134 | inc(count) 135 | of "hubpass": 136 | state.cfg.hubPass = n.value 137 | of "ftpuser": 138 | state.cfg.ftpUser = n.value 139 | of "ftppass": 140 | state.cfg.ftpPass = n.value 141 | of "ftpport": 142 | state.cfg.ftpPort = parseInt(n.value).TPort 143 | of "ftpuploaddir": 144 | state.cfg.ftpUploadDir = n.value 145 | of "requestnewest": 146 | state.cfg.requestNewest = 147 | if normalize(n.value) == "true": true else: false 148 | of "deleteoutgoing": 149 | state.cfg.deleteOutgoing = 150 | if normalize(n.value) == "true": true else: false 151 | of cfgError: 152 | raise newException(EInvalidValue, "Configuration parse error: " & n.msg) 153 | if count < 7: 154 | quit("Not all settings have been specified in the .ini file", QuitFailure) 155 | if state.cfg.ftpUser != "" and state.cfg.ftpPass == "": 156 | quit("When ftpUser is specified so must the ftpPass.") 157 | 158 | close(p) 159 | else: 160 | quit("Cannot open configuration file: " & path, QuitFailure) 161 | 162 | proc defaultState(): PState = 163 | new(result) 164 | result.cfg.hubAddr = "127.0.0.1" 165 | result.cfg.hubPass = "" 166 | 167 | result.cfg.ftpUser = "" 168 | result.cfg.ftpPass = "" 169 | result.cfg.ftpPort = TPort(21) 170 | 171 | result.lastMsgTime = epochTime() 172 | result.pinged = -1.0 173 | 174 | result.cfg.csourceExtraBuildArgs = "" 175 | 176 | proc initJob(): TJob = 177 | result.payload = nil 178 | 179 | # Build of Nimrod/tests/docs gen 180 | 181 | template sendHubMsg(m: string): stmt = 182 | var bp: TBuildProgress 183 | bp.kind = HubMsg 184 | bp.msg = m 185 | hubChan.send(bp) 186 | 187 | proc hubSendBuildStart(hash, branch: string) = 188 | var obj = %{"eventType": %(int(bStart)), 189 | "hash": %hash, 190 | "branch": %branch} 191 | sendHubMsg($obj & "\c\L") 192 | 193 | proc hubSendProcessStart(process: PProcess, cmd, args: string) = 194 | var bp: TBuildProgress 195 | bp.kind = ProcessStart 196 | bp.p = process 197 | hubChan.send(bp) 198 | var obj = %{"desc": %("\"" & cmd & " " & args & "\" started."), 199 | "eventType": %(int(bProcessStart)), 200 | "cmd": %cmd, 201 | "args": %args} 202 | sendHubMsg($obj & "\c\L") 203 | 204 | proc hubSendProcessLine(line: string) = 205 | var obj = %{"eventType": %(int(bProcessLine)), 206 | "line": %line} 207 | sendHubMsg($obj & "\c\L") 208 | 209 | proc hubSendProcessExit(exitCode: int) = 210 | var bp: TBuildProgress 211 | bp.kind = ProcessExit 212 | hubChan.send(bp) 213 | var obj = %{"eventType": %(int(bProcessExit)), 214 | "exitCode": %exitCode} 215 | sendHubMsg($obj & "\c\L") 216 | 217 | proc hubSendFTPUploadSpeed(speed: float) = 218 | var obj = %{"desc": %("FTP Upload at " & formatFloat(speed) & "KB/s"), 219 | "eventType": %(int(bFTPUploadSpeed)), 220 | "speed": %speed} 221 | sendHubMsg($obj & "\c\L") 222 | 223 | proc hubSendJobUpdate(job: TBuilderJob) = 224 | var obj = %{"job": %(int(job))} 225 | sendHubMsg($obj & "\c\L") 226 | 227 | proc hubSendBuildFail(msg: string) = 228 | var obj = %{"result": %(int(Failure)), 229 | "detail": %msg} 230 | sendHubMsg($obj & "\c\L") 231 | 232 | proc hubSendBuildSuccess() = 233 | var obj = %{"result": %(int(Success))} 234 | sendHubMsg($obj & "\c\L") 235 | 236 | proc hubSendBuildTestSuccess(total, passed, skipped, failed: BiggestInt, 237 | diff, results: PJsonNode) = 238 | var obj = %{"result": %(int(Success)), 239 | "total": %(total), 240 | "passed": %(passed), 241 | "skipped": %(skipped), 242 | "failed": %(failed), 243 | "diff": diff, 244 | "results": results} 245 | sendHubMsg($obj & "\c\L") 246 | 247 | proc hubSendBuildEnd() = 248 | var bp: TBuildProgress 249 | bp.kind = BuildEnd 250 | hubChan.send(bp) 251 | 252 | var obj = %{"eventType": %(int(bEnd))} 253 | sendHubMsg($obj & "\c\L") 254 | 255 | proc dCopyFile(src, dest: string) = 256 | echo("[INFO] Copying ", src, " to ", dest) 257 | copyFile(src, dest) 258 | 259 | proc dMoveFile(src, dest: string) = 260 | echo("[INFO] Moving ", src, " to ", dest) 261 | copyFile(src, dest) 262 | removeFile(src) 263 | 264 | proc dCopyDir(src, dest: string) = 265 | echo("[INFO] Copying directory ", src, " to ", dest) 266 | copyDir(src, dest) 267 | 268 | proc dCreateDir(s: string) = 269 | echo("[INFO] Creating directory ", s) 270 | createDir(s) 271 | 272 | proc dMoveDir(s: string, s1: string) = 273 | echo("[INFO] Moving directory ", s, " to ", s1) 274 | copyDir(s, s1) 275 | removeDir(s) 276 | 277 | proc dRemoveDir(s: string) = 278 | echo("[INFO] Removing directory ", s) 279 | removeDir(s) 280 | 281 | proc dRemoveFile(s: string) = 282 | echo("[INFO] Removing file ", s) 283 | removeFile(s) 284 | 285 | proc copyForArchive(nimLoc, dest, bin: string) = 286 | dCreateDir(dest / "bin") 287 | var nimBin = "bin" / addFileExt(bin, ExeExt) 288 | dCopyFile(nimLoc / nimBin, dest / nimBin) 289 | dCopyFile(nimLoc / "readme.txt", dest / "readme.txt") 290 | dCopyFile(nimLoc / "copying.txt", dest / "copying.txt") 291 | #dCopyFile(nimLoc / "gpl.html", dest / "gpl.html") 292 | writeFile(dest / "readme2.txt", buildReadme) 293 | dCopyDir(nimLoc / "config", dest / "config") 294 | dCopyDir(nimLoc / "lib", dest / "lib") 295 | 296 | proc clearOutgoing(websitePath, platform: string) = 297 | echo("Clearing outgoing folder...") 298 | dRemoveDir(websitePath / "commits" / platform) 299 | dCreateDir(websitePath / "commits" / platform) 300 | 301 | # TODO: Make this a template? 302 | proc tally3(obj: PJsonNode, name: string, 303 | total, passed, skipped: var BiggestInt) = 304 | total = total + obj[name]["total"].num 305 | passed = passed + obj[name]["passed"].num 306 | skipped = skipped + obj[name]["skipped"].num 307 | 308 | proc tallyTestResults(path: string): 309 | tuple[total, passed, skipped, failed: BiggestInt, diff, results: PJsonNode] = 310 | # TODO: Refactor this monstrosity. 311 | var f = readFile(path) 312 | var obj = parseJson(f) 313 | var total: BiggestInt = 0 314 | var passed: BiggestInt = 0 315 | var skipped: BiggestInt = 0 316 | var diff: PJsonNode = newJNull() 317 | var results: PJsonNode = newJNull() 318 | if obj.hasKey("reject") and obj.hasKey("compile") and obj.hasKey("run"): 319 | tally3(obj, "reject", total, passed, skipped) 320 | tally3(obj, "compile", total, passed, skipped) 321 | tally3(obj, "run", total, passed, skipped) 322 | elif obj.hasKey("total") and obj.hasKey("passed") and obj.hasKey("skipped"): 323 | total = obj["total"].num 324 | passed = obj["passed"].num 325 | skipped = obj["skipped"].num 326 | if obj.hasKey("diff"): 327 | diff = obj["diff"] 328 | if obj.hasKey("results"): 329 | results = obj["results"] 330 | else: 331 | raise newException(EBuildEnd, "Invalid testresults.json.") 332 | 333 | return (total, passed, skipped, total - (passed + skipped), diff, results) 334 | 335 | proc fileInModified(json: PJsonNode, file: string): bool = 336 | if json.hasKey("commits"): 337 | for commit in items(json["commits"].elems): 338 | for f in items(commit["modified"].elems): 339 | if f.str == file: return true 340 | 341 | template buildTmpl(infoName: expr, body: stmt): stmt {.immediate.} = 342 | while true: 343 | let thrCmd = threadCommandChan.recv() 344 | case thrCmd.kind: 345 | of BuildTerminate: 346 | echo("[Warning] No bootstrap running.") 347 | of BuildStart: 348 | var infoName = thrCmd 349 | try: 350 | body 351 | except EBuildEnd: 352 | hubSendBuildFail(getCurrentExceptionMsg()) 353 | hubSendBuildEnd() 354 | if info.cfg.deleteOutgoing: 355 | clearOutgoing(info.cfg.websiteLoc, info.cfg.platform) 356 | 357 | proc runProcess(env: PStringTable = nil, workDir, execFile: string, 358 | args: openarray[string]): bool = 359 | ## Returns ``true`` if process finished successfully. Otherwise ``false``. 360 | result = true 361 | var cmd = "" 362 | if isAbsolute(execFile): 363 | cmd = execFile.changeFileExt(ExeExt) 364 | else: 365 | cmd = workDir / execFile.changeFileExt(ExeExt) 366 | var process = startProcess(cmd, workDir, args, env) 367 | hubSendProcessStart(process, execFile.extractFilename, join(args, " ")) 368 | var pStdout = process.outputStream 369 | proc hasProcessTerminated(process: PProcess, exitCode: var int): bool = 370 | result = false 371 | exitCode = process.peekExitCode() 372 | if exitCode != -1: 373 | hubSendProcessExit(exitCode) 374 | return true 375 | var line = "" 376 | var exitCode = -1 377 | while true: 378 | line = "" 379 | if pStdout.readLine(line) and line != "": 380 | hubSendProcessLine(line) 381 | if hasProcessTerminated(process, exitCode): 382 | break 383 | result = exitCode == QuitSuccess 384 | echo("! " & execFile.extractFilename & " " & join(args, " ") & " exited with ", exitCode) 385 | process.close() 386 | 387 | proc changeNimrodInPATH(bindir: string): string = 388 | var paths = getEnv("PATH").split(PathSep) 389 | for i in 0 .. 0: 406 | let thrCmd = threadCommandChan.recv() 407 | case thrCmd.kind: 408 | of BuildTerminate: 409 | raise newException(EBuildEnd, "Bootstrap aborted.") 410 | of BuildStart: 411 | threadCommandChan.send(TThreadCommand(kind: BuildTerminate)) 412 | threadCommandChan.send(thrCmd) 413 | 414 | proc run(workDir: string, exec: string, args: varargs[string]) = 415 | run(nil, workDir, exec, args) 416 | 417 | proc exe(f: string): string = return addFileExt(f, ExeExt) 418 | 419 | proc restoreBranchSpecificBin(dir, bin, branch: string) = 420 | let branchSpecificBin = dir / (bin & "_" & branch).exe 421 | if existsFile(branchSpecificBin): 422 | copyFile(branchSpecificBin, dir / bin.exe) 423 | # Make sure that the binary has +x permissions. 424 | inclFilePermissions(dir / bin.exe, {fpUserExec, fpGroupExec, fpOthersExec}) 425 | elif existsFile(dir / bin.exe): 426 | # Delete the current binary to prevent any issues with old binaries. 427 | removeFile(dir / bin.exe) 428 | 429 | proc backupBranchSpecificBin(dir, bin, branch: string) = 430 | if existsFile(dir / bin.exe): 431 | copyFile(dir / bin.exe, dir / (bin & "_" & branch).exe) 432 | 433 | proc setGIT(payload: PJsonNode, nimLoc: string) = 434 | ## Cleans working tree, changes branch and pulls. 435 | let branch = payload["ref"].str[11 .. ^1] 436 | let commitHash = payload["after"].str 437 | 438 | run(nimLoc, findExe("git"), "checkout", "--", ".") 439 | run(nimLoc, findExe("git"), "fetch", "--all") 440 | run(nimLoc, findExe("git"), "checkout", "-f", "origin/" & branch) 441 | # TODO: Capture changed files from output? 442 | run(nimLoc, findExe("git"), "checkout", commitHash) 443 | 444 | # Determine the nim binary name. Likely 'nim' now. 445 | if existsFile(nimLoc / "compiler" / "nim.nim"): 446 | payload["nimBin"] = %"nim" 447 | else: 448 | payload["nimBin"] = %"nimrod" 449 | 450 | # If a branch specific nimrod binary exists. Change to it. 451 | restoreBranchSpecificBin(nimLoc / "bin", payload["nimBin"].str, branch) 452 | restoreBranchSpecificBin(nimLoc, "koch", branch) 453 | 454 | # Handle C sources 455 | let prevCSourcesHead = 456 | if existsFile(nimLoc / "csources" / ".git" / "refs" / "heads" / branch): 457 | readFile(nimLoc / "csources" / ".git" / "refs" / "heads" / branch) 458 | else: 459 | "" 460 | if existsDir(nimLoc / "csources" / ".git"): 461 | removeDir(nimLoc / "csources") 462 | run(nimLoc, findExe("git"), "clone", "-b", branch, "--depth", "1", 463 | "https://github.com/nimrod-code/csources") 464 | 465 | let currCSourcesHead = readFile(nimLoc / "csources" / ".git" / 466 | "refs" / "heads" / branch) 467 | # Save whether C sources have changed in the payload so that ``nimBootstrap`` 468 | # is aware of it. 469 | payload["csources"] = %(not (prevCSourcesHead == currCSourcesHead)) 470 | 471 | proc clean(nimLoc: string) = 472 | echo "Cleaning up." 473 | proc removePattern(pattern: string) = 474 | for f in walkFiles(pattern): 475 | removeFile(f) 476 | removePattern(nimLoc / "web/*.html") 477 | removePattern(nimLoc / "doc/*.html") 478 | removeFile(nimLoc / "testresults.json") 479 | removeFile(nimLoc / "testresults.html") 480 | 481 | proc nimBootstrap(payload: PJsonNode, nimLoc, csourceExtraBuildArgs: string) = 482 | ## Set of steps to bootstrap Nimrod. In debug and release mode. 483 | ## Does not perform any git actions! 484 | 485 | let nimBin = payload["nimBin"].str 486 | 487 | # skipCSource is already set to true if 'csources.zip' changed. 488 | # force running of ./build.sh if the nimrod binary is nonexistent. 489 | if payload["csources"].bval or 490 | not existsFile(nimLoc / "bin" / nimBin.exe): 491 | clean(nimLoc) 492 | 493 | # Unzip C Sources 494 | when defined(windows): 495 | # build.bat 496 | run(nimLoc / "csources", getEnv("COMSPEC"), "/c", "build.bat", csourceExtraBuildArgs) 497 | else: 498 | # ./build.sh 499 | run(nimLoc / "csources", findExe("sh"), "build.sh", csourceExtraBuildArgs) 500 | 501 | if (not existsFile(nimLoc / "koch".exe)) or 502 | fileInModified(payload, "koch.nim"): 503 | run(nimLoc, "bin" / nimBin.exe, "c", "koch.nim") 504 | backupBranchSpecificBin(nimLoc, "koch", payload["ref"].str[11 .. ^1]) 505 | 506 | # Bootstrap! 507 | run(nimLoc, "koch".exe, "boot") 508 | run(nimLoc, "koch".exe, "boot", "-d:release") 509 | backupBranchSpecificBin(nimLoc / "bin", nimBin, payload["ref"].str[11 .. ^1]) 510 | 511 | proc archiveNimrod(platform, commitPath, commitHash, websiteLoc, 512 | nimLoc, nimBin, rootZipLoc: string): string = 513 | ## Zips up the build. 514 | ## Returns the full absolute path to where the zipped file resides. 515 | 516 | # Set +x on nimrod binary 517 | setFilePermissions(nimLoc / "bin" / nimBin.exe, webFP) 518 | let zipPath = rootZipLoc / commitPath 519 | let zipFile = addFileExt(commitPath, "zip") 520 | 521 | dCreateDir(zipPath) 522 | copyForArchive(nimLoc, zipPath, nimBin) 523 | 524 | # Remove the .zip in case it already exists... 525 | if existsFile(rootZipLoc / zipFile): removeFile(rootZipLoc / zipFile) 526 | when defined(windows): 527 | run(rootZipLoc, findExe("7za"), "a", "-tzip", 528 | zipFile.extractFilename, commitPath) 529 | else: 530 | run(rootZipLoc, findExe("zip"), "-r", zipFile, commitPath) 531 | 532 | # Copy the .zip file 533 | var zipFinalPath = addFileExt(makeZipPath(platform, commitHash), "zip") 534 | # Remove the pre-zipped folder with the binaries. 535 | dRemoveDir(zipPath) 536 | # Move the .zip file to the website 537 | when defined(windows): 538 | dMoveFile(rootZipLoc / zipFile.extractFilename, 539 | websiteLoc / "commits" / zipFinalPath) 540 | else: 541 | dMoveFile(rootZipLoc / zipFile, websiteLoc / "commits" / zipFinalPath) 542 | # Remove the original .zip file 543 | dRemoveFile(rootZipLoc / zipFile) 544 | 545 | result = websiteLoc / "commits" / zipFinalPath 546 | 547 | proc uploadFile(ftpAddr: string, ftpPort: TPort, user, pass, workDir, 548 | uploadDir, file, destFile: string) = 549 | 550 | proc handleEvent(f: PAsyncFTPClient, ev: TFTPEvent) = 551 | case ev.typ 552 | of EvStore: 553 | f.chmod(destFile, webFP) 554 | f.close() 555 | of EvTransferProgress: 556 | hubSendFTPUploadSpeed(ev.speed.float / 1024.0) 557 | else: assert false 558 | 559 | try: 560 | var ftpc = asyncFTPClient(ftpAddr, ftpPort, user, pass, handleEvent) 561 | echo("Connecting to ftp://" & user & "@" & ftpAddr & ":" & $ftpPort) 562 | ftpc.connect() 563 | assert ftpc.pwd().startsWith("/home/" & user) # /home/nimrod 564 | ftpc.cd(workDir) 565 | echo("FTP: Work dir is " & workDir) 566 | echo("FTP: Creating " & uploadDir) 567 | try: ftpc.createDir(uploadDir, true) 568 | except EInvalidReply: nil # TODO: Check properly whether the folder exists 569 | 570 | ftpc.chmod(uploadDir, webFP) 571 | ftpc.cd(uploadDir) 572 | echo("FTP: Work dir is " & ftpc.pwd()) 573 | var disp = newDispatcher() 574 | disp.register(ftpc) 575 | echo("FTP: Uploading ", file, " to ", destFile) 576 | ftpc.store(file, destFile, async = true) 577 | while true: 578 | if not disp.poll(5000): break 579 | 580 | except EInvalidReply: raise newException(EBuildEnd, getCurrentExceptionMsg()) 581 | 582 | proc nimTest(commitPath, nimLoc, websiteLoc: string): string = 583 | ## Runs the tester, returns the full absolute path to where the tests 584 | ## have been saved. 585 | result = websiteLoc / "commits" / commitPath / "testresults.html" 586 | run({"PATH": changeNimrodInPATH(nimLoc / "bin")}.newStringTable(), 587 | nimLoc, "koch".exe, "tests") 588 | # Copy the testresults.html file. 589 | dCreateDir(websiteLoc / "commits" / commitPath) 590 | setFilePermissions(websiteLoc / "commits" / commitPath, 591 | webFP) 592 | dCopyFile(nimLoc / "testresults.html", result) 593 | 594 | proc bootstrapTmpl(dummy: int) {.thread.} = 595 | ## Template for a full bootstrap. 596 | buildTmpl(info): 597 | let cfg = info.cfg 598 | let commitHash = info.payload["after"].str 599 | let commitBranch = info.payload["ref"].str[11 .. ^1] 600 | let commitPath = makeCommitPath(cfg.platform, commitHash) 601 | hubSendBuildStart(commitHash, commitBranch) 602 | hubSendJobUpdate(jBuild) 603 | 604 | # GIT 605 | setGIT(info.payload, cfg.nimLoc) 606 | 607 | # Bootstrap 608 | nimBootstrap(info.payload, cfg.nimLoc, cfg.csourceExtraBuildArgs) 609 | 610 | var buildZipFilePath = archiveNimrod(cfg.platform, commitPath, commitHash, 611 | cfg.websiteLoc, cfg.nimLoc, info.payload["nimBin"].str, cfg.zipLoc) 612 | 613 | # --- Upload zip with build --- 614 | if cfg.hubAddr != "127.0.0.1": 615 | uploadFile(cfg.hubAddr, cfg.ftpPort, cfg.ftpUser, 616 | cfg.ftpPass, 617 | cfg.ftpUploadDir / "commits", cfg.platform, # TODO: Make sure user doesn't add the "commits" in the config. 618 | buildZipFilePath, 619 | buildZipFilePath.extractFilename) 620 | 621 | hubSendBuildSuccess() 622 | hubSendJobUpdate(jTest) 623 | var testResultsPath = nimTest(commitPath, cfg.nimLoc, cfg.websiteLoc) 624 | 625 | # --- Upload testresults.html --- 626 | if cfg.hubAddr != "127.0.0.1": 627 | uploadFile(cfg.hubAddr, cfg.ftpPort, cfg.ftpUser, 628 | cfg.ftpPass, cfg.ftpUploadDir / "commits", commitPath, 629 | testResultsPath, "testresults.html") 630 | var (total, passed, skipped, failed, diff, results) = 631 | tallyTestResults(cfg.nimLoc / "testresults.json") 632 | hubSendBuildTestSuccess(total, passed, skipped, failed, diff, results) 633 | 634 | # --- Start of doc gen --- 635 | # Create the upload directory and the docs directory on the website 636 | if cfg.docgen: 637 | hubSendJobUpdate(jDocGen) 638 | dCreateDir(cfg.nimLoc / "web" / "upload") 639 | dCreateDir(cfg.websiteLoc / "docs") 640 | run({"PATH": changeNimrodInPATH(cfg.nimLoc / "bin")}.newStringTable(), 641 | cfg.nimLoc, "koch", "web") 642 | # Copy all the docs to the website. 643 | dCopyDir(cfg.nimLoc / "web" / "upload", cfg.websiteLoc / "docs") 644 | 645 | hubSendBuildSuccess() 646 | if cfg.innoSetupGen: 647 | # We want docs to be generated for inno setup, so that the setup file 648 | # includes them. 649 | hubSendJobUpdate(jDocGen) 650 | run({"PATH": changeNimrodInPATH(cfg.nimLoc / "bin")}.newStringTable(), 651 | cfg.nimLoc, "koch", "web") 652 | hubSendBuildSuccess() 653 | 654 | 655 | # --- Start of csources gen --- 656 | if cfg.csourceGen: 657 | # Rename the build directory so that the csources from the git repo aren't 658 | # overwritten 659 | hubSendJobUpdate(jCSrcGen) 660 | dMoveDir(cfg.nimLoc / "build", cfg.nimLoc / "build_old") 661 | dCreateDir(cfg.nimLoc / "build") 662 | 663 | run({"PATH": changeNimrodInPATH(cfg.nimLoc / "bin")}.newStringTable(), 664 | cfg.nimLoc, "koch", "csource") 665 | 666 | # Zip up the csources. 667 | # -- Move the build directory to the zip location 668 | let csourcesPath = makeZipPath(cfg.platform, commitHash) & "_csources" 669 | var csourcesZipFile = csourcesPath.addFileExt("zip") 670 | dMoveDir(cfg.nimLoc / "build", cfg.zipLoc / csourcesPath) 671 | # -- Move `build_old` to where it was previously. 672 | dMoveDir(cfg.nimLoc / "build_old", cfg.nimLoc / "build") 673 | # -- License 674 | dCopyFile(cfg.nimLoc / "copying.txt", 675 | cfg.zipLoc / csourcesPath / "copying.txt") 676 | 677 | writeFile(cfg.zipLoc / csourcesPath / "readme2.txt", buildReadme) 678 | # -- ZIP! 679 | if existsFile(cfg.zipLoc / csourcesZipFile): 680 | removeFile(cfg.zipLoc / csourcesZipFile) 681 | when defined(windows): 682 | echo("Not implemented") 683 | doAssert(false) 684 | run(cfg.zipLoc, findexe("zip"), "-r", csourcesZipFile, csourcesPath) 685 | # -- Remove the directory which was zipped 686 | dRemoveDir(cfg.zipLoc / csourcesPath) 687 | # -- Move the .zip file 688 | dMoveFile(cfg.zipLoc / csourcesZipFile, 689 | cfg.websiteLoc / "commits" / csourcesZipFile) 690 | 691 | hubSendBuildSuccess() 692 | 693 | # --- Start of inno setup gen --- 694 | if cfg.innoSetupGen: 695 | hubSendJobUpdate(jInnoSetup) 696 | run({"PATH": changeNimrodInPATH(cfg.nimLoc / "bin")}.newStringTable(), 697 | cfg.nimLoc, "koch", "inno", "-d:release") 698 | if cfg.hubAddr != "127.0.0.1": 699 | uploadFile(cfg.hubAddr, cfg.ftpPort, cfg.ftpUser, 700 | cfg.ftpPass, cfg.ftpUploadDir / "commits", cfg.platform, 701 | cfg.nimLoc / "build" / "nimrod_setup.exe", 702 | makeInnoSetupPath(commitHash)) 703 | hubSendBuildSuccess() 704 | 705 | proc stopBuild(state: PState) = 706 | ## Terminates a build 707 | # TODO: Send a message to the website, make it record it to the database 708 | # as "terminated". 709 | if state.building: 710 | # Send the termination command first. 711 | threadCommandChan.send(TThreadCommand(kind: BuildTerminate)) 712 | 713 | # Simply terminate the currently running process, should hopefully work. 714 | if state.buildJob.p != nil: 715 | echo("Terminating build") 716 | state.buildJob.p.terminate() 717 | 718 | proc beginBuild(state: PState) = 719 | ## This procedure starts the process of building nimrod. 720 | 721 | # First make sure to stop any currently running process. 722 | state.stopBuild() 723 | 724 | # Tell the thread to start a build. 725 | state.building = true 726 | let thrCmd = TThreadCommand(kind: BuildStart, 727 | payload: state.buildJob.payload, cfg: state.cfg) 728 | threadCommandChan.send(thrCmd) 729 | 730 | proc pollBuild(state: PState) = 731 | ## This is called from the main loop; it checks whether the bootstrap 732 | ## thread has sent any messages through the channel and it then processes 733 | ## the messages. 734 | let msgCount = hubChan.peek() 735 | if msgCount > 0: 736 | for i in 0..msgCount-1: 737 | var msg = hubChan.recv() 738 | case msg.kind 739 | of ProcessStart: 740 | #p: PProcess 741 | state.buildJob.p = msg.p 742 | of ProcessExit: 743 | state.buildJob.p = nil 744 | of HubMsg: 745 | state.sock.send(msg.msg) 746 | of BuildEnd: 747 | state.building = false 748 | 749 | # Communication 750 | proc parseReply(line: string, expect: string): bool = 751 | var jsonDoc = parseJson(line) 752 | return jsonDoc["reply"].str == expect 753 | 754 | proc hubConnect(state: PState, reconnect: bool) {.gcsafe.} 755 | proc handleConnect(s: PAsyncSocket, state: PState) {.gcsafe.} = 756 | try: 757 | # Send greeting 758 | var obj = newJObject() 759 | obj["name"] = newJString("builder") 760 | obj["platform"] = newJString(state.cfg.platform) 761 | obj["version"] = %"1" 762 | if state.cfg.hubPass != "": obj["pass"] = newJString(state.cfg.hubPass) 763 | state.sock.send($obj & "\c\L") 764 | # Wait for reply. 765 | var readSocks = @[state.sock.getSocket] 766 | # TODO: Don't use select here. Just use readLine with a timeout. 767 | if select(readSocks, 1500) == 1: 768 | var line = "" 769 | if not state.sock.readLine(line): 770 | raise newException(EInvalidValue, "recvLine failed.") 771 | if not parseReply(line, "OK"): 772 | raise newException(EInvalidValue, "Incorrect welcome message from hub") 773 | 774 | echo("The hub accepted me!") 775 | 776 | if state.cfg.requestNewest and not state.reconnecting: 777 | echo("Requesting newest commit.") 778 | var req = newJObject() 779 | req["latestCommit"] = newJNull() 780 | state.sock.send($req & "\c\L") 781 | 782 | else: 783 | raise newException(EInvalidValue, 784 | "Hub didn't accept me. Waited 1.5 seconds.") 785 | except EOS, EInvalidValue: 786 | echo(getCurrentExceptionMsg()) 787 | s.close() 788 | echo("Waiting 5 seconds...") 789 | sleep(5000) 790 | try: hubConnect(state, true) except EOS: echo(getCurrentExceptionMsg()) 791 | 792 | proc handleHubMessage(s: PAsyncSocket, state: PState) {.gcsafe.} 793 | proc hubConnect(state: PState, reconnect: bool) = 794 | state.sock = asyncSocket() 795 | state.sock.handleConnect = 796 | proc (s: PAsyncSocket) {.gcsafe.} = 797 | handleConnect(s, state) 798 | state.sock.handleRead = 799 | proc (s: PAsyncSocket) {.gcsafe.} = 800 | handleHubMessage(s, state) 801 | state.reconnecting = reconnect 802 | state.sock.connect(state.cfg.hubAddr, TPort(state.cfg.hubPort)) 803 | state.dispatcher.register(state.sock) 804 | 805 | proc open(configPath: string): PState = 806 | var cres: PState 807 | cres = defaultState() 808 | # Get config 809 | parseConfig(cres, configPath) 810 | if not existsDir(cres.cfg.nimLoc): 811 | quit(cres.cfg.nimLoc & " does not exist!", QuitFailure) 812 | 813 | # Init dispatcher 814 | cres.dispatcher = newDispatcher() 815 | 816 | # Connect to the hub 817 | try: cres.hubConnect(false) 818 | except EOS: 819 | echo("Could not connect to hub: " & getCurrentExceptionMsg()) 820 | quit(QuitFailure) 821 | 822 | # Open log file 823 | cres.logFile = open(cres.cfg.logLoc, fmAppend) 824 | 825 | # Init job 826 | cres.buildJob = initJob() 827 | 828 | # Start build thread 829 | createThread(cres.buildThread, bootstrapTmpl, 0) 830 | 831 | result = cres 832 | 833 | proc initJob(payload: PJsonNode): TJob = 834 | result.payload = payload 835 | 836 | proc hubDisconnect(state: PState) = 837 | state.sock.close() 838 | 839 | state.lastMsgTime = epochTime() 840 | state.pinged = -1.0 841 | 842 | proc parseMessage(state: PState, line: string) = 843 | echo("Got message from hub: ", line) 844 | state.lastMsgTime = epochTime() 845 | var json = parseJson(line) 846 | if json.hasKey("payload"): 847 | if json["rebuild"].bval: 848 | # This commit has already been built. We don't get a full payload as 849 | # it is not stored. 850 | # Because the build process depends on "after" that is all that is 851 | # needed. 852 | assert(json["payload"].hasKey("after")) 853 | state.buildJob = initJob(json["payload"]) 854 | echo("Re-bootstrapping!") 855 | state.beginBuild() 856 | else: 857 | # This should be a message from the "github" module 858 | # The payload object should have a `after` string. 859 | assert(json["payload"].hasKey("after")) 860 | state.buildJob = initJob(json["payload"]) 861 | echo("Bootstrapping!") 862 | state.beginBuild() 863 | 864 | elif json.hasKey("ping"): 865 | # Website is making sure that the connection is alive. 866 | # All we do is change the "ping" to "pong" and reply. 867 | json["pong"] = json["ping"] 868 | json.delete("ping") 869 | state.sock.send($json & "\c\L") 870 | echo("Replying to Ping") 871 | 872 | elif json.hasKey("pong"): 873 | # Website replied. Connection is still alive. 874 | state.pinged = -1.0 875 | echo("Hub replied to PING. Still connected") 876 | 877 | elif json.hasKey("fatal"): 878 | # Fatal error occurred in the website. We must exit. 879 | echo("FATAL ERROR") 880 | echo(json["fatal"]) 881 | hubDisconnect(state) 882 | quit(QuitFailure) 883 | 884 | elif json.hasKey("do"): 885 | case json["do"].str 886 | of "stop": 887 | ## Terminate build 888 | state.stopBuild() 889 | else: 890 | echo("[FATAL] Don't understand message from hub") 891 | assert false 892 | 893 | proc reconnect(state: PState) = 894 | state.hubDisconnect() 895 | echo("Waiting 5 seconds before reconnecting...") 896 | sleep(5000) 897 | try: state.hubConnect(true) 898 | except EOS: 899 | echo("Could not reconnect: ", getCurrentExceptionMsg()) 900 | reconnect(state) 901 | 902 | proc handleHubMessage(s: PAsyncSocket, state: PState) = 903 | try: 904 | var line = "" 905 | if state.sock.readLine(line): 906 | if line != "": 907 | state.parseMessage(line) 908 | else: 909 | echo("Disconnected from hub (recvLine returned \"\"): ", 910 | osErrorMsg(osLastError())) 911 | reconnect(state) 912 | except EOS: 913 | echo("Disconnected from hub: ", getCurrentExceptionMsg()) 914 | reconnect(state) 915 | 916 | proc checkTimeout(state: PState) = 917 | const timeoutSeconds = 110.0 # If no message received in that long, ping the server. 918 | 919 | if state.cfg.hubAddr != "127.0.0.1": 920 | # Check how long ago the last message was sent. 921 | if state.pinged == -1.0: 922 | if epochTime() - state.lastMsgTime >= timeoutSeconds: 923 | echo("We seem to be timing out! PINGing server.") 924 | var jsonObject = newJObject() 925 | jsonObject["ping"] = newJString(formatFloat(epochTime())) 926 | try: 927 | state.sock.send($jsonObject & "\c\L") 928 | except EOS: 929 | echo("Disconnected from server due to: ", getCurrentExceptionMsg()) 930 | reconnect(state) 931 | return 932 | 933 | state.pinged = epochTime() 934 | 935 | else: 936 | if epochTime() - state.pinged >= 5.0: # 5 seconds 937 | echo("Server has not replied with a pong in 5 seconds.") 938 | # TODO: What happens if the builder gets disconnected in the middle of a 939 | # build? Maybe implement restoration of that. 940 | reconnect(state) 941 | 942 | proc showHelp() = 943 | const help = """Usage: builder [options] configFile 944 | -h --help Show this help message 945 | -v --version Show version 946 | """ 947 | quit(help, QuitSuccess) 948 | 949 | proc showVersion() = 950 | const version = """builder $1 - built on $2 951 | This software is part of the nimbuild website.""" 952 | quit(version % [builderVer, CompileDate & " " & CompileTime], QuitSuccess) 953 | 954 | proc parseArgs(): string = 955 | result = "" 956 | for kind, key, val in getopt(): 957 | case kind 958 | of cmdArgument: 959 | result = key 960 | of cmdLongOption, cmdShortOption: 961 | case key 962 | of "help", "h": showHelp() 963 | of "version", "v": showVersion() 964 | of cmdEnd: assert(false) # cannot happen 965 | if result == "": 966 | showHelp() 967 | 968 | proc createFolders(state: PState) = 969 | if not existsDir(state.cfg.websiteLoc / "commits" / state.cfg.platform): 970 | dCreateDir(state.cfg.websiteLoc / "commits" / state.cfg.platform) 971 | 972 | proc checkDepends() = 973 | when defined(windows): 974 | if findExe("7za") == "": 975 | quit("Could not find 7za for archiving.") 976 | else: 977 | if findExe("zip") == "": 978 | quit("Could not find zip for archiving.") 979 | 980 | when isMainModule: 981 | echo("Started builder: built at ", CompileDate, " ", CompileTime) 982 | 983 | var state = builder.open(parseArgs()) 984 | checkDepends() 985 | createFolders(state) 986 | while true: 987 | discard state.dispatcher.poll() 988 | 989 | state.pollBuild() 990 | 991 | state.checkTimeout() 992 | 993 | 994 | -------------------------------------------------------------------------------- /src/website.nim: -------------------------------------------------------------------------------- 1 | ## This is the SCGI Website and the hub. 2 | import 3 | sockets, asyncio, json, strutils, os, scgi, strtabs, times, streams, parsecfg, 4 | algorithm, tables, base64 5 | import htmlgen except del 6 | import types, db, htmlhelp 7 | from httpclient import nil 8 | from net import nil 9 | 10 | import asyncdispatch except newDispatcher, Port 11 | 12 | import jester 13 | 14 | type 15 | TBQCommit = object 16 | hash: string 17 | branch: string 18 | payload: PJsonNode 19 | 20 | PState = ref TState 21 | TState = object of TObject 22 | dispatcher: PDispatcher 23 | sock: PAsyncSocket ## Hub server socket. All modules connect to this. 24 | req: Request 25 | jesterSettings: jester.Settings 26 | modules: seq[TModule] 27 | database: TDb 28 | platforms: TTable[string, TStatus] 29 | buildQueue: TTable[string, seq[TBQCommit]] # Platform, [(hash, branch, payload)] 30 | password: string ## The password that foreign modules need to be accepted. 31 | bindAddr: string 32 | bindPort: int 33 | scgiPort: int 34 | redisPort: int 35 | isHttp: bool 36 | ircLogsPath: string 37 | packagesJson: string # List of babel packages from nimrod-code/packages 38 | 39 | TModuleStatus = enum 40 | MSConnecting, ## Module connected, but has not sent the greeting. 41 | MSConnected ## Module is ready to do work. 42 | 43 | TModule = object 44 | name: string 45 | sock: PAsyncSocket ## Client socket 46 | status: TModuleStatus 47 | platform: string 48 | lastPong: float 49 | pinged: bool # whether we are waiting for a pong from the module. 50 | ping: float # in seconds 51 | ip: string # IP address this module is connecting from. 52 | delegID: PDelegate 53 | logFile: TFile # Only applicable to a module of type builder. 54 | 55 | proc parseConfig(state: PState, path: string) = 56 | var f = newFileStream(path, fmRead) 57 | if f != nil: 58 | var p: TCfgParser 59 | open(p, f, path) 60 | var count = 0 61 | while true: 62 | var n = next(p) 63 | case n.kind 64 | of cfgEof: 65 | break 66 | of cfgSectionStart: 67 | raise newException(EInvalidValue, "Unknown section: " & n.section) 68 | of cfgKeyValuePair, cfgOption: 69 | case normalize(n.key) 70 | of "bindaddr": 71 | state.bindAddr = n.value 72 | inc(count) 73 | of "bindport": 74 | state.bindPort = parseInt(n.value) 75 | inc(count) 76 | of "scgiport": 77 | state.scgiPort = parseInt(n.value) 78 | inc(count) 79 | of "redisport": 80 | state.redisPort = parseInt(n.value) 81 | inc(count) 82 | of "password": 83 | state.password = n.value 84 | inc(count) 85 | of "ishttp": 86 | state.isHttp = n.value.normalize == "true" 87 | inc(count) 88 | of "irclogspath": 89 | state.ircLogsPath = n.value 90 | inc(count) 91 | of cfgError: 92 | raise newException(EInvalidValue, "Configuration parse error: " & n.msg) 93 | if count < 7: 94 | quit("Not all settings have been specified in the .ini file") 95 | close(p) 96 | else: 97 | quit("Cannot open configuration file: " & path) 98 | 99 | proc handleAccept(s: PAsyncSocket, state: PState) {.gcsafe.} 100 | proc open(configPath: string): PState = 101 | var cres: PState 102 | new(cres) 103 | parseConfig(cres, configPath) 104 | cres.dispatcher = newDispatcher() 105 | 106 | cres.sock = asyncSocket() 107 | cres.sock.bindAddr(TPort(cres.bindPort), cres.bindAddr) 108 | cres.sock.listen() 109 | cres.sock.handleAccept = proc (s: PAsyncSocket) = handleAccept(s, cres) 110 | cres.modules = @[] 111 | cres.platforms = initTable[string, TStatus]() 112 | cres.buildQueue = initTable[string, seq[TBQCommit]]() 113 | 114 | cres.dispatcher.register(cres.sock) 115 | 116 | # Connect to the database 117 | try: 118 | cres.database = db.open("localhost", TPort(cres.redisPort)) 119 | except EOS: 120 | quit("Couldn't connect to redis: " & getCurrentExceptionMsg()) 121 | 122 | result = cres 123 | 124 | # Modules 125 | 126 | proc contains(modules: seq[TModule], name: string): bool = 127 | for i in items(modules): 128 | if i.name == name: return true 129 | 130 | return false 131 | 132 | proc handleModuleMsg(s: PAsyncSocket, arg: PObject) {.gcsafe.} 133 | proc addModule(state: PState, client: PAsyncSocket, IPAddr: string) = 134 | var module: TModule 135 | module.sock = client 136 | module.ip = IPAddr 137 | module.lastPong = epochTime() 138 | module.pinged = false 139 | module.status = MSConnecting 140 | echo(IPAddr, " connected.") 141 | 142 | # Add this module to the dispatcher. 143 | client.handleRead = proc (s:PAsyncSocket) = handleModuleMsg(s, state) 144 | module.delegID = state.dispatcher.register(client) 145 | 146 | state.modules.add(module) 147 | 148 | proc findBuilderModule(state: PState, platf: string, module: var TModule): bool = 149 | result = false 150 | for i in state.modules: 151 | if i.name == "builder" and i.platform == platf: 152 | module = i 153 | return true 154 | 155 | proc mGetBuilderModule(state: PState, platf: string): var TModule = 156 | for i in 0..state.modules.len-1: 157 | if state.modules[i].name == "builder" and state.modules[i].platform == platf: 158 | return state.modules[i] 159 | raise newException(EInvalidValue, "Platform could not be found.") 160 | 161 | proc parseGreeting(state: PState, m: var TModule, line: string, errMsg: var string): bool = 162 | # { "name": "modulename", "version": "1" } 163 | # optional params: settings 164 | var json: PJsonNode 165 | try: 166 | json = parseJson(line) 167 | except EJsonParsingError: 168 | return false 169 | 170 | if m.ip != "127.0.0.1": 171 | # Check for password 172 | var fail = true 173 | if json.existsKey("pass"): 174 | if json["pass"].str == state.password: 175 | fail = false 176 | else: 177 | echo("Got incorrect password: ", json["pass"].str) 178 | errMsg = "Invalid password" 179 | 180 | if fail: return false 181 | 182 | if not (json.existsKey("name") and json.existsKey("platform")): 183 | errMsg = "Invalid greeting." 184 | return false 185 | if not json.existsKey("version"): 186 | errMsg = "Required version field missing." 187 | return false 188 | else: 189 | if json["version"].str != "1": 190 | errMsg = "Invalid version." 191 | return false 192 | 193 | m.name = json["name"].str 194 | m.platform = json["platform"].str 195 | 196 | # Only add this module platform to platforms if it's a `builder`, and 197 | # if platform doesn't already exist. 198 | if m.name == "builder": 199 | if not state.platforms.hasKey(m.platform): 200 | state.platforms[m.platform] = initStatus() 201 | else: 202 | echo("Platform(", m.platform, ") already exists.") 203 | errMsg = "This platform already exists." 204 | return false 205 | 206 | m.status = MSConnected 207 | 208 | return true 209 | 210 | proc uniqueMName(module: TModule): string = 211 | result = "" 212 | case module.status 213 | of MSConnected: 214 | result.add module.name 215 | result.add "-" 216 | result.add module.platform 217 | result.add "(" & module.ip & ")" 218 | of MSConnecting: 219 | result.add "Unknown (" & module.ip & ")" 220 | 221 | proc IRCAnnounce(state: PState, msg: string, important = false) = 222 | if "irc" in state.modules: 223 | for module in items(state.modules): 224 | if module.name == "irc": 225 | let json = %{"announce": %msg, "important": %important} 226 | module.sock.send($json & "\c\L") 227 | 228 | proc remove(state: PState, module: TModule) = 229 | for i in 0..len(state.modules)-1: 230 | var m = state.modules[i] 231 | if m.name == module.name and 232 | m.platform == module.platform and 233 | m.ip == module.ip: 234 | state.dispatcher.unregister(state.modules[i].delegID) 235 | state.modules[i].sock.close() 236 | echo(uniqueMName(state.modules[i]), " disconnected.") 237 | if m.name != "irc": 238 | IRCAnnounce(state, uniqueMName(state.modules[i]) & " disconnected.", true) 239 | # if module is a builder remove it from platforms. 240 | if m.name == "builder": 241 | state.platforms.del(m.platform) 242 | 243 | state.modules.delete(i) 244 | return 245 | 246 | proc setJob(state: PState, p: string, job: TBuilderJob) = 247 | var s = state.platforms[p] 248 | s.isInProgress = true 249 | s.desc = "" 250 | s.jobs[job] = jInProgress 251 | state.platforms[p] = s 252 | 253 | proc setResult(state: PState, p: string, res: TResult, detail: string) = 254 | var s = state.platforms[p] 255 | let job = jobInProgress(s) 256 | s.isInProgress = false 257 | s.jobs[job] = if res == Success: jSuccess else: jFail 258 | s.desc = detail 259 | state.platforms[p] = s 260 | 261 | proc setDesc(state: PState, p: string, desc: string) = 262 | var s = state.platforms[p] 263 | assert s.isInProgress 264 | s.desc = desc 265 | state.platforms[p] = s 266 | 267 | proc writeBuildSpecificLogs(state: PState, platf: string, line: string) = 268 | let m = mGetBuilderModule(state, platf) 269 | if m.logFile == nil: 270 | # TODO: This will happen if a builder reconnects during a build, 271 | # the builder should send some sort of command to reopen the logFile upon 272 | # reconnection 273 | echo("Warning: Could not write to logfile as it is nil.") 274 | return 275 | m.logFile.write(line & "\n") 276 | 277 | proc checkBuilderQueue(state: PState, platform: string) = 278 | ## Checks builder queue and sends a message to the builder immediatelly. 279 | if state.buildQueue.hasKey(platform) and 280 | state.buildQueue[platform].len != 0: 281 | let cm = state.buildQueue.mget(platform).pop() 282 | if not state.database.platformExists(cm.payload["payload"]["after"].str, 283 | platform): 284 | state.database.addPlatform(cm.payload["payload"]["after"].str, 285 | platform) 286 | let json = %{"payload": cm.payload["payload"], "rebuild": %false} 287 | var builder: TModule 288 | doAssert findBuilderModule(state, platform, builder) 289 | builder.sock.send($json & "\c\L") 290 | 291 | # TODO: Instead of using assertions provide a function which checks whether the 292 | # key exists and throw an exception if it doesn't. 293 | 294 | proc createGist(filename, content: string, description = "Nimbuild gist"): string {.raises: [].} = 295 | except: return "An error occurred creating gist: " & getCurrentExceptionMsg() 296 | var body = 297 | %{ "description": %description, 298 | "public": %false, 299 | "files": %{ 300 | filename: %{ 301 | "content": %content 302 | } 303 | } 304 | } 305 | let resp = httpclient.post("https://api.github.com/gists", body = $body, 306 | timeout = 2000) 307 | if resp.status.startsWith("201"): 308 | let respJson = resp.body.parseJSON() 309 | return respJson["html_url"].str 310 | else: 311 | return "Gist creation failed. Got status: " & resp.status 312 | 313 | proc refreshPackagesJson(state: PState) = 314 | let resp = httpclient.get("https://raw.githubusercontent.com/nimrod-code/" & 315 | "packages/master/packages.json", timeout = 2000) 316 | if resp.status.startsWith("200"): 317 | try: 318 | var test = parseJson(resp.body) 319 | state.packagesJson = base64.encode(resp.body) 320 | except: 321 | echo("Got incorrect packages.json, not saving.") 322 | echo(getCurrentExceptionMsg()) 323 | if state.packagesJson == nil: raise 324 | else: 325 | echo("Could not retrieve packages.json.") 326 | 327 | proc parseMessage(state: PState, mIndex: int, line: string) = 328 | var json = parseJson(line) 329 | var m = state.modules[mIndex] 330 | if json.existsKey("job"): 331 | # { job: TBuilderJob} 332 | # Change of a builder job. 333 | setJob(state, m.platform, TBuilderJob(json["job"].num)) 334 | 335 | elif json.existsKey("result"): 336 | let result = TResult(json["result"].num) 337 | let platf = state.platforms[m.platform] 338 | 339 | proc IRCInfo(): string = 340 | "[$1 $2 $3]" % [m.platform, platf.hash[0..11], platf.branch] 341 | 342 | let currentJob = jobInProgress(platf) 343 | case currentJob 344 | of jBuild: 345 | if result == Success: 346 | state.database.updateProperty(platf.hash, m.platform, "buildResult", 347 | $int(bSuccess)) 348 | state.IRCAnnounce(IRCInfo() & " Build OK.") 349 | else: 350 | assert json.existsKey("detail") 351 | state.database.updateProperty(platf.hash, m.platform, "buildResult", 352 | $int(bFail)) 353 | state.database.updateProperty(platf.hash, m.platform, "failReason", 354 | json["detail"].str) 355 | # This implies that the tests failed too. If we leave this as unknown, 356 | # the website will show the 'progress.gif' image, which we don't want. 357 | state.database.updateProperty(platf.hash, m.platform, 358 | "testResult", $int(tFail)) 359 | var important = false 360 | if platf.branch == "master": 361 | important = true 362 | state.IRCAnnounce("$1 Build failed: $2" % 363 | [IRCInfo(), json["detail"].str], important) 364 | of jTest: 365 | if result == Success: 366 | assert(json.existsKey("total")) 367 | assert(json.existsKey("passed")) 368 | assert(json.existsKey("skipped")) 369 | assert(json.existsKey("failed")) 370 | state.database.updateProperty(platf.hash, m.platform, 371 | "testResult", $int(tSuccess)) 372 | state.database.updateProperty(platf.hash, m.platform, 373 | "total", $json["total"].num) 374 | state.database.updateProperty(platf.hash, m.platform, 375 | "passed", $json["passed"].num) 376 | state.database.updateProperty(platf.hash, m.platform, 377 | "skipped", $json["skipped"].num) 378 | state.database.updateProperty(platf.hash, m.platform, 379 | "failed", $json["failed"].num) 380 | state.IRCAnnounce("$1 Test results: $2/$3." % 381 | [IRCInfo(), $json["passed"].num, $json["total"].num]) 382 | 383 | # Diff functionality 384 | if json.hasKey("diff") and json["diff"].kind == JArray: 385 | var succeedNow = "" 386 | var succeedNowCount = 0 387 | var failNow = "" 388 | var failNowCount = 0 389 | for i in 0 .. " & json["line"].str) 473 | of bFtpUploadSpeed: 474 | state.platforms.mget(m.platform).FTPSpeed = json["speed"].fnum 475 | of bEnd: 476 | # Close file 477 | state.modules[mIndex].logFile.close() 478 | 479 | # Build ended. Check queue for more builds awaiting. 480 | checkBuilderQueue(state, m.platform) 481 | of bStart: 482 | # Build started, open log file. 483 | assert json.existsKey("hash") 484 | assert json.existsKey("branch") 485 | let commitHash = json["hash"].str 486 | let commitBranch = json["branch"].str 487 | state.platforms.mget(m.platform).hash = commitHash 488 | state.platforms.mget(m.platform).branch = commitBranch 489 | let commitPath = state.jesterSettings.staticDir / "commits" / 490 | makeCommitPath(m.platform, commitHash) 491 | if not existsDir(commitPath.parentDir): 492 | createDir(commitPath.parentDir) 493 | if not existsDir(commitPath): 494 | createDir(commitPath) 495 | let logFilepath = state.jesterSettings.staticDir / "commits" / 496 | makeCommitPath(m.platform, commitHash) / "logs.txt" 497 | state.modules[mIndex].logFile = open(logFilepath, fmWrite) 498 | 499 | elif json.existsKey("payload"): 500 | # { "payload": { .. } } 501 | # Check if this is the Nim repo. 502 | if "nim-lang/nim" in json["payload"]["repository"]["url"].str.toLower(): 503 | # Check if the commit exists. 504 | if not state.database.commitExists(json["payload"]["after"].str): 505 | # Get the branch. 506 | let branch = json["payload"]["ref"].str[11 .. ^1] 507 | var commits = json["payload"]["commits"] 508 | var latestCommit = commits[commits.len-1] 509 | # Add commit to database 510 | state.database.addCommit(json["payload"]["after"].str, 511 | latestCommit["message"].str, 512 | latestCommit["author"]["username"].str, 513 | branch) 514 | 515 | # Send this message to the "builder" modules 516 | for module in items(state.modules): 517 | if module.name == "builder": 518 | # Check build queue. 519 | var toBuildQueue = false 520 | if state.buildQueue.hasKey(module.platform): 521 | toBuildQueue = state.buildQueue[module.platform].len > 0 522 | 523 | # Check if builder is currently building. 524 | if state.platforms[module.platform].isInProgress: 525 | toBuildQueue = true 526 | 527 | if not toBuildQueue: 528 | # Send immediately. 529 | state.database.addPlatform(json["payload"]["after"].str, 530 | module.platform) 531 | 532 | # Add "rebuild" flag. 533 | json["rebuild"] = newJBool(false) 534 | 535 | module.sock.send($json & "\c\L") 536 | else: 537 | var cm: TBQCommit 538 | cm.hash = json["payload"]["after"].str 539 | cm.branch = branch 540 | cm.payload = json 541 | if not state.buildQueue.hasKey(module.platform): 542 | state.buildQueue[module.platform] = @[] 543 | state.buildQueue.mget(module.platform).add(cm) 544 | 545 | else: 546 | echo("Commit already exists. Not rebuilding.") 547 | elif "nimrod-code/packages" in 548 | json["payload"]["repository"]["url"].str.toLower(): 549 | state.refreshPackagesJson() 550 | else: 551 | echo("Repo is not Nim. Got: " & 552 | json["payload"]["repository"]["url"].str) 553 | 554 | # Send this message to the "irc" module. 555 | if "irc" in state.modules: 556 | for module in items(state.modules): 557 | if module.name == "irc": 558 | module.sock.send($json & "\c\L") 559 | 560 | elif json.existsKey("rebuild"): 561 | # { "rebuild": "hash" } 562 | # TODO: Is this ever used? 563 | var hash = json["rebuild"].str 564 | var reply = newJObject() 565 | if state.database.commitExists(hash, true): 566 | # You can only rebuild the newest commit. (For now, TODO?) 567 | var fullHash = state.database.expandHash(hash) 568 | let branch = state.database.getBranch(fullHash) 569 | if state.database.isNewest(hash): 570 | var success = false 571 | for module in items(state.modules): 572 | if module.name == "builder": 573 | var jobj = newJObject() 574 | jobj["payload"] = newJObject() 575 | jobj["payload"]["after"] = newJString(fullHash) 576 | jobj["payload"]["ref"] = newJString("refs/heads/" & branch) 577 | jobj["rebuild"] = newJBool(true) 578 | 579 | if not state.database.platformExists(fullHash, module.platform): 580 | state.database.addPlatform(fullHash, module.platform) 581 | 582 | module.sock.send($jobj & "\c\L") 583 | success = true 584 | 585 | if success: 586 | reply["success"] = newJNull() 587 | else: 588 | reply["fail"] = newJString("No builders available.") 589 | else: 590 | reply["fail"] = newJString("Given commit is not newest.") 591 | else: 592 | reply["fail"] = newJString("Commit could not be found") 593 | 594 | m.sock.send($reply & "\c\L") 595 | 596 | elif json.existsKey("latestCommit"): 597 | let commit = state.database.getNewest() 598 | let branch = state.database.getBranch(commit) 599 | var reply = newJObject() 600 | reply["payload"] = newJObject() 601 | reply["payload"]["after"] = newJString(commit) 602 | reply["payload"]["ref"] = newJString("refs/heads/" & branch) 603 | reply["payload"]["commits"] = newJArray() 604 | reply["payload"]["commits"].add(%({"modified": %([%"build/csources.zip"])})) 605 | reply["rebuild"] = newJBool(true) 606 | m.sock.send($reply & "\c\L") 607 | if not state.database.platformExists(commit, m.platform): 608 | state.database.addPlatform(commit, m.platform) 609 | 610 | elif json.existsKey("pong"): 611 | # Module received PING and replied with PONG. 612 | state.modules[mIndex].pinged = false 613 | state.modules[mIndex].ping = epochTime() - json["pong"].str.parseFloat() 614 | state.modules[mIndex].lastPong = epochTime() 615 | 616 | elif json.existsKey("ping"): 617 | # Module thinks it's disconnected! Reply quickly! 618 | json["pong"] = json["ping"] 619 | json.delete("ping") 620 | m.sock.send($json & "\c\L") 621 | 622 | elif json.existsKey("do"): 623 | # { "do": "command to do (Info to get)" } 624 | if json["do"].str == "redisinfo": 625 | # This command asks the website for redis connection info. 626 | # { "redisinfo": { "port": ..., "password" } } 627 | var jobj = newJObject() 628 | jobj["redisinfo"] = newJObject() 629 | jobj["redisinfo"]["port"] = newJInt(state.redisPort) 630 | m.sock.send($jobj & "\c\L") 631 | else: 632 | echo("[Fatal] Can't understand message from " & m.name & ": ", 633 | line) 634 | assert(false) 635 | 636 | else: 637 | echo("[Fatal] Can't understand message from " & m.name & ": ", 638 | line) 639 | assert(false) 640 | 641 | proc handleModuleMsg(s: PAsyncSocket, arg: PObject) = 642 | var state = PState(arg) 643 | # Module sent a message to us 644 | var disconnect: seq[TModule] = @[] # Modules which disconnected 645 | for i in 0..state.modules.len()-1: 646 | template m: expr = state.modules[i] 647 | template onEOSDisconnect(operation: expr): stmt {.immediate.} = 648 | try: 649 | operation 650 | except EOS: 651 | disconnect.add(m()) 652 | continue 653 | 654 | if m.sock == s: 655 | var line = "" 656 | var ret = false 657 | onEOSDisconnect: 658 | ret = readLine(s, line) 659 | if ret: 660 | if line == "": 661 | disconnect.add(m) 662 | continue 663 | case m.status 664 | of MSConnecting: 665 | var errMsg = "" 666 | if state.parseGreeting(m, line, errMsg): 667 | onEOSDisconnect: 668 | m.sock.send($(%{ "reply": %"OK" }) & "\c\L") 669 | echo(uniqueMName(m), " accepted.") 670 | else: 671 | onEOSDisconnect: 672 | m.sock.send($(%{ "reply": %"FAIL", "reason": %errMsg }) & "\c\L") 673 | echo("Rejected ", uniqueMName(m)) 674 | disconnect.add(m) 675 | of MSConnected: 676 | echo("Got line from $1: $2" % [m.name, line]) 677 | 678 | # Getting a message is a sign of the module still being alive. 679 | state.modules[i].lastPong = epochTime() 680 | try: 681 | state.parseMessage(i, line) 682 | except: 683 | onEOSDisconnect: 684 | m.sock.send($(%{ 685 | "fatal": %getStackTrace(getCurrentException())}) & "\c\L") 686 | m.sock.close() 687 | echo("Fatal error for ", uniqueMName(m)) 688 | echo(getStackTrace(getCurrentException())) 689 | echo("--------------------------") 690 | IRCAnnounce(state, uniqueMName(m) & " created a fatal error.", true) 691 | disconnect.add(m) 692 | 693 | # Remove disconnected modules 694 | for m in items(disconnect): 695 | state.remove(m) 696 | 697 | proc handlePings(state: PState) = 698 | var remove: seq[TModule] = @[] # Modules that have timed out. 699 | for i in 0..state.modules.len-1: 700 | template module: expr = state.modules[i] 701 | var pingEvery = 100.0 702 | case module.status 703 | of MSConnected: 704 | if module.name == "builder": 705 | #if module.platform.startsWith("windows"): pingEvery = 15000.0 706 | 707 | if module.pinged and (epochTime() - module.lastPong) >= 25.0: 708 | echo(uniqueMName(module), 709 | " has not replied to PING. Assuming timeout!!!") 710 | remove.add(module) 711 | continue 712 | 713 | if (epochTime() - module.lastPong) >= pingEvery: 714 | var obj = newJObject() 715 | obj["ping"] = newJString(formatFloat(epochTime())) 716 | module.sock.send($obj & "\c\L") 717 | module.lastPong = epochTime() # This is a bit misleading, but I don't 718 | # want to add lastPing 719 | module.pinged = true 720 | echo("Pinging ", uniqueMName(module)) 721 | 722 | of MSConnecting: 723 | if (epochTime() - module.lastPong) >= 2.0: 724 | echo(uniqueMName(module), " did not send a greeting.") 725 | try: 726 | module.sock.send("{ \"reply\": \"FAIL\", \"desc\": \"Took too long\" }\c\L") 727 | except EOS: 728 | echo("Could not send error message for module: ", getCurrentExceptionMsg()) 729 | remove.add(module) 730 | 731 | # Remove the modules that have timed out. 732 | for m in items(remove): 733 | state.remove(m) 734 | 735 | # HTML Generation 736 | 737 | proc joinUrl(u, u2: string): string = 738 | if u.endswith("/"): 739 | return u & u2 740 | else: return u & "/" & u2 741 | 742 | proc getWebUrl(state: PState, c: TCommit, p: TPlatform): string = 743 | result = state.req.makeUri("commits/$2/$1/" % [c.hash[0..11], p.platform], 744 | absolute = false) 745 | 746 | proc getLogUrl(state: PState, c: TCommit, p: TPlatform): string = 747 | result = joinUrl(getWebUrl(state, c, p), "logs.txt") 748 | 749 | proc isBuilding(platforms: TTable[string, TStatus], p: string, c: TCommit): bool = 750 | return platforms[p].isInProgress and platforms[p].hash == c.hash 751 | 752 | proc genPlatformResult(state: PState, c: TCommit, p: TPlatform, 753 | platforms: TTable[string, TStatus], 754 | req: Request): string = 755 | result = "" 756 | case p.buildResult 757 | of bUnknown: 758 | # Check whether this platform is currently building this commit. 759 | if isBuilding(platforms, p.platform, c): 760 | result.add("" % 761 | [req.makeUri("static/images/progress.gif", absolute = false)]) 762 | of bFail: 763 | let logUrl = getLogUrl(state, c, p) 764 | result.add("fail" % [logUrl]) 765 | of bSuccess: result.add("ok") 766 | result.add(" ") 767 | case p.testResult 768 | of tUnknown: 769 | if isBuilding(platforms, p.platform, c): 770 | result.add("" % 771 | [req.makeUri("static/images/progress.gif", absolute = false)]) 772 | of tFail: 773 | let logUrl = getLogUrl(state, c, p) 774 | result.add("fail" % [logUrl]) 775 | of tSuccess: 776 | var testresultsURL = joinUrl(getWebUrl(state, c, p), "testresults.html") 777 | var percentage = float(p.passed) / float(p.total - p.skipped) * 100.0 778 | result.add("" % [testresultsURL] & 779 | formatFloat(percentage, precision=4) & "%") 780 | 781 | proc genBuildResult(state: PState, c: TCommit, p: TPlatform): string = 782 | result = "" 783 | case p.buildResult 784 | of bUnknown: 785 | # Check whether this platform is currently building this commit. 786 | if isBuilding(state.platforms, p.platform, c): 787 | result.add(htmlgen.`div`(class = "half indivUnknown", 788 | img(alt = "Building", 789 | src = state.req.makeUri("public/images/progress.gif", 790 | absolute = false)) 791 | )) 792 | else: 793 | result.add(htmlgen.`div`(class = "half indivUnknown", 794 | htmlgen.p("Unknown"))) 795 | of bFail: 796 | result.add(htmlgen.`div`(class = "half indivFailure", 797 | a(href = getLogUrl(state, c, p), class = "fail", "Fail") 798 | )) 799 | of bSuccess: 800 | result.add(htmlgen.`div`(class = "half indivSuccess", "OK")) 801 | 802 | proc genTestResult(state: PState, c: TCommit, p: TPlatform): string = 803 | result = "" 804 | case p.testResult 805 | of tUnknown: 806 | if isBuilding(state.platforms, p.platform, c): 807 | result.add(htmlgen.`div`(class = "half indivUnknown", 808 | img(alt = "Building", 809 | src = state.req.makeUri("public/images/progress.gif", 810 | absolute = false)) 811 | )) 812 | else: 813 | result.add(htmlgen.`div`(class = "half indivUnknown", 814 | htmlgen.p("Unknown"))) 815 | of tFail: 816 | result.add(htmlgen.`div`(class = "half indivFailure", 817 | a(href = getLogUrl(state, c, p), class = "fail", "Fail") 818 | )) 819 | of tSuccess: 820 | var testresultsURL = joinUrl(getWebUrl(state, c, p), "testresults.html") 821 | result.add(htmlgen.`div`(class = "half indivSuccess", 822 | a(href = testResultsURL, class = "success", 823 | $(p.passed) & "/" & $(p.total-p.skipped)) 824 | )) 825 | 826 | proc cmpPlatforms(a, b: string): int = 827 | if a == b: return 0 828 | var dashes = a.split('-') 829 | var dashes2 = b.split('-') 830 | if dashes[0] == dashes2[0]: 831 | if dashes[1] == dashes2[1]: return system.cmp(a,b) 832 | case dashes[1] 833 | of "x86": 834 | return 1 835 | of "x86_64": 836 | if dashes2[1] == "x86": return -1 837 | else: return 1 838 | of "ppc64": 839 | if dashes2[1] == "x86" or dashes2[1] == "x86_64": return -1 840 | else: return 1 841 | else: 842 | return system.cmp(dashes[1], dashes2[1]) 843 | else: 844 | case dashes[0] 845 | of "linux": 846 | return 1 847 | of "windows": 848 | if dashes2[0] == "linux": return -1 849 | else: return 1 850 | of "macosx": 851 | if dashes2[0] == "linux" or dashes2[0] == "windows": return -1 852 | else: return 1 853 | else: 854 | if dashes2[0] == "linux" or dashes2[0] == "windows" or 855 | dashes2[0] == "macosx": return -1 856 | else: 857 | return system.cmp(a, b) 858 | 859 | proc findLatestCommit(entries: seq[TEntry], 860 | platform: string, 861 | res: var tuple[entry: TEntry, latest: bool]): bool = 862 | result = false 863 | var i = 0 864 | for c, p in items(entries): 865 | if platform in p: 866 | let platf = p[platform] 867 | if platf.buildResult == bSuccess: 868 | res = ((c, p), i == 0 and entries[0].c.hash == c.hash) 869 | return true 870 | 871 | i.inc() 872 | 873 | proc genDownloadTable(req: Request, entries: seq[TEntry], 874 | platforms: seq[string]): string = 875 | result = "" 876 | 877 | var OSes: seq[string] = @[] 878 | var CPUs: seq[string] = @[] 879 | var versions: seq[tuple[ver: string, os: string]] = @[] 880 | for p in platforms: 881 | var spl = p.split('-') 882 | 883 | if spl.len() > 2: 884 | if (ver: spl[2], os: spl[0]) notin versions: 885 | versions.add((spl[2], spl[0])) 886 | elif spl[0] notin OSes: 887 | versions.add(("", spl[0])) 888 | 889 | if spl[0] notin OSes: OSes.add(spl[0]) 890 | if spl[1] notin CPUs: CPUs.add(spl[1]) 891 | 892 | var table = htmlhelp.initTable() 893 | table.addRow() 894 | table.addRow() # For Versions. 895 | table[0].addCol("", true) 896 | table[1].addCol("", true) 897 | for os in OSes: 898 | table[0].addCol(os, true) # Add OS. 899 | for cpuI, cpu in CPUs: 900 | if cpuI+2 > table.len()-1: table.addRow() 901 | table[2+cpuI].addCol(cpu, true) 902 | 903 | # Loop through versions. 904 | var currentVerI = 0 905 | 906 | while currentVerI < versions.len(): 907 | var columnAdded = false 908 | var pName = "" 909 | if versions[currentVerI].ver != "": 910 | pName = versions[currentVerI].os & "-" & 911 | cpu & "-" & versions[currentVerI].ver 912 | else: 913 | pName = versions[currentVerI].os & "-" & cpu 914 | 915 | if pName in platforms: 916 | var latestCommit: tuple[entry: TEntry, latest: bool] 917 | if entries.findLatestCommit(pName, latestCommit): 918 | var (entry, latest) = latestCommit 919 | var attrs: seq[tuple[name, content: string]] = @[] 920 | attrs.add(("class", if latest: "link green" else: "link orange")) 921 | if pName in entry.p: 922 | var weburl = req.makeUri("commits/$2/nimrod_$1.zip" % 923 | [entry.c.hash[0..11], entry.p[pName].platform], 924 | absolute = false) 925 | table[2+cpuI].addCol(a(entry.c.hash[0..11], href = weburl), attrs=attrs) 926 | columnAdded = true 927 | 928 | if not columnAdded: 929 | # Add an empty column. 930 | table[2+cpuI].addCol("") 931 | 932 | currentVerI.inc() 933 | 934 | for v in versions: 935 | table[1].addCol(v.ver, true) 936 | if v.ver != "": 937 | # Add +1 to colspan of OS 938 | var cols = findCols(table[0], v.os) 939 | assert cols.len > 0 940 | if not cols[0].attrs.hasKey("colspan"): 941 | cols[0].attrs["colspan"] = "1" 942 | else: 943 | cols[0].attrs["colspan"] = $(cols[0].attrs["colspan"].parseInt + 1) 944 | 945 | result = table.toHtml("id=\"downloads\"") 946 | 947 | proc genTopButtons(req: Request, platforms: TTable[string, TStatus], 948 | entries: seq[TEntry]): string = 949 | # Generate buttons for C sources and docs. 950 | # Find the latest C sources. 951 | result = "" 952 | var csourceWeb = "" 953 | var csourceFound = false 954 | var csourceLatest = false 955 | var i = 0 956 | for c, p in items(entries): 957 | for platf in p: 958 | if platf.csources: 959 | csourceWeb = req.makeUri("commits/$2/nimrod_$1_csources.zip" % 960 | [c.hash[0..11], platf.platform], absolute=false) 961 | csourceFound = true 962 | csourceLatest = i == 0 963 | break 964 | if csourceFound: break 965 | i.inc() 966 | 967 | # Find out whether latest doc gen succeeded. 968 | var docgenSuccess = true # By default it succeeded. 969 | for p, s in pairs(platforms): 970 | if s.jobs[jDocGen] == jFail: 971 | docgenSuccess = false 972 | break 973 | 974 | var csourceClass = "right " & (if csourceLatest: "active" else: "warning") & 975 | " button" 976 | var docClass = "left " & (if docgenSuccess: "active" else: "warning") & 977 | " button" 978 | 979 | var docWeb = req.makeUri("docs/lib.html", absolute=false) 980 | 981 | result.add(a(span("", class = "download") & 982 | span("C Sources", class = "platform"), 983 | class = csourceClass, href = csourceWeb)) 984 | 985 | result.add(a(span("", class = "book") & 986 | span("Documentation", class = "platform"), 987 | class = docClass, href = docWeb)) 988 | 989 | proc genCommitUrl(hash: string): string = 990 | return joinUrl("https://github.com/Araq/Nimrod/commit/", hash) 991 | 992 | proc genUserUrl(user: string): string = 993 | return joinUrl("https://github.com/", user) 994 | 995 | proc genSpecificBranchHTML(state: PState, branch: string, 996 | info: tuple[c: TCommit, buildInfo: TPlatform]): string = 997 | let (commit, build) = (info.c, info.buildInfo) 998 | const month = 2_628_000 999 | let dateClass = "date " & 1000 | (if (commit.date - getTime()) > month: "outdated" else: "") 1001 | result = 1002 | htmlgen.`div`(class = "lastResults", 1003 | htmlgen.`div`(class = "branch " & (if branch == "master": "master" else: ""), 1004 | span(title="Branch tested", branch) 1005 | ), 1006 | state.genBuildResult(commit, build), 1007 | state.genTestResult(commit, build), 1008 | p(a(href = genCommitUrl(commit.hash),commit.hash[0..11]), " by ", 1009 | a(href=genUserUrl(commit.username), commit.username), " (", 1010 | a(href=getLogUrl(state, commit, build), "logs"), ")" 1011 | ), 1012 | p(class = "commitMsg", commit.commitMsg), 1013 | p(class = dateClass, $(commit.date)) 1014 | ) 1015 | 1016 | proc genSpecificBuilderHTML(state: PState, 1017 | platfName: string): tuple[inProgress: bool, html: string] = 1018 | result = (false, "") 1019 | let imgProgress = "\"Busy\"" % 1020 | [state.req.makeUri("images/progress.gif", absolute = false)] 1021 | var builderModule: TModule 1022 | if findBuilderModule(state, platfName, builderModule): 1023 | let job = state.platforms[platfName] 1024 | let lag = int(builderModule.ping * 1000.0) 1025 | var lagTxt = "" 1026 | if lag == 0: 1027 | lagTxt = "<0ms" 1028 | else: 1029 | lagTxt = $lag & "ms" 1030 | 1031 | var progressSpecific = "" 1032 | if job.isInProgress: 1033 | progressSpecific.add imgProgress 1034 | let masterSpecific = if job.branch == "master": "master" else: "" 1035 | progressSpecific.add p("Current: " & job.hash[0..11] & " (" & 1036 | span(class="branch " & masterSpecific, job.branch) & 1037 | ")") 1038 | var queueSpecific = "" 1039 | if state.buildQueue.hasKey(platfName): 1040 | let q = state.buildQueue[platfName] 1041 | if q.len != 0: 1042 | queueSpecific = p($q.len & " commits in build queue") 1043 | 1044 | result.html = htmlgen.`div`(class = "buildInfo", 1045 | progressSpecific, 1046 | p($job), 1047 | p(lagTxt), 1048 | queueSpecific 1049 | ) 1050 | result.inProgress = job.isInProgress 1051 | else: 1052 | result.html = htmlgen.`div`(class = "buildInfo", 1053 | p("Builder not connected.")) 1054 | 1055 | proc genBuildResults(state: PState, platforms: seq[string], entr: seq[TEntry]): string = 1056 | # Platform name -> [branch, html generated] 1057 | var platformBuilds = initTable[ 1058 | string, 1059 | TTable[string, tuple[c: TCommit, buildInfo: TPlatform]]]() 1060 | 1061 | # TODO: Move to MongoDB and a better more efficient db layout. 1062 | # the following code is extremely slow and complicated. 1063 | 1064 | # The following sorts the commits into a list more suited to the new layout. 1065 | for entry in items(entr): 1066 | let (commit, builds) = (entry.c, entry.p) 1067 | for build in builds: 1068 | if isBuilding(state.platforms, build.platform, commit): 1069 | continue # If the builder is currently building it, don't show it here. 1070 | if build.buildResult == bUnknown: 1071 | continue # No point in showing an unknown for both build&test result. 1072 | # So skip it. 1073 | if not platformBuilds.hasKey(build.platform): 1074 | platformBuilds[build.platform] = initTable[ 1075 | string, 1076 | tuple[c: TCommit, buildInfo: TPlatform]]() 1077 | let thisBranch = (if isNil(commit.branch): "master" else: commit.branch) 1078 | assert thisBranch != "" 1079 | if platformBuilds[build.platform].hasKey(thisBranch): 1080 | # Already got the latest commit for this branch 1081 | continue 1082 | platformBuilds.mget(build.platform)[thisBranch] = (c: commit, buildInfo: build) 1083 | 1084 | # platfClass = 1085 | # If build in progress: blue (Progress) 1086 | # If master branch failed: red (fail) 1087 | # If master branch tests not 100%: orange 1088 | # If master branch fully successful: green. 1089 | 1090 | proc genPlatfBuildRes(state: PState, class, name, 1091 | branches, builderStatus: string): string = 1092 | result = htmlgen.`div`(class="platfBuildResult " & class, 1093 | htmlgen.`div`(class="header", span(name)), 1094 | branches, 1095 | htmlgen.`div`(class="header", span("Builder status")), 1096 | builderStatus) 1097 | 1098 | result = "" 1099 | # 3 platforms per single row, only needed to keep the boxes in one single row 1100 | # ... layout fix basically. 1101 | var platfsCol = "" 1102 | var platfsCount = 0 1103 | for platfName in platforms: 1104 | if not platformBuilds.hasKey(platfName): 1105 | let (inProgress, html) = genSpecificBuilderHTML(state, platfName) 1106 | if inProgress: 1107 | result.add genPlatfBuildRes(state, "platfProgress", platfName, "", html) 1108 | continue 1109 | else: 1110 | continue # No commits were built for this, and nothing is building. 1111 | 1112 | let value = platformBuilds[platfName] 1113 | var branches = "" 1114 | if value.hasKey("master"): 1115 | branches.add(genSpecificBranchHTML(state, "master", value["master"])) 1116 | for branch, info in value: 1117 | if branch == "master": continue 1118 | branches.add(genSpecificBranchHTML(state, branch, info)) 1119 | 1120 | let (inProgress, builderHtml) = genSpecificBuilderHTML(state, platfName) 1121 | var platfClass = "platfWarning" 1122 | if inProgress: platfClass = "platfProgress" 1123 | if value.hasKey("master"): 1124 | if value["master"].buildInfo.buildResult == bFail or 1125 | value["master"].buildInfo.testResult == tFail: 1126 | platfClass = "platfFailure" 1127 | elif value["master"].buildInfo.testResult == tSuccess and 1128 | value["master"].buildInfo.failed == 0: 1129 | platfClass = "platfSuccess" 1130 | else: 1131 | platfClass = "platfWarning" 1132 | else: 1133 | platfClass = "platfWarning" # Just to be explicit. 1134 | 1135 | platfsCol.add genPlatfBuildRes(state, platfClass, platfName, branches, builderHtml) 1136 | inc(platfsCount) 1137 | 1138 | if platfsCount == 3: 1139 | result.add(htmlgen.`div`(style="float: left; width: 100%;", platfsCol)) 1140 | platfsCount = 0 1141 | platfsCol = "" 1142 | # Add the rest of the rows. 1143 | result.add(htmlgen.`div`(style="float: left; width: 100%;", platfsCol)) 1144 | 1145 | 1146 | include "index.html" 1147 | # Jester 1148 | 1149 | proc cleanup(state: PState) = 1150 | echo("^C detected. Cleaning up...") 1151 | for m in items(state.modules): 1152 | # TODO: Send something to the modules to warn them? 1153 | m.sock.close() 1154 | 1155 | proc handleAccept(s: PAsyncSocket, state: PState) = 1156 | # Connection from a module 1157 | var client: PAsyncSocket; new(client) 1158 | var IPAddr = "" 1159 | s.acceptAddr(client, IPAddr) 1160 | state.addModule(client, IPAddr) 1161 | 1162 | when isMainModule: 1163 | var configPath = "" 1164 | if paramCount() > 0: 1165 | configPath = paramStr(1) 1166 | echo("Loading config ", configPath, "...") 1167 | else: 1168 | quit("Usage: ./website configPath") 1169 | 1170 | echo("Started website: built at ", CompileDate, " ", CompileTime) 1171 | 1172 | var state = website.open(configPath) 1173 | var settings = newSettings(port = net.Port(state.scgiPort)) 1174 | state.jesterSettings = settings 1175 | 1176 | routes: 1177 | get "/": 1178 | state.req = request 1179 | let html = state.genHtml() 1180 | resp html 1181 | 1182 | while true: 1183 | doAssert state.dispatcher.poll(1) 1184 | 1185 | asyncdispatch.poll(1) 1186 | 1187 | state.handlePings() 1188 | 1189 | state.database.keepAlive() 1190 | 1191 | 1192 | --------------------------------------------------------------------------------