├── .gitignore ├── app ├── coffee │ ├── IndexController.coffee │ ├── mongojs.coffee │ ├── WebHookManager.coffee │ ├── RepositoryController.coffee │ ├── AuthenticationController.coffee │ ├── WebHookController.coffee │ ├── RepositoryManager.coffee │ ├── BuildController.coffee │ └── BuildManager.coffee └── views │ ├── builds │ ├── show.jade │ └── _build.jade │ ├── _help.jade │ ├── layout.jade │ ├── badges │ └── pdf.jade │ ├── index.jade │ └── repos │ ├── list.jade │ └── show.jade ├── LICENSE ├── package.json ├── config └── settings.defaults.coffee ├── test └── unit │ └── coffee │ ├── WebHookControllerTests.coffee │ ├── RepositoryManagerTests.coffee │ └── BuildManagerTests.coffee ├── README.md ├── Gruntfile.coffee └── app.coffee /.gitignore: -------------------------------------------------------------------------------- 1 | app.js 2 | app/js 3 | test/unit/js 4 | node_modules 5 | -------------------------------------------------------------------------------- /app/coffee/IndexController.coffee: -------------------------------------------------------------------------------- 1 | module.exports = IndexController = 2 | index: (req, res, next) -> 3 | res.render "index" -------------------------------------------------------------------------------- /app/views/builds/show.jade: -------------------------------------------------------------------------------- 1 | extends ../layout 2 | 3 | block content 4 | .card 5 | .page-header 6 | h1 7 | a(href="#{settings.mountPoint}/repos/#{build.repo}") #{repo} 8 | 9 | include ./_build -------------------------------------------------------------------------------- /app/coffee/mongojs.coffee: -------------------------------------------------------------------------------- 1 | settings = require "settings-sharelatex" 2 | mongojs = require "mongojs" 3 | db = mongojs.connect(settings.mongo.url, ["githubRepos", "githubBuilds"]) 4 | module.exports = 5 | db: db 6 | ObjectId: mongojs.ObjectId 7 | -------------------------------------------------------------------------------- /app/views/_help.jade: -------------------------------------------------------------------------------- 1 | .page-header 2 | h2 Configuration options 3 | 4 | h3 Compiler 5 | 6 | p 7 | | You can set the LaTeX engine/compiler by including the following line in your LaTeX file: 8 | pre.alert.alert-info 9 | | % In your .tex file 10 | | % !TEX program = xelatex 11 | | Valid values are pdflatex, lualatex, xelatex, and 12 | | latex. 13 | 14 | p 15 | | You can also specify this option in a .latex.yml file in your project: 16 | pre.alert.alert-info 17 | | # In .latex.yml 18 | | compiler: xelatex 19 | 20 | 21 | h3 Root file 22 | 23 | p 24 | | If your project contains multiple .tex files, we'll try to automatically detect which one to 25 | | run LaTeX on. However, if we get it wrong, you can specify it manually in 26 | | .latex.yml: 27 | pre.alert.alert-info 28 | | # In .latex.yml 29 | | root_file: thesis.tex -------------------------------------------------------------------------------- /app/views/layout.jade: -------------------------------------------------------------------------------- 1 | doctype html 2 | html 3 | head 4 | title Github LaTeX CI 5 | link(rel="stylesheet", href=settings.style.css) 6 | script(type="text/javascript", src="//ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js") 7 | script(type="text/javascript", src="//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/js/bootstrap.min.js") 8 | body 9 | .navbar.navbar-default 10 | .container-fluid 11 | .navbar-header 12 | a(href="#{settings.style.homePageUrl}").navbar-brand= settings.style.brandText 13 | 14 | ul.nav.navbar-nav.navbar-right 15 | li.subdued 16 | a(href="https://www.sharelatex.com") The ShareLaTeX Cloud Compiler is a service developed and provided by ShareLaTeX 17 | 18 | .content.content-alt 19 | .container 20 | block content 21 | 22 | footer.site-footer 23 | .container 24 | .row 25 | .col-xs-12 26 | p.text-center 27 | a(href="https://github.com/sharelatex/github-latex-ci") The ShareLaTeX Cloud Compiler is Open Source. Run your own, or fork on GitHub. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 ShareLaTeX 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /app/views/badges/pdf.jade: -------------------------------------------------------------------------------- 1 | - if (typeof(build) == "undefined") 2 | - var text = "unknown"; 3 | - var color = "#9f9f9f"; 4 | - var width = 61; 5 | - else if (build.status == "in_progress") 6 | - var text = "building"; 7 | - var color = "#dfb317"; 8 | - var width = 54; 9 | - else if (build.status == "success") 10 | - var text = "success"; 11 | - var color = "#4c1"; 12 | - var width = 54; 13 | - else 14 | - var text = "failed"; 15 | - var color = "#e05d44"; 16 | - var width = 41; 17 | 18 | svg(xmlns="http://www.w3.org/2000/svg", width="#{width + 28}", height="20") 19 | 20 | lineargradient#smooth(x2="0", y2="100%") 21 | stop(offset="0", stop-color="#bbb", stop-opacity=".1") 22 | stop(offset="1", stop-opacity=".1") 23 | 24 | mask#round 25 | rect(width="#{width + 28}", height="20", rx="3", fill="#fff") 26 | 27 | g(mask="url(#round)") 28 | rect(width="28", height="20", fill="#555") 29 | rect(x="28", width="#{width}", height="20", fill="#{color}") 30 | rect(width="#{width + 28}", height="20", fill="url(#smooth)") 31 | 32 | g(fill="#fff", text-anchor="middle", font-family="DejaVu Sans,Verdana,Geneva,sans-serif", font-size="11") 33 | text(x="14", y="15", fill="#010101", fill-opacity=".3") pdf 34 | text(x="14", y="14") pdf 35 | text(x="#{28 + width / 2 - 1}", y="15", fill="#010101", fill-opacity=".3") #{text} 36 | text(x="#{28 + width / 2 - 1}", y="14") #{text} 37 | -------------------------------------------------------------------------------- /app/views/index.jade: -------------------------------------------------------------------------------- 1 | extends ./layout 2 | 3 | block content 4 | 5 | .row 6 | .col-md-10.col-md-offset-1 7 | .text-center.row-spaced 8 | .alert.alert-danger 9 | strong Warning:  10 | | ShareLaTeX Cloud Compiler will be closing down on the 21st of July. Github Sync in the ShareLaTeX editor will continue to be supported.  11 | a(href="/learn/kb/where_is_github_ci", style="text-decoration: underline") 12 | | Find out more. 13 | .page-header 14 | h1 ShareLaTeX Cloud Compiler 15 | p 16 | | Connect your public LaTeX GitHub repositories and 17 | | we'll automatically compile 18 | | your LaTeX code to a PDF on each change. 19 | 20 | p 21 | a.btn.btn-lg.btn-info(href="#{settings.mountPoint}/login") Log in with GitHub 22 | 23 | .row-spaced 24 | p   25 | 26 | .row.row-spaced 27 | .col-md-10.col-md-offset-1 28 | .card 29 | .page-header 30 | h2 How does this work? 31 | 32 | p 33 | | We run a LaTeX compiler in the cloud which listens for changes to your Github repository. 34 | | When you make a change, we compile the latest content and host the PDF and log files for 35 | | you. 36 | 37 | p 38 | | You can embed a badge into your project's README or website so that you can see the latest 39 | | compile status and access the PDF with a single click. 40 | 41 | include ./_help -------------------------------------------------------------------------------- /app/coffee/WebHookManager.coffee: -------------------------------------------------------------------------------- 1 | settings = require "settings-sharelatex" 2 | {publicUrl, mountPoint} = settings.internal.github_latex_ci 3 | crypto = require "crypto" 4 | logger = require "logger-sharelatex" 5 | 6 | # Monkey patch in a delete hook method 7 | Repo = require("octonode").repo 8 | Repo::deleteHook = (id, cb) -> 9 | url = "/repos/#{@name}/hooks/#{id}" 10 | @client.del url, null, (err, s, b, h) -> 11 | return cb(err) if err 12 | if s isnt 204 then cb(new Error("Repo deleteHook error")) else cb null, b, h 13 | 14 | module.exports = WebHookManager = 15 | createWebHook: (ghclient, repo, callback = (error, response) ->) -> 16 | ghclient.repo(repo).hook({ 17 | "name": "web", 18 | "active": true, 19 | "events": ["push"], 20 | "config": { 21 | "url": "#{publicUrl}#{mountPoint}/events", 22 | "content_type": "json", 23 | "secret": settings.github.webhook_secret 24 | } 25 | }, callback) 26 | 27 | destroyWebHook: (ghclient, repo, webhook_id, callback = (error) ->) -> 28 | ghclient.repo(repo).deleteHook(webhook_id, callback) 29 | 30 | verifyWebHookEvent: (header, body) -> 31 | hmac = crypto.createHmac('sha1', settings.github.webhook_secret) 32 | hmac.setEncoding("hex") 33 | hmac.write body 34 | hmac.end() 35 | hash = hmac.read().toString("hex") 36 | valid = (header == "sha1=#{hash}") 37 | logger.log hash: hash, header: header, valid: valid, "verifying body" 38 | return valid -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-latex-ci", 3 | "version": "0.0.0", 4 | "description": "A continuous build system for LaTeX on Github", 5 | "main": "app.js", 6 | "author": "", 7 | "license": "MIT", 8 | "devDependencies": { 9 | "bunyan": "^1.0.1", 10 | "chai": "^1.9.1", 11 | "grunt": "^0.4.5", 12 | "grunt-bunyan": "^0.5.0", 13 | "grunt-contrib-clean": "^0.6.0", 14 | "grunt-contrib-coffee": "^0.11.1", 15 | "grunt-execute": "^0.2.2", 16 | "grunt-forever": "^0.4.4", 17 | "grunt-mocha-test": "^0.11.0", 18 | "grunt-shell": "^1.0.1", 19 | "sandboxed-module": "^1.0.1", 20 | "sinon": "^1.10.3" 21 | }, 22 | "dependencies": { 23 | "async": "^0.9.0", 24 | "body-parser": "^1.6.6", 25 | "connect-redis": "^2.0.0", 26 | "csurf": "^1.5.0", 27 | "express": "^4.8.5", 28 | "express-session": "^1.7.6", 29 | "jade": "^1.5.0", 30 | "js-yaml": "^3.2.1", 31 | "knox": "^0.9.1", 32 | "lodash": "^3.10.1", 33 | "logger-sharelatex": "git+https://github.com/sharelatex/logger-sharelatex.git#v1.0.0", 34 | "metrics-sharelatex": "git+https://github.com/sharelatex/metrics-sharelatex", 35 | "mime": "^1.3.4", 36 | "mongojs": "^0.14.0", 37 | "octonode": "^0.6.4", 38 | "redis-sharelatex": "~0.0.4", 39 | "request": "^2.57.0", 40 | "settings-sharelatex": "git+https://github.com/sharelatex/settings-sharelatex.git#v1.0.0" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /app/views/repos/list.jade: -------------------------------------------------------------------------------- 1 | extends ../layout 2 | 3 | block content 4 | .page-header 5 | h1.text-center ShareLaTeX Cloud Compiler 6 | 7 | .card 8 | .page-header 9 | h1 Your GitHub Repositories 10 | 11 | p.text-center After you enable a repository, we will automatically compile your LaTeX code to a PDF everytime you make a change. 12 | 13 | - if (activeRepos.length > 0) 14 | table.table.table-hover.table-striped 15 | thead 16 | tr 17 | th Enabled Repos 18 | tbody 19 | each repo in activeRepos 20 | tr 21 | td 22 | .row 23 | .col-xs-6 24 | a(href=repo.html_url)= repo.full_name 25 | .col-xs-6.text-right 26 | form(action="#{settings.mountPoint}/repos/#{repo.full_name}/hook/destroy", method="POST") 27 | a.btn.btn-info(href="#{settings.mountPoint}/repos/#{repo.full_name}") View Latest Build 28 | | 29 | input(type="hidden", name="_csrf", value=csrfToken) 30 | button.btn.btn-danger(type="submit") Turn Off 31 | 32 | table.table.table-hover.table-striped 33 | thead 34 | tr 35 | th Your Repos 36 | tbody 37 | each repo in otherRepos 38 | tr 39 | td 40 | .row 41 | .col-xs-6 42 | a(href=repo.html_url)= repo.full_name 43 | .col-xs-6.text-right 44 | form(action="#{settings.mountPoint}/repos/#{repo.full_name}/hook", method="POST") 45 | input(type="hidden", name="_csrf", value=csrfToken) 46 | button.btn.btn-default(type="submit") Turn On 47 | 48 | -------------------------------------------------------------------------------- /app/views/builds/_build.jade: -------------------------------------------------------------------------------- 1 | if build.status == "in_progress" 2 | p 3 | .alert.alert-info 4 | | Building... 5 | script(type="text/javascript"). 6 | setTimeout(function() { 7 | window.location.reload(); 8 | }, 5000) 9 | else 10 | if build.status == "success" 11 | p 12 | .alert.alert-success 13 | .row 14 | .col-xs-12 15 | a.btn.btn-success(href="#{settings.mountPoint}/repos/#{build.repo}/builds/#{build.sha}/raw/output.pdf") Download PDF 16 | |   Build was successful! 17 | else 18 | p 19 | .alert.alert-danger 20 | .row 21 | .col-xs-12 22 | | This build failed. Please check the logs below. 23 | 24 | - if (build.commit) 25 | p 26 | .alert.alert-info 27 | small.pull-right Commit 28 | a.text-info(href="https://github.com/#{repo}/commit/#{build.sha}")= build.sha.slice(0, 8) 29 | p 30 | a.text-info(href="https://github.com/#{repo}/commit/#{build.sha}")= build.commit.message 31 | small 32 | | #{build.commit.author.name} authored on #{build.commit.author.date} 33 | 34 | .row-spaced 35 | .page-header 36 | .pull-right 37 | a.btn.btn-info(href="#{settings.mountPoint}/repos/#{build.repo}/builds/#{build.sha}/raw/output.log") Download logs 38 | h3 Logs 39 | 40 | .well 41 | iframe(width="100%", height="400px", style="border: none", src="#{settings.mountPoint}/repos/#{build.repo}/builds/#{build.sha}/raw/output.log") 42 | 43 | .row-spaced 44 | .page-header 45 | h3 All Output Files 46 | 47 | ul.unstyled-list 48 | each file in outputFiles 49 | li 50 | a(href="#{settings.mountPoint}/repos/#{build.repo}/builds/#{build.sha}/raw/#{file}")= file -------------------------------------------------------------------------------- /config/settings.defaults.coffee: -------------------------------------------------------------------------------- 1 | if !process.env.GITHUB_CLIENT_ID? or !process.env.GITHUB_CLIENT_SECRET? 2 | console.log """ 3 | Please set GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET environment variables 4 | """ 5 | process.exit(1) 6 | 7 | if !process.env.AWS_ACCESS_KEY? or !process.env.AWS_SECRET_KEY? 8 | console.log """ 9 | Please set AWS_ACCESS_KEY and AWS_SECRET_KEY environment variables 10 | """ 11 | process.exit(1) 12 | 13 | 14 | module.exports = 15 | internal: 16 | github_latex_ci: 17 | mountPoint: "" 18 | url: "http://localhost:3021" 19 | publicUrl: "http://localhost:3021" 20 | host: "localhost" 21 | port: 3021 22 | userAgent: "sharelatex/github-latex-ci" 23 | 24 | style: 25 | css: "//maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css" 26 | brandText: "ShareLaTeX Cloud Compiler" 27 | # css: "https://www.sharelatex.com/stylesheets/style.css" 28 | # brandText: "" 29 | homePageUrl: "/" 30 | 31 | apis: 32 | clsi: 33 | url: "http://localhost:3013" 34 | 35 | github: 36 | client_id: process.env.GITHUB_CLIENT_ID 37 | client_secret: process.env.GITHUB_CLIENT_SECRET 38 | webhook_secret: process.env.GITHUB_WEBHOOK_SECRET || "webhook_secret" 39 | 40 | redis: 41 | web: 42 | host: "localhost" 43 | port: "6379" 44 | password: "" 45 | 46 | mongo: 47 | url: 'mongodb://127.0.0.1/sharelatex' 48 | 49 | s3: 50 | key: process.env.AWS_ACCESS_KEY 51 | secret: process.env.AWS_SECRET_KEY 52 | github_latex_ci_bucket: "sl-github-latex-ci-dev" 53 | 54 | security: 55 | sessionSecret: "banana" 56 | 57 | cookieName: "github-latex-ci.sid" 58 | cookieDomain: null 59 | secureCookie: false 60 | behindProxy: false -------------------------------------------------------------------------------- /test/unit/coffee/WebHookControllerTests.coffee: -------------------------------------------------------------------------------- 1 | sandboxedModule = require "sandboxed-module" 2 | modulePath = "../../../app/js/WebHookController" 3 | sinon = require "sinon" 4 | chai = require "chai" 5 | chai.should() 6 | 7 | describe "WebHookController", -> 8 | beforeEach -> 9 | @WebHookController = sandboxedModule.require modulePath, requires: 10 | "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } 11 | "./WebHookManager": @WebHookManager = {} 12 | "./RepositoryManager": @RepositoryManager = {} 13 | "./BuildManager": @BuildManager = {} 14 | "settings-sharelatex": 15 | internal: github_latex_ci: { publicUrl: "http://example.com", mountPoint: @mountPoint = "/github" } 16 | 17 | describe "createHook", -> 18 | beforeEach -> 19 | @hook_id = "hook-id" 20 | @WebHookManager.createWebHook = sinon.stub().callsArgWith(2, null, { id: @hook_id }) 21 | @RepositoryManager.saveWebHook = sinon.stub().callsArg(2) 22 | @repo = "owner/repo" 23 | @req = 24 | ghclient: "mock-ghclient" 25 | params: 26 | owner: @repo.split("/")[0] 27 | repo: @repo.split("/")[1] 28 | @res = 29 | redirect: sinon.stub() 30 | 31 | @WebHookController.createHook @req, @res 32 | 33 | it "should send a request to Github to create the webhook", -> 34 | @WebHookManager.createWebHook 35 | .calledWith(@req.ghclient, @repo) 36 | .should.equal true 37 | 38 | it "should store the webhook in the database", -> 39 | @RepositoryManager.saveWebHook 40 | .calledWith(@repo, @hook_id) 41 | .should.equal true 42 | 43 | it "should redirect to the repo list page", -> 44 | @res.redirect 45 | .calledWith("#{@mountPoint}/repos") 46 | .should.equal true 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | github-latex-ci 2 | =============== 3 | 4 | An automated LaTeX build service for compiling continuous compiling of github repositories. 5 | 6 | Dependencies 7 | ------------ 8 | 9 | The ShareLaTeX Cloud Compiler (github-latex-ci) requires the following dependencies: 10 | 11 | * [MongoDB](http://www.mongodb.org/) 12 | * [Node.js](http://nodejs.org/) 13 | * [The ShareLaTeX Compiler](https://github.com/sharelatex/clsi-sharelatex) 14 | * [Grunt](http://gruntjs.com/) (Install with `npm install grunt-cli`). 15 | 16 | Installation 17 | ------------ 18 | 19 | After ensuring you have MongoDB and Node.js installed and running, and have set up the [ShareLaTeX Compiler](https://github.com/sharelatex/clsi-sharelatex), you can download and run this repository: 20 | 21 | Get the code from Github: 22 | 23 | ``` 24 | $ git clone https://github.com/sharelatex/github-latex-ci.git 25 | ``` 26 | 27 | Install the required dependencies: 28 | 29 | ``` 30 | $ npm install 31 | ``` 32 | 33 | Run it: 34 | 35 | ``` 36 | $ grunt run 37 | ``` 38 | 39 | FAQ 40 | --- 41 | 42 | Q. Can I compile private repositories? 43 | 44 | A. No. To keep things simple, we only support public repositories. These do not need any authentication logic, since we can fetch the files without needing to authentication ourselves to github. To support private repositories we would need to keep a record of your github OAuth login, and then provide our own login/authentication system to make sure only you could view the PDF. This is beyond the scope of the project at the moment. 45 | 46 | Q. Do you support package X? 47 | 48 | A. If it's not included in the compile environment by default, you can upload a package to your GitHub repository and the ShareLaTeX compiler will find it. 49 | -------------------------------------------------------------------------------- /app/coffee/RepositoryController.coffee: -------------------------------------------------------------------------------- 1 | logger = require "logger-sharelatex" 2 | settings = require "settings-sharelatex" 3 | metrics = require "metrics-sharelatex" 4 | request = require "request" 5 | 6 | RepositoryManager = require "./RepositoryManager" 7 | BuildManager = require "./BuildManager" 8 | 9 | {publicUrl, mountPoint, userAgent} = settings.internal.github_latex_ci 10 | 11 | module.exports = RepositoryController = 12 | list: (req, res, next) -> 13 | RepositoryManager.gitReposOnGithub req.ghclient, (error, repos) -> 14 | return next(error) if error? 15 | RepositoryManager.injectWebHookStatus repos, (error, repos) -> 16 | return next(error) if error? 17 | res.render "repos/list", 18 | activeRepos: repos.filter (r) -> r.webhook 19 | otherRepos: repos.filter (r) -> !r.webhook 20 | csrfToken: req.csrfToken() 21 | 22 | show: (req, res, next) -> 23 | {owner, repo} = req.params 24 | repo = "#{owner}/#{repo}" 25 | BuildManager.getAllBuilds repo, (error, allBuilds) -> 26 | return next(error) if error? 27 | RepositoryManager.getLatestCommit repo, (error, sha) -> 28 | return next(error) if error? 29 | BuildManager.getBuildAndOutputFiles repo, sha, (error, latestBuild, outputFiles) -> 30 | return next(error) if error? 31 | res.render "repos/show", 32 | repo: repo, 33 | builds: allBuilds, 34 | latestBuild: latestBuild, 35 | outputFiles: outputFiles, 36 | csrfToken: req.csrfToken() 37 | 38 | proxyBlob: (req, res, next) -> 39 | url = req.url 40 | url = url.slice(mountPoint.length) 41 | metrics.inc "github-api-requests" 42 | request.get({ 43 | uri: "https://api.github.com#{url}", 44 | qs: { 45 | client_id: settings.github.client_id 46 | client_secret: settings.github.client_secret 47 | }, 48 | headers: 49 | "Accept": "application/vnd.github.v3.raw" 50 | "User-Agent": userAgent 51 | }).pipe(res) 52 | -------------------------------------------------------------------------------- /test/unit/coffee/RepositoryManagerTests.coffee: -------------------------------------------------------------------------------- 1 | assert = require("assert") 2 | sandboxedModule = require "sandboxed-module" 3 | modulePath = "../../../app/js/RepositoryManager" 4 | sinon = require "sinon" 5 | chai = require "chai" 6 | chai.should() 7 | 8 | describe "RespositoryController", -> 9 | beforeEach -> 10 | @RepositoryManager = sandboxedModule.require modulePath, requires: 11 | "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } 12 | "settings-sharelatex": 13 | internal: github_latex_ci: { publicUrl: "http://example.com", mountPoint: "/github" } 14 | "./mongojs": {} 15 | "./WebHookManager": @WebHookManager = {} 16 | @callback = sinon.stub() 17 | @ghclient = {} 18 | 19 | describe "gitReposOnGithub", -> 20 | it "should return all personal and organisation repos", (done)-> 21 | personalRepos = [{ 22 | full_name: "me/repo1" 23 | }, { 24 | full_name: "me/repo2" 25 | }, { 26 | full_name: "org1/repo1" 27 | }, { 28 | full_name: "org1/repo2" 29 | }, { 30 | full_name: "org2/repo1" 31 | }, { 32 | full_name: "org2/repo2" 33 | }] 34 | 35 | @ghclient.me = -> 36 | return repos: (opts, cb)-> 37 | cb(null, personalRepos) 38 | 39 | @RepositoryManager.gitReposOnGithub @ghclient, (err, returnedRepos)-> 40 | assert.deepEqual returnedRepos, personalRepos 41 | assert.equal err, undefined 42 | done() 43 | 44 | describe "injectWebhookStatus", -> 45 | it "should set webhook = true on repos with webhooks", (done) -> 46 | repos = [{ 47 | full_name: "me/repo1" 48 | }, { 49 | full_name: "me/repo2" 50 | }, { 51 | full_name: "me/repo3" 52 | }] 53 | 54 | @RepositoryManager.getRepos = sinon.stub().callsArgWith(1, null, [{ 55 | repo: "me/repo1" 56 | }, { 57 | repo: "me/repo3" 58 | }]) 59 | 60 | @RepositoryManager.injectWebHookStatus repos, (error, repos) -> 61 | repos.should.deep.equal [{ 62 | full_name: "me/repo1" 63 | webhook: true 64 | }, { 65 | full_name: "me/repo2" 66 | }, { 67 | full_name: "me/repo3" 68 | webhook: true 69 | }] 70 | done() 71 | -------------------------------------------------------------------------------- /app/coffee/AuthenticationController.coffee: -------------------------------------------------------------------------------- 1 | github = require "octonode" 2 | settings = require "settings-sharelatex" 3 | logger = require "logger-sharelatex" 4 | metrics = require "metrics-sharelatex" 5 | request = require "request" 6 | mountPoint = settings.internal.github_latex_ci.mountPoint 7 | auth = github.auth.config({ 8 | id: settings.github.client_id, 9 | secret: settings.github.client_secret 10 | }) 11 | 12 | module.exports = 13 | login: (req, res, next = (error) ->) -> 14 | auth_url = auth.login(['user:email', 'read:org', 'repo:status', 'admin:repo_hook']) 15 | req.session ||= {} 16 | req.session.state = auth_url.match(/&state=([0-9a-z]{32})/i)[1]; 17 | logger.log state: req.session.state, url: auth_url, "redirecting to github login page" 18 | res.redirect(auth_url) 19 | 20 | auth: (req, res, next = (error) ->) -> 21 | if !req.query.state? or req.query.state != req.session?.state 22 | logger.log received_state: req.query.state, stored_state: req.session?.state, "/auth CSRF check failed" 23 | res.status(403).send() 24 | else 25 | code = req.query.code 26 | logger.log code: code, "getting access_token from github" 27 | auth.login code, (error, token) -> 28 | return next(error) if error? 29 | req.session.token = token 30 | res.redirect "#{mountPoint}/repos" 31 | 32 | setupLoginStatus: (req, res, next = (error) ->) -> 33 | 34 | baseRequest = request.defaults({ 35 | headers: {'Accept':'application/vnd.github.moondragon-preview+json'} 36 | }) 37 | if req.session?.token? 38 | res.locals.loggedIn = req.loggedIn = true 39 | req.ghclient = github.client(req.session.token, {request:baseRequest}) 40 | else 41 | res.locals.loggedIn = req.loggedIn = false 42 | 43 | req.ghclient = github.client({id: settings.github.client_id, secret: settings.github.client_secret},{request:baseRequest}) 44 | 45 | if !req.ghclient._buildUrl? 46 | req.ghclient._buildUrl = req.ghclient.buildUrl 47 | req.ghclient.buildUrl = () -> 48 | metrics.inc "github-api-requests" 49 | req.ghclient._buildUrl.apply(req.ghclient, arguments) 50 | 51 | next() 52 | 53 | requireLogin: (req, res, next = (error) ->) -> 54 | if req.loggedIn 55 | next() 56 | else 57 | res.redirect("#{mountPoint}/login") -------------------------------------------------------------------------------- /app/coffee/WebHookController.coffee: -------------------------------------------------------------------------------- 1 | logger = require "logger-sharelatex" 2 | WebHookManager = require "./WebHookManager" 3 | RepositoryManager = require "./RepositoryManager" 4 | BuildManager = require "./BuildManager" 5 | 6 | settings = require "settings-sharelatex" 7 | {mountPoint} = settings.internal.github_latex_ci 8 | 9 | module.exports = WebHookController = 10 | createHook: (req, res, next) -> 11 | {owner, repo} = req.params 12 | repo = "#{owner}/#{repo}" 13 | logger.log repo: repo, "creating web hook" 14 | WebHookManager.createWebHook req.ghclient, repo, (error, response) -> 15 | return next(error) if error? 16 | RepositoryManager.saveWebHook repo, response.id, (error) -> 17 | return next(error) if error? 18 | logger.log repo: repo, hook_id: response.id, "created web hook" 19 | res.redirect("#{mountPoint}/repos") 20 | 21 | destroyHook: (req, res, next) -> 22 | {owner, repo} = req.params 23 | repo_name = "#{owner}/#{repo}" 24 | RepositoryManager.getRepo repo_name, (error, repo) -> 25 | return next(error) if error? 26 | WebHookManager.destroyWebHook req.ghclient, repo_name, repo.hook_id, (error) -> 27 | return next(error) if error? 28 | RepositoryManager.deleteRepo repo_name, (error) -> 29 | return next(error) if error? 30 | logger.log repo: repo_name, hook_id: repo.hook_id, "destroyed web hook" 31 | res.redirect("#{mountPoint}/repos") 32 | 33 | webHookEvent: (req, res, next) -> 34 | body = "" 35 | req.setEncoding "utf8" 36 | req.on "data", (chunk) -> body += chunk 37 | req.on "end", -> 38 | valid = WebHookManager.verifyWebHookEvent req.headers['x-hub-signature'], body 39 | if !valid 40 | res.status(403).end() 41 | else if req.headers['x-github-event'] != "push" 42 | logger.log event: req.headers['x-github-event'], "webhook event was not a push" 43 | res.status(200).end() 44 | else 45 | try 46 | data = JSON.parse(body) 47 | catch 48 | data = {} 49 | sha = data.after 50 | repo = data.repository.full_name 51 | logger.log repo: repo, commit: sha, "got push webhook request" 52 | RepositoryManager.setLatestCommit repo, sha, (error) -> 53 | return next(error) if error? 54 | BuildManager.startBuildingCommitInBackground req.ghclient, repo, sha, (error) -> 55 | return next(error) if error? 56 | res.status(200).end() 57 | 58 | -------------------------------------------------------------------------------- /app/coffee/RepositoryManager.coffee: -------------------------------------------------------------------------------- 1 | logger = require "logger-sharelatex" 2 | settings = require "settings-sharelatex" 3 | async = require "async" 4 | WebHookManager = require "./WebHookManager" 5 | {db, ObjectId} = require "./mongojs" 6 | 7 | module.exports = RepositoryManager = 8 | gitReposOnGithub: (ghclient, callback = (error, repos) ->) -> 9 | pageSize = 100 10 | 11 | populateRepos = (page = 1, repos = [], cb)-> 12 | ghclient.me().repos page: page, per_page: pageSize, (error, myRepos) -> 13 | return callback(error) if error? 14 | repos = repos.concat(myRepos) 15 | hasMore = myRepos.length == pageSize 16 | if hasMore 17 | populateRepos(++page, repos, cb) 18 | else 19 | cb(error, repos) 20 | 21 | populateRepos 1, [], (err, repos)-> 22 | return callback(error) if error? 23 | callback null, repos 24 | 25 | injectWebHookStatus: (repos, callback = (error, repos) ->) -> 26 | RepositoryManager.getRepos repos.map((r) -> r.full_name), (error, webhooks) -> 27 | return callback(error) if error? 28 | webhooksDict = {} 29 | for webhook in webhooks 30 | webhooksDict[webhook.repo] = webhook 31 | for repo in repos 32 | if webhooksDict[repo.full_name] 33 | repo.webhook = true 34 | callback null, repos 35 | 36 | getLatestCommitOnGitHub: (ghclient, repo, callback = (error, sha) ->) -> 37 | ghclient.repo(repo).branch "master", (error, branch) -> 38 | return callback(error) if error? 39 | callback null, branch?.commit?.sha 40 | 41 | _getOrgs: (ghclient, callback = (error, orgs) ->) -> 42 | ghclient.me().orgs callback 43 | 44 | saveWebHook: (repo, id, callback = (error) ->) -> 45 | db.githubRepos.update({ 46 | repo: repo 47 | }, { 48 | $set: 49 | hook_id: id 50 | }, { 51 | upsert: true 52 | }, callback) 53 | 54 | deleteRepo: (repo, callback = (error) ->) -> 55 | db.githubRepos.remove({ 56 | repo: repo 57 | }, callback) 58 | 59 | getRepo: (repo_name, callback = (error, repo) ->) -> 60 | db.githubRepos.find { 61 | repo: repo_name 62 | }, (error, repos = []) -> 63 | callback error, repos[0] 64 | 65 | getRepos: (repo_names, callback = (error, repos) ->) -> 66 | db.githubRepos.find({ 67 | repo: { $in: repo_names } 68 | }, callback) 69 | 70 | setLatestCommit: (repo, sha, callback = (error) ->) -> 71 | db.githubRepos.update({ 72 | repo: repo 73 | }, { 74 | $set: { latest_commit: sha } 75 | }, callback) 76 | 77 | getLatestCommit: (repo, callback = (error, sha) ->) -> 78 | db.githubRepos.find { 79 | repo: repo 80 | }, (error, repos = []) -> 81 | callback error, repos[0]?.latest_commit 82 | -------------------------------------------------------------------------------- /Gruntfile.coffee: -------------------------------------------------------------------------------- 1 | module.exports = (grunt) -> 2 | grunt.initConfig 3 | forever: 4 | app: 5 | options: 6 | index: "app.js" 7 | 8 | coffee: 9 | app_src: 10 | expand: true, 11 | flatten: true, 12 | cwd: "app" 13 | src: ['coffee/*.coffee'], 14 | dest: 'app/js/', 15 | ext: '.js' 16 | 17 | app: 18 | src: "app.coffee" 19 | dest: "app.js" 20 | 21 | unit_tests: 22 | expand: true 23 | cwd: "test/unit/coffee" 24 | src: ["**/*.coffee"] 25 | dest: "test/unit/js/" 26 | ext: ".js" 27 | 28 | acceptance_tests: 29 | expand: true 30 | cwd: "test/acceptance/coffee" 31 | src: ["**/*.coffee"] 32 | dest: "test/acceptance/js/" 33 | ext: ".js" 34 | 35 | smoke_tests: 36 | expand: true 37 | cwd: "test/smoke/coffee" 38 | src: ["**/*.coffee"] 39 | dest: "test/smoke/js" 40 | ext: ".js" 41 | 42 | clean: 43 | app: ["app/js/"] 44 | unit_tests: ["test/unit/js"] 45 | acceptance_tests: ["test/acceptance/js"] 46 | smoke_tests: ["test/smoke/js"] 47 | 48 | execute: 49 | app: 50 | src: "app.js" 51 | 52 | mochaTest: 53 | unit: 54 | options: 55 | reporter: grunt.option('reporter') or 'spec' 56 | src: ["test/unit/js/**/*.js"] 57 | acceptance: 58 | options: 59 | reporter: grunt.option('reporter') or 'spec' 60 | timeout: 40000 61 | grep: grunt.option("grep") 62 | src: ["test/acceptance/js/**/*.js"] 63 | smoke: 64 | options: 65 | reporter: grunt.option('reporter') or 'spec' 66 | timeout: 10000 67 | src: ["test/smoke/js/**/*.js"] 68 | 69 | grunt.loadNpmTasks 'grunt-contrib-coffee' 70 | grunt.loadNpmTasks 'grunt-contrib-clean' 71 | grunt.loadNpmTasks 'grunt-mocha-test' 72 | grunt.loadNpmTasks 'grunt-shell' 73 | grunt.loadNpmTasks 'grunt-execute' 74 | grunt.loadNpmTasks 'grunt-bunyan' 75 | grunt.loadNpmTasks 'grunt-forever' 76 | 77 | grunt.registerTask 'compile:app', ['clean:app', 'coffee:app', 'coffee:app_src'] 78 | grunt.registerTask 'run', ['compile:app', 'bunyan', 'execute'] 79 | 80 | grunt.registerTask 'compile:unit_tests', ['clean:unit_tests', 'coffee:unit_tests'] 81 | grunt.registerTask 'test:unit', ['compile:app', 'compile:unit_tests', 'mochaTest:unit'] 82 | 83 | grunt.registerTask 'compile:acceptance_tests', ['clean:acceptance_tests', 'coffee:acceptance_tests'] 84 | grunt.registerTask 'test:acceptance', ['compile:acceptance_tests', 'mochaTest:acceptance'] 85 | 86 | grunt.registerTask 'compile:smoke_tests', ['clean:smoke_tests', 'coffee:smoke_tests'] 87 | grunt.registerTask 'test:smoke', ['compile:smoke_tests', 'mochaTest:smoke'] 88 | 89 | grunt.registerTask 'install', 'compile:app' 90 | 91 | grunt.registerTask 'default', ['run'] 92 | 93 | 94 | -------------------------------------------------------------------------------- /app/coffee/BuildController.coffee: -------------------------------------------------------------------------------- 1 | BuildManager = require "./BuildManager" 2 | RepositoryManager = require "./RepositoryManager" 3 | settings = require "settings-sharelatex" 4 | mountPoint = settings.internal.github_latex_ci.mountPoint 5 | mime = require('mime') 6 | mimeWhiteList = ["application/pdf", "application/octet-stream", "text/plain"] 7 | _ = require("lodash") 8 | 9 | module.exports = BuildController = 10 | buildLatestCommit: (req, res, next) -> 11 | {owner, repo} = req.params 12 | repo = "#{owner}/#{repo}" 13 | RepositoryManager.getLatestCommitOnGitHub req.ghclient, repo, (error, sha) -> 14 | return next(error) if error? 15 | RepositoryManager.setLatestCommit repo, sha, (error) -> 16 | return next(error) if error? 17 | req.params.sha = sha 18 | BuildController.buildCommit(req, res, next) 19 | 20 | buildCommit: (req, res, next) -> 21 | {sha, owner, repo} = req.params 22 | repo = "#{owner}/#{repo}" 23 | BuildManager.startBuildingCommitInBackground req.ghclient, repo, sha, (error) -> 24 | return next(error) if error? 25 | res.redirect "#{mountPoint}/repos/#{repo}" 26 | 27 | showBuild: (req, res, next) -> 28 | {sha, owner, repo} = req.params 29 | repo = "#{owner}/#{repo}" 30 | BuildManager.getBuild repo, sha, (error, build) -> 31 | return next(error) if error? 32 | BuildManager.getOutputFiles repo, sha, (error, outputFiles) -> 33 | return next(error) if error? 34 | res.render "builds/show", 35 | repo: repo, 36 | build: build, 37 | outputFiles: outputFiles 38 | 39 | downloadOutputFile: (req, res, next) -> 40 | {sha, owner, repo, path} = req.params 41 | repo = "#{owner}/#{repo}" 42 | BuildManager.getOutputFileStream repo, sha, path, (error, s3res) -> 43 | return next(error) if error? 44 | recomendedMime = mime.lookup(path) 45 | fileMime = if _.includes(mimeWhiteList, recomendedMime) then recomendedMime else "application/octet-stream" 46 | res.header "Content-Type", fileMime 47 | res.header("Content-Length", s3res.headers['content-length']) 48 | s3res.pipe(res) 49 | 50 | downloadLatestBuild: (req, res, next) -> 51 | {owner, repo} = req.params 52 | repo = "#{owner}/#{repo}" 53 | RepositoryManager.getLatestCommit repo, (error, sha) -> 54 | return next(error) if error? 55 | BuildManager.getBuild repo, sha, (error, build) -> 56 | return next(error) if error? 57 | if build? and build.status != "success" 58 | res.redirect "#{mountPoint}/repos/#{repo}" 59 | else 60 | res.redirect "#{mountPoint}/repos/#{repo}/builds/#{sha}/raw/output.pdf" 61 | 62 | 63 | latestPdfBadge: (req, res, next) -> 64 | {owner, repo} = req.params 65 | repo = "#{owner}/#{repo}" 66 | RepositoryManager.getLatestCommit repo, (error, sha) -> 67 | return next(error) if error? 68 | BuildManager.getBuild repo, sha, (error, build) -> 69 | return next(error) if error? 70 | res.header("Content-Type", "image/svg+xml") 71 | res.render "badges/pdf.jade", 72 | build: build 73 | -------------------------------------------------------------------------------- /app/views/repos/show.jade: -------------------------------------------------------------------------------- 1 | extends ../layout 2 | 3 | block content 4 | .card 5 | .page-header 6 | .pull-right 7 | a(href, data-toggle="modal", data-target="#badge-modal") 8 | img(src="#{settings.mountPoint}/repos/#{repo}/builds/latest/badge.svg") 9 | h1 10 | a(href="#{settings.mountPoint}/repos/#{repo}") #{repo} 11 | a(href="#{settings.mountPoint}/repos") ≪ All your projects 12 | 13 | div.modal.fade#badge-modal(tabindex="-1" role="dialog" aria-labelledby="badgeModal" aria-hidden="true") 14 | .modal-dialog 15 | .modal-content 16 | .modal-header 17 | h4 Badge Codes 18 | .modal-body 19 | h5 Badge Image URL 20 | pre(style="word-wrap: break-word; word-break: break-all;") #{settings.publicUrl}#{settings.mountPoint}/repos/#{repo}/builds/latest/badge.svg 21 | h5 Link to Latest Build 22 | pre(style="word-wrap: break-word; word-break: break-all;") #{settings.publicUrl}#{settings.mountPoint}/repos/#{repo}/builds/latest/output.pdf 23 | h5 Markdown 24 | pre(style="word-wrap: break-word; word-break: break-all;") [![PDF Status](#{settings.publicUrl}#{settings.mountPoint}/repos/#{repo}/builds/latest/badge.svg)](#{settings.publicUrl}#{settings.mountPoint}/repos/#{repo}/builds/latest/output.pdf) 25 | 26 | div.modal.fade#help-modal(tabindex="-1" role="dialog" aria-labelledby="helpModal" aria-hidden="true") 27 | .modal-dialog 28 | .modal-content 29 | .modal-body 30 | include ../_help 31 | 32 | ul.nav.nav-tabs(role="tablist") 33 | li.active 34 | a(href="#latestBuild", role="tab", data-toggle="tab") Latest 35 | li 36 | a(href="#previousBuilds", role="tab", data-toggle="tab") Previous Builds 37 | if loggedIn 38 | li.pull-right 39 | form(action="#{settings.mountPoint}/repos/#{repo}/builds/latest", method="POST") 40 | a(href, data-toggle="modal", data-target="#help-modal") Settings 41 | |          42 | input(type="hidden", name="_csrf", value=csrfToken) 43 | input.btn.btn-info(type="submit", value="Build Now") 44 | 45 | .tab-content 46 | .tab-pane.active#latestBuild 47 | - var build = latestBuild 48 | if (build) 49 | include ../builds/_build 50 | else 51 | p 52 | p.small No builds yet 53 | .tab-pane#previousBuilds 54 | table.table.table-hover.table-striped 55 | tbody 56 | each build in builds 57 | tr 58 | td 59 | .row 60 | .col-xs-6 61 | - if (build.commit) 62 | a(href="#{settings.mountPoint}/repos/#{repo}/builds/#{build.sha}")= build.commit.message 63 | .small by #{build.commit.author.name} on #{build.commit.author.date} 64 | - else 65 | | Pending... 66 | strong.col-xs-6.text-right 67 | if build.status == "success" 68 | a.btn.btn-success(href="#{settings.mountPoint}/repos/#{repo}/builds/#{build.sha}/output/output.pdf") Download PDF 69 | else if build.status == "in_progress" 70 | a.btn.btn-info(disabled, href) Building... 71 | else 72 | a.btn.btn-danger(href="#{settings.mountPoint}/repos/#{repo}/builds/#{build.sha}") Failed :( 73 | -------------------------------------------------------------------------------- /app.coffee: -------------------------------------------------------------------------------- 1 | logger = require "logger-sharelatex" 2 | logger.initialize("github-latex-ci") 3 | 4 | metrics = require "metrics-sharelatex" 5 | metrics.initialize "github-latex-ci" 6 | metrics.memory.monitor(logger) 7 | metrics.memory.Check(logger) 8 | 9 | settings = require "settings-sharelatex" 10 | {mountPoint, publicUrl} = settings.internal.github_latex_ci 11 | 12 | express = require "express" 13 | app = express() 14 | 15 | app.use metrics.http.monitor(logger) 16 | 17 | csrf = require("csurf")() 18 | # We use forms to do POST requests, with a _csrf field. bodyParser takes 19 | # care of parsing these to req.body._csrf 20 | bodyParser = require("body-parser") 21 | 22 | 23 | redis = require("redis-sharelatex") 24 | rclient = redis.createClient(settings.redis.web) 25 | 26 | session = require('express-session') 27 | RedisStore = require('connect-redis')(session) 28 | 29 | sessionStore = new RedisStore(client:rclient) 30 | 31 | 32 | IndexController = require "./app/js/IndexController" 33 | AuthenticationController = require "./app/js/AuthenticationController" 34 | RepositoryController = require "./app/js/RepositoryController" 35 | WebHookController = require "./app/js/WebHookController" 36 | BuildController = require "./app/js/BuildController" 37 | 38 | app.set('views', './app/views') 39 | app.set('view engine', 'jade') 40 | 41 | app.set("mountPoint", mountPoint) 42 | app.set("publicUrl", publicUrl) 43 | app.set("style", settings.style) 44 | 45 | # Cookies and sessions. 46 | fiveDaysInMilliseconds = 5 * 24 * 60 * 60 * 1000 47 | app.use(session( 48 | store:sessionStore 49 | secret: settings.security.sessionSecret, 50 | name: settings.cookieName, 51 | cookie: 52 | domain: settings.cookieDomain 53 | maxAge: fiveDaysInMilliseconds 54 | secure: settings.secureCookie 55 | proxy: settings.behindProxy 56 | )) 57 | 58 | destroySession = (req, res, next) -> 59 | if req.session? 60 | req.session.destroy() 61 | next() 62 | 63 | app.use(AuthenticationController.setupLoginStatus) 64 | 65 | app.get("#{mountPoint}/", IndexController.index) 66 | 67 | app.get("#{mountPoint}/login", AuthenticationController.login) 68 | app.get("#{mountPoint}/auth", AuthenticationController.auth) 69 | 70 | app.get("#{mountPoint}/repos", csrf, AuthenticationController.requireLogin, RepositoryController.list) 71 | app.get("#{mountPoint}/repos/:owner/:repo", csrf, RepositoryController.show) 72 | app.get("#{mountPoint}/repos/:owner/:repo/git/blobs/:sha", RepositoryController.proxyBlob) 73 | 74 | app.post("#{mountPoint}/repos/:owner/:repo/hook", bodyParser(), csrf, AuthenticationController.requireLogin, WebHookController.createHook) 75 | app.post("#{mountPoint}/repos/:owner/:repo/hook/destroy", bodyParser(), csrf, AuthenticationController.requireLogin, WebHookController.destroyHook) 76 | app.post("#{mountPoint}/events", destroySession, WebHookController.webHookEvent) 77 | 78 | app.get("#{mountPoint}/repos/:owner/:repo/builds/:sha", BuildController.showBuild) 79 | app.get("#{mountPoint}/repos/:owner/:repo/builds/latest/badge.svg", BuildController.latestPdfBadge) 80 | 81 | # Note that ../output.pdf is a clever link which will redirect to the build page if there is no PDF, 82 | # while ../raw/output.pdf will fail if the PDF is not there. 83 | app.get("#{mountPoint}/repos/:owner/:repo/builds/latest/output.pdf", BuildController.downloadLatestBuild) 84 | app.get regex = new RegExp("^#{mountPoint.replace('/', '\/')}\/repos\/([^\/]+)\/([^\/]+)\/builds\/([^\/]+)\/raw\/(.*)$"), (req, res, next) -> 85 | params = { 86 | owner: req.params[0] 87 | repo: req.params[1] 88 | sha: req.params[2] 89 | path: req.params[3] 90 | } 91 | req.params = params 92 | next() 93 | , BuildController.downloadOutputFile 94 | 95 | app.post("#{mountPoint}/repos/:owner/:repo/builds/latest", bodyParser(), csrf, AuthenticationController.requireLogin, BuildController.buildLatestCommit) 96 | 97 | app.get "#{mountPoint}/status", destroySession, (req, res, next) -> 98 | res.send("github-latex-ci is alive") 99 | 100 | port = settings.internal.github_latex_ci.port 101 | host = settings.internal.github_latex_ci.host 102 | app.listen port, host, (error) -> 103 | throw error if error? 104 | logger.info "github-latex-ci starting up, listening on #{host}:#{port}" 105 | 106 | if global.gc? 107 | gcTimer = setInterval () -> 108 | global.gc() 109 | logger.log process.memoryUsage(), "global.gc" 110 | , 3 * oneMinute = 60 * 1000 111 | gcTimer.unref() -------------------------------------------------------------------------------- /test/unit/coffee/BuildManagerTests.coffee: -------------------------------------------------------------------------------- 1 | sandboxedModule = require "sandboxed-module" 2 | modulePath = "../../../app/js/BuildManager" 3 | sinon = require "sinon" 4 | chai = require "chai" 5 | chai.should() 6 | 7 | describe "BuildManager", -> 8 | beforeEach -> 9 | @BuildManager = sandboxedModule.require modulePath, requires: 10 | "logger-sharelatex": @logger = { log: sinon.stub(), error: sinon.stub() } 11 | "settings-sharelatex": 12 | internal: github_latex_ci: { url: "http://example.com", mountPoint: @mountPoint = "/github" } 13 | s3: { key: "", secret: "", github_latex_ci_bucket: "" } 14 | "request": {} 15 | "./mongojs": {} 16 | "metrics-sharelatex": {} 17 | "knox": @knox = createClient: () -> 18 | "js-yaml": require("js-yaml") # Slow so only load once 19 | @repo = "owner-id/repo-id" 20 | @sha = "mock-sha" 21 | @ghclient = "mock-ghclient" 22 | @callback = sinon.stub() 23 | 24 | describe "compileCommit", -> 25 | beforeEach -> 26 | @tree = { 27 | tree: [{ 28 | path: "main.tex" 29 | url: "https://api.github.com/repos/#{@repo}/git/blobs/#{@main_blob = "bedbc8228ccc23414c177b75a930c74c614a6e78"}" 30 | }, { 31 | path: "chapters/chapter1.tex" 32 | url: "https://api.github.com/repos/#{@repo}/git/blobs/#{@chapter_blob = "41202e1f414e699050aa631ee06117f1d04260a7"}" 33 | }] 34 | } 35 | @clsiReq = { 36 | compile: 37 | options: 38 | compiler: "pdflatex" 39 | rootResourcePath: "main.tex" 40 | resources: [{ 41 | path: "main.tex" 42 | url: "http://example.com/github/repos/#{@repo}/git/blobs/#{@main_blob = "bedbc8228ccc23414c177b75a930c74c614a6e78"}" 43 | }, { 44 | path: "chapters/chapter1.tex" 45 | url: "http://example.com/github/repos/#{@repo}/git/blobs/#{@chapter_blob = "41202e1f414e699050aa631ee06117f1d04260a7"}" 46 | }] 47 | } 48 | @clsiRes = { 49 | compile: 50 | status: "success" 51 | outputFiles: [{ 52 | "url": "http://localhost:3013/project/project-id/output/output.log", 53 | "type": "log" 54 | }, { 55 | "url": "http://localhost:3013/project/project-id/output/output.pdf", 56 | "type": "pdf" 57 | }] 58 | } 59 | @BuildManager._getTree = sinon.stub().callsArgWith(3, null, @tree) 60 | @BuildManager._createClsiRequest = sinon.stub().callsArgWith(1, null, @clsiReq) 61 | @BuildManager._sendClsiRequest = sinon.stub().callsArgWith(2, null, @clsiRes) 62 | 63 | @BuildManager.compileCommit(@ghclient, @repo, @sha, @callback) 64 | 65 | it "should get the tree from Github", -> 66 | @BuildManager._getTree 67 | .calledWith(@ghclient, @repo, @sha) 68 | .should.equal true 69 | 70 | it "should send a request to the CLSI", -> 71 | @BuildManager._sendClsiRequest 72 | .calledWith(@repo, @clsiReq) 73 | .should.equal true 74 | 75 | it "should return the status and output files", -> 76 | @callback 77 | .calledWith(null, @clsiRes.compile.status, @clsiRes.compile.outputFiles) 78 | .should.equal true 79 | 80 | describe "saveCompile", -> 81 | beforeEach -> 82 | @BuildManager._saveBuildInDatabase = sinon.stub().callsArg(4) 83 | @BuildManager._saveOutputFileToS3 = sinon.stub().callsArg(3) 84 | 85 | @BuildManager.saveCompile @repo, @sha, @commit = { 86 | message: "message", author: "author" 87 | }, @status = "success", @outputFiles = [{ 88 | "url": "http://localhost:3013/project/project-id/output/output.log", 89 | "type": "log" 90 | }, { 91 | "url": "http://localhost:3013/project/project-id/output/output.pdf", 92 | "type": "pdf" 93 | }], @callback 94 | 95 | it "should save the build in the database", -> 96 | @BuildManager._saveBuildInDatabase 97 | .calledWith(@repo, @sha, @commit, @status) 98 | .should.equal true 99 | 100 | it "should save each output file to S3", -> 101 | for outputFile in @outputFiles 102 | @BuildManager._saveOutputFileToS3 103 | .calledWith(@repo, @sha, outputFile.url) 104 | .should.equal true 105 | 106 | describe "_createClsiRequest", -> 107 | beforeEach -> 108 | @files = {} 109 | @BuildManager._getBlobContent = (url, callback) => 110 | path = url.slice("https://example.com/".length) 111 | callback null, @files[path] 112 | 113 | @treeFromFiles = (files) -> 114 | tree = [] 115 | for path, contents of files 116 | tree.push { 117 | path: path 118 | url: "https://example.com/#{path}" 119 | type: "blob" 120 | } 121 | return tree: tree 122 | 123 | describe 'with a root document determined by \documentclass', -> 124 | beforeEach (done) -> 125 | @files["chapter1.tex"] = """ 126 | Contents of chapter 1 127 | """ 128 | @files["thesis.tex"] = """ 129 | % Confused it with a comment first lien 130 | \\documentclass{article} 131 | \\begin{document} 132 | Hello world 133 | \\end{document} 134 | """ 135 | @BuildManager._createClsiRequest @treeFromFiles(@files), (error, @request) => 136 | done() 137 | 138 | it "should set the root document", -> 139 | @request.compile.rootResourcePath.should.equal "thesis.tex" 140 | 141 | describe 'with a compiler determined by % !TEX program = LuaLaTeX', -> 142 | beforeEach (done) -> 143 | @files["chapter1.tex"] = """ 144 | Contents of chapter 1 145 | """ 146 | @files["thesis.tex"] = """ 147 | \\documentclass{article} 148 | % !TEX program = LuaLaTeX 149 | \\begin{document} 150 | Hello world 151 | \\end{document} 152 | """ 153 | @BuildManager._createClsiRequest @treeFromFiles(@files), (error, @request) => 154 | done() 155 | 156 | it "should set the root document", -> 157 | @request.compile.options.compiler.should.equal "lualatex" 158 | 159 | describe 'with a compiler and root resource overridden by .latex.yml', -> 160 | beforeEach (done) -> 161 | @files["chapter1.tex"] = """ 162 | Contents of chapter 1 163 | """ 164 | @files["thesis.tex"] = """ 165 | \\documentclass{article} 166 | % !TEX program = LuaLaTeX 167 | \\begin{document} 168 | Hello world 169 | \\end{document} 170 | """ 171 | @files[".latex.yml"] = """ 172 | compiler: pdfLaTeX 173 | root_file: chapter1.tex 174 | """ 175 | @BuildManager._createClsiRequest @treeFromFiles(@files), (error, @request) => 176 | done() 177 | 178 | it "should set the root document", -> 179 | @request.compile.options.compiler.should.equal "pdflatex" 180 | @request.compile.rootResourcePath.should.equal "chapter1.tex" 181 | -------------------------------------------------------------------------------- /app/coffee/BuildManager.coffee: -------------------------------------------------------------------------------- 1 | request = require "request" 2 | logger = require "logger-sharelatex" 3 | settings = require "settings-sharelatex" 4 | metrics = require "metrics-sharelatex" 5 | async = require "async" 6 | knox = require "knox" 7 | yaml = require "js-yaml" 8 | {db, ObjectId} = require "./mongojs" 9 | 10 | {url, mountPoint, userAgent} = settings.internal.github_latex_ci 11 | 12 | s3client = knox.createClient { 13 | key: settings.s3.key 14 | secret: settings.s3.secret 15 | bucket: settings.s3.github_latex_ci_bucket 16 | } 17 | 18 | module.exports = BuildManager = 19 | startBuildingCommitInBackground: (ghclient, repo, sha, callback = (error) ->) -> 20 | BuildManager.markBuildAsInProgress repo, sha, (error) -> 21 | return callback(error) if error? 22 | # Build in the background 23 | BuildManager.buildCommit ghclient, repo, sha, (error) -> 24 | if error? 25 | logger.error err:error, repo: repo, sha: sha, "background build failed" 26 | callback() 27 | 28 | buildCommit: (ghclient, repo, sha, callback = (error) ->) -> 29 | BuildManager._getCommit ghclient, repo, sha, (error, commit) -> 30 | return callback(error) if error? 31 | commit = 32 | message: commit.commit.message 33 | author: commit.commit.author 34 | logger.log commit: commit, repo: repo, sha: sha, "building repo" 35 | BuildManager.compileCommit ghclient, repo, sha, (error, status, outputFiles) -> 36 | return callback(error) if error? 37 | BuildManager.saveCompile repo, sha, commit, status, outputFiles, callback 38 | 39 | compileCommit: (ghclient, repo, sha, callback = (error, status, outputFiles) ->) -> 40 | BuildManager._getTree ghclient, repo, sha, (error, tree) -> 41 | return callback(error) if error? 42 | BuildManager._createClsiRequest tree, (error, clsiRequest) -> 43 | return callback(error) if error? 44 | logger.log clsiRequest: clsiRequest, "sending CLSI request" 45 | BuildManager._sendClsiRequest repo, clsiRequest, (error, response) -> 46 | return callback(error) if error? 47 | logger.log response: response, "got CLSI response" 48 | callback(null, response?.compile?.status, response?.compile?.outputFiles) 49 | 50 | saveCompile: (repo, sha, commit, status, outputFiles, callback = (error) ->) -> 51 | BuildManager._saveBuildInDatabase repo, sha, commit, status, (error) -> 52 | return callback(error) if error? 53 | 54 | jobs = [] 55 | for file in outputFiles or [] 56 | do (file) -> 57 | jobs.push (cb) -> 58 | if !file.url? 59 | cb() 60 | else 61 | BuildManager._saveOutputFileToS3 repo, sha, file.url, cb 62 | 63 | 64 | async.parallelLimit jobs, 5, callback 65 | 66 | getBuildAndOutputFiles: (repo, sha, callback = (error, build, outputFiles) ->) -> 67 | BuildManager.getBuild repo, sha, (error, build) -> 68 | return callback(error) if error? 69 | return callback(null, null, null) if !build? 70 | BuildManager.getOutputFiles repo, sha, (error, outputFiles) -> 71 | callback error, build, outputFiles 72 | 73 | getAllBuilds: (repo, callback = (error, builds) ->) -> 74 | db.githubBuilds.find({ 75 | repo: repo 76 | }).sort({ 77 | "commit.author.date": -1 78 | }, callback) 79 | 80 | getBuild: (repo, sha, callback = (error, builds) ->) -> 81 | db.githubBuilds.find { 82 | repo: repo 83 | sha: sha 84 | }, (error, builds = []) -> 85 | return callback(error) if error? 86 | callback null, builds[0] 87 | 88 | markBuildAsInProgress: (repo, sha, callback = (error) ->) -> 89 | BuildManager._saveBuildInDatabase repo, sha, null, "in_progress", callback 90 | 91 | getOutputFiles: (repo, sha, callback = (error, files) ->) -> 92 | prefix = "#{repo}/#{sha}" 93 | logger.log prefix: prefix, "listing output files" 94 | s3client.list { prefix: prefix }, (error, data) -> 95 | return callback(error) if error? 96 | files = [] 97 | for file in data.Contents 98 | files.push file.Key.slice(prefix.length + 1) 99 | callback null, files 100 | 101 | getOutputFileStream: (repo, sha, name, callback = (error, res) ->) -> 102 | path = "#{repo}/#{sha}/#{name}" 103 | s3client.getFile path, callback 104 | 105 | _getTree: (ghclient, repo, sha, callback = (error, tree) ->) -> 106 | ghclient.repo(repo).tree(sha, true, callback) 107 | 108 | _getCommit: (ghclient, repo, sha, callback = (error, commit) ->) -> 109 | ghclient.repo(repo).commit(sha, callback) 110 | 111 | _createClsiRequest: (tree, callback = (error, clsiRequest) ->) -> 112 | resources = [] 113 | jobs = [] 114 | 115 | docClassRootResourcePath = null 116 | ymlRootResourcePath = null 117 | texCompiler = null 118 | ymlCompiler = null 119 | 120 | logger.log tree: tree, "building CLSI request for Github tree" 121 | 122 | for entry in tree.tree or [] 123 | do (entry) -> 124 | if entry.type == "blob" 125 | jobs.push (callback) -> 126 | if entry.path.match(/\.tex$/) 127 | BuildManager._getBlobContent entry.url, (error, content) -> 128 | return callback(error) if error? 129 | resources.push { 130 | path: entry.path, 131 | content: content 132 | } 133 | 134 | if content.match(/^\s*\\documentclass/m) 135 | docClassRootResourcePath = entry.path 136 | 137 | if (m = content.match(/\%\s*!TEX\s*(?:TS-)?program\s*=\s*(.*)$/m)) 138 | texCompiler = BuildManager._canonicaliseCompiler(m[1]) 139 | 140 | callback() 141 | else if entry.path == ".latex.yml" 142 | BuildManager._getBlobContent entry.url, (error, content) -> 143 | if error? 144 | data = {} 145 | else 146 | try 147 | data = yaml.safeLoad content 148 | catch 149 | data = {} 150 | ymlRootResourcePath = data?['root_file'] 151 | if data['compiler']? 152 | ymlCompiler = BuildManager._canonicaliseCompiler(data['compiler']) 153 | callback() 154 | else 155 | resources.push { 156 | path: entry.path 157 | url: entry.url.replace(/^https:\/\/api\.github\.com/, url + mountPoint) 158 | modified: new Date(0) # The blob sha is a unique id for the content so cache forever 159 | } 160 | callback() 161 | 162 | async.series jobs, (error) -> 163 | return callback(error) if error? 164 | 165 | clsiRequest = 166 | compile: 167 | options: 168 | compiler: ymlCompiler or texCompiler or "pdflatex" 169 | rootResourcePath: ymlRootResourcePath or docClassRootResourcePath or "main.tex" 170 | resources: resources 171 | 172 | callback null, clsiRequest 173 | 174 | _canonicaliseCompiler: (compiler) -> 175 | COMPILERS = { 176 | 'pdflatex': 'pdflatex' 177 | 'latex': 'latex' 178 | 'luatex': 'lualatex' 179 | 'lualatex': 'lualatex' 180 | 'xetex': 'xelatex' 181 | 'xelatex': 'xelatex' 182 | } 183 | return COMPILERS[compiler.toString().trim().toLowerCase()] or "pdflatex" 184 | 185 | _getBlobContent: (url, callback = (error, content) ->) -> 186 | metrics.inc "github-api-requests" 187 | request.get { 188 | uri: url, 189 | qs: { 190 | client_id: settings.github.client_id 191 | client_secret: settings.github.client_secret 192 | }, 193 | headers: { 194 | "Accept": "application/vnd.github.v3.raw" 195 | "User-Agent": userAgent 196 | } 197 | }, (error, response, body) -> 198 | callback error, body 199 | 200 | _sendClsiRequest: (repo, req, callback = (error, data) ->) -> 201 | repo = repo.replace(/\//g, "-") 202 | request.post { 203 | uri: "#{settings.apis.clsi.url}/project/#{repo}/compile" 204 | json: req 205 | }, (error, response, body) -> 206 | return callback(error) if error? 207 | callback null, body 208 | 209 | _saveBuildInDatabase: (repo, sha, commit, status, callback = (error) ->) -> 210 | db.githubBuilds.update({ 211 | repo: repo 212 | sha: sha 213 | }, { 214 | $set: 215 | status: status 216 | commit: commit 217 | }, { 218 | upsert: true 219 | }, callback) 220 | 221 | _saveOutputFileToS3: (repo, sha, sourceUrl, callback = (error) ->) -> 222 | m = sourceUrl.match(/\/project\/[a-zA-Z0-9_-]+(\/build\/[a-f0-9-]+)?\/output\/(.*)$/) 223 | if !m? or m.length < 1 224 | return callback() 225 | name = m[2] 226 | name = "#{repo}/#{sha}/#{name}" 227 | 228 | logger.log url: sourceUrl, location: name, "saving output file" 229 | 230 | req = request.get sourceUrl 231 | req.on "response", (res) -> 232 | headers = { 233 | 'Content-Length': res.headers['content-length'] 234 | 'Content-Type': res.headers['content-type'] 235 | } 236 | logger.log location: name, content_length: res.headers['content-length'], "streaming to S3" 237 | s3client.putStream res, name, headers, (error, s3Req) -> 238 | return callback(error) if error? 239 | s3Req.resume() 240 | s3Req.on "end", callback 241 | 242 | --------------------------------------------------------------------------------