├── .gitignore ├── Dockerfile ├── Dockerrun.aws.json ├── diff-static ├── diff2html.min.css ├── diff2html.min.js └── github.min.css ├── edgar-monitor.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | config.json 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | 3 | RUN apt-get update -y 4 | 5 | # install system deps 6 | RUN apt-get install -y git curl w3m 7 | 8 | # install node 9 | RUN curl -sL https://deb.nodesource.com/setup_4.x | sudo -E bash - 10 | RUN apt-get install -y nodejs 11 | 12 | # install rss monitor and edgar monitor 13 | RUN git clone https://github.com/buzzfeed-openlab/rss-puppy.git /opt/rss-puppy 14 | COPY . /opt/edgar-monitor 15 | 16 | # install deps 17 | RUN cd /opt/edgar-monitor; npm install 18 | RUN cd /opt/rss-puppy; npm install 19 | 20 | EXPOSE 7654 21 | 22 | # run 23 | CMD ["node", "/opt/rss-puppy/run.js", "/opt/edgar-monitor/config.json"] 24 | -------------------------------------------------------------------------------- /Dockerrun.aws.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSEBDockerrunVersion": "1", 3 | "Volumes": [ 4 | ], 5 | "Logging": "/var/eb_log" 6 | } -------------------------------------------------------------------------------- /diff-static/diff2html.min.css: -------------------------------------------------------------------------------- 1 | .d2h-wrapper{display:block;margin:0 auto;text-align:left;width:100%}.d2h-file-wrapper{border:1px solid #ddd;border-radius:3px;margin-bottom:1em}.d2h-file-header{padding:5px 10px;border-bottom:1px solid #d8d8d8;background-color:#f7f7f7;font:13px Helvetica,arial,freesans,clean,sans-serif,"Segoe UI Emoji","Segoe UI Symbol"}.d2h-file-stats{display:inline;font-size:12px;text-align:center;max-width:15%}.d2h-lines-added{text-align:right}.d2h-lines-added>*{background-color:#ceffce;border:1px solid #b4e2b4;color:#399839;border-radius:5px 0 0 5px;padding:2px;width:25px}.d2h-lines-deleted{text-align:left}.d2h-lines-deleted>*{background-color:#f7c8c8;border:1px solid #e9aeae;color:#c33;border-radius:0 5px 5px 0;padding:2px;width:25px}.d2h-file-name{display:inline;height:33px;line-height:33px;max-width:80%;white-space:nowrap;text-overflow:ellipsis;overflow:hidden}.d2h-diff-table{border-collapse:collapse;font-family:Consolas,"Liberation Mono",Menlo,Courier,monospace;font-size:12px;height:18px;line-height:18px;width:100%}.d2h-files-diff{width:100%}.d2h-file-diff{overflow-x:scroll;overflow-y:hidden}.d2h-file-side-diff{display:inline-block;overflow-x:scroll;overflow-y:hidden;width:50%;margin-right:-4px}.d2h-code-line{display:block;white-space:pre;padding:0 10px;height:18px;line-height:18px;margin-left:80px;color:inherit;overflow-x:inherit;background:none}.d2h-code-side-line{display:block;white-space:pre;padding:0 10px;height:18px;line-height:18px;margin-left:50px;color:inherit;overflow-x:inherit;background:none}.d2h-code-line del,.d2h-code-side-line del{display:inline-block;margin-top:-1px;text-decoration:none;background-color:#ffb6ba;border-radius:.2em}.d2h-code-line ins,.d2h-code-side-line ins{display:inline-block;margin-top:-1px;text-decoration:none;background-color:#97f295;border-radius:.2em}.d2h-code-line-prefix{float:left;background:none;padding:0}.d2h-code-line-ctn{background:none;padding:0}.line-num1{display:inline-block;float:left;width:30px;overflow:hidden;text-overflow:ellipsis}.line-num2{display:inline-block;float:right;width:30px;overflow:hidden;text-overflow:ellipsis}.d2h-code-linenumber{position:absolute;width:2%;min-width:65px;padding-left:10px;padding-right:10px;height:18px;line-height:18px;background-color:#fff;color:rgba(0,0,0,0.3);text-align:right;border:solid #eeeeee;border-width:0 1px 0 1px;cursor:pointer}.d2h-code-side-linenumber{position:absolute;width:35px;padding-left:10px;padding-right:10px;height:18px;line-height:18px;background-color:#fff;color:rgba(0,0,0,0.3);text-align:right;border:solid #eeeeee;border-width:0 1px 0 1px;cursor:pointer;overflow:hidden;text-overflow:ellipsis}.d2h-del{background-color:#fee8e9;border-color:#e9aeae}.d2h-ins{background-color:#dfd;border-color:#b4e2b4}.d2h-info{background-color:#f8fafd;color:rgba(0,0,0,0.3);border-color:#d5e4f2}.d2h-file-list-wrapper{margin-bottom:10px;padding:0 10px}.d2h-file-list-wrapper a{text-decoration:none;color:#3572b0}.d2h-file-list-wrapper a:visited{color:#3572b0}.d2h-file-list-header{font-weight:bold;margin-bottom:5px;text-align:left;display:inline;float:left}.d2h-file-list-line{text-align:left;font:13px Helvetica,arial,freesans,clean,sans-serif,"Segoe UI Emoji","Segoe UI Symbol"}.d2h-file-list-line .d2h-file-name{line-height:21px}.d2h-file-list{display:none}.d2h-clear{display:block;clear:both}.d2h-show{display:none;float:left}.d2h-hide{float:left}.d2h-hide:target+.d2h-show{display:inline}.d2h-hide:target{display:none}.d2h-hide:target~.d2h-file-list{display:block} -------------------------------------------------------------------------------- /diff-static/diff2html.min.js: -------------------------------------------------------------------------------- 1 | !function(modules){function __webpack_require__(moduleId){if(installedModules[moduleId])return installedModules[moduleId].exports;var module=installedModules[moduleId]={exports:{},id:moduleId,loaded:!1};return modules[moduleId].call(module.exports,module,module.exports,__webpack_require__),module.loaded=!0,module.exports}var installedModules={};return __webpack_require__.m=modules,__webpack_require__.c=installedModules,__webpack_require__.p="",__webpack_require__(0)}([function(module,exports,__webpack_require__){(function(global){!function(){function Diff2Html(){}var diffParser=__webpack_require__(1).DiffParser,fileLister=__webpack_require__(3).FileListPrinter,htmlPrinter=__webpack_require__(20).HtmlPrinter;Diff2Html.prototype.getJsonFromDiff=function(diffInput){return diffParser.generateDiffJson(diffInput)},Diff2Html.prototype.getPrettyHtml=function(diffInput,config){var configOrEmpty=config||{},diffJson=diffInput;configOrEmpty.inputFormat&&"diff"!==configOrEmpty.inputFormat||(diffJson=diffParser.generateDiffJson(diffInput));var fileList="";configOrEmpty.showFiles===!0&&(fileList=fileLister.generateFileList(diffJson,configOrEmpty));var diffOutput="";return diffOutput="side-by-side"===configOrEmpty.outputFormat?htmlPrinter.generateSideBySideJsonHtml(diffJson,configOrEmpty):htmlPrinter.generateLineByLineJsonHtml(diffJson,configOrEmpty),fileList+diffOutput},Diff2Html.prototype.getPrettyHtmlFromDiff=function(diffInput,config){var configOrEmpty=config||{};return configOrEmpty.inputFormat="diff",configOrEmpty.outputFormat="line-by-line",this.getPrettyHtml(diffInput,configOrEmpty)},Diff2Html.prototype.getPrettyHtmlFromJson=function(diffJson,config){var configOrEmpty=config||{};return configOrEmpty.inputFormat="json",configOrEmpty.outputFormat="line-by-line",this.getPrettyHtml(diffJson,configOrEmpty)},Diff2Html.prototype.getPrettySideBySideHtmlFromDiff=function(diffInput,config){var configOrEmpty=config||{};return configOrEmpty.inputFormat="diff",configOrEmpty.outputFormat="side-by-side",this.getPrettyHtml(diffInput,configOrEmpty)},Diff2Html.prototype.getPrettySideBySideHtmlFromJson=function(diffJson,config){var configOrEmpty=config||{};return configOrEmpty.inputFormat="json",configOrEmpty.outputFormat="side-by-side",this.getPrettyHtml(diffJson,configOrEmpty)};var diffName="Diff2Html",diffObject=new Diff2Html;module.exports[diffName]=diffObject,global[diffName]=diffObject}(this)}).call(exports,function(){return this}())},function(module,exports,__webpack_require__){!function(){function DiffParser(){}function getExtension(filename,language){var nameSplit=filename.split(".");return nameSplit.length>1?nameSplit[nameSplit.length-1]:language}var utils=__webpack_require__(2).Utils,LINE_TYPE={INSERTS:"d2h-ins",DELETES:"d2h-del",CONTEXT:"d2h-cntx",INFO:"d2h-info"};DiffParser.prototype.LINE_TYPE=LINE_TYPE,DiffParser.prototype.generateDiffJson=function(diffInput){var files=[],currentFile=null,currentBlock=null,oldLine=null,newLine=null,saveBlock=function(){currentBlock&&(currentFile.blocks.push(currentBlock),currentBlock=null)},saveFile=function(){currentFile&¤tFile.newName&&(files.push(currentFile),currentFile=null)},startFile=function(){saveBlock(),saveFile(),currentFile={},currentFile.blocks=[],currentFile.deletedLines=0,currentFile.addedLines=0},startBlock=function(line){saveBlock();var values;(values=/^@@ -(\d+),\d+ \+(\d+),\d+ @@.*/.exec(line))?currentFile.isCombined=!1:(values=/^@@@ -(\d+),\d+ -\d+,\d+ \+(\d+),\d+ @@@.*/.exec(line))?currentFile.isCombined=!0:(values=[0,0],currentFile.isCombined=!1),oldLine=values[1],newLine=values[2],currentBlock={},currentBlock.lines=[],currentBlock.oldStartLine=oldLine,currentBlock.newStartLine=newLine,currentBlock.header=line},createLine=function(line){var currentLine={};currentLine.content=line;var newLinePrefixes=currentFile.isCombined?["+"," +"]:["+"],delLinePrefixes=currentFile.isCombined?["-"," -"]:["-"];utils.startsWith(line,newLinePrefixes)?(currentFile.addedLines++,currentLine.type=LINE_TYPE.INSERTS,currentLine.oldNumber=null,currentLine.newNumber=newLine++,currentBlock.lines.push(currentLine)):utils.startsWith(line,delLinePrefixes)?(currentFile.deletedLines++,currentLine.type=LINE_TYPE.DELETES,currentLine.oldNumber=oldLine++,currentLine.newNumber=null,currentBlock.lines.push(currentLine)):(currentLine.type=LINE_TYPE.CONTEXT,currentLine.oldNumber=oldLine++,currentLine.newNumber=newLine++,currentBlock.lines.push(currentLine))},diffLines=diffInput.split("\n"),oldMode=/^old mode (\d{6})/,newMode=/^new mode (\d{6})/,deletedFileMode=/^deleted file mode (\d{6})/,newFileMode=/^new file mode (\d{6})/,copyFrom=/^copy from (.+)/,copyTo=/^copy to (.+)/,renameFrom=/^rename from (.+)/,renameTo=/^rename to (.+)/,similarityIndex=/^similarity index (\d+)%/,dissimilarityIndex=/^dissimilarity index (\d+)%/,index=/^index ([0-9a-z]+)..([0-9a-z]+) (\d{6})?/,combinedIndex=/^index ([0-9a-z]+),([0-9a-z]+)..([0-9a-z]+)/,combinedMode=/^mode (\d{6}),(\d{6})..(\d{6})/,combinedNewFile=/^new file mode (\d{6})/,combinedDeletedFile=/^deleted file mode (\d{6}),(\d{6})/;return diffLines.forEach(function(line){if(line&&!utils.startsWith(line,"*")){var values=[];utils.startsWith(line,"diff")?startFile():currentFile&&!currentFile.oldName&&(values=/^--- [aiwco]\/(.+)$/.exec(line))?(currentFile.oldName=values[1],currentFile.language=getExtension(currentFile.oldName,currentFile.language)):currentFile&&!currentFile.newName&&(values=/^\+\+\+ [biwco]?\/(.+)$/.exec(line))?(currentFile.newName=values[1],currentFile.language=getExtension(currentFile.newName,currentFile.language)):currentFile&&utils.startsWith(line,"@@")?startBlock(line):(values=oldMode.exec(line))?currentFile.oldMode=values[1]:(values=newMode.exec(line))?currentFile.newMode=values[1]:(values=deletedFileMode.exec(line))?currentFile.deletedFileMode=values[1]:(values=newFileMode.exec(line))?currentFile.newFileMode=values[1]:(values=copyFrom.exec(line))?(currentFile.oldName=values[1],currentFile.isCopy=!0):(values=copyTo.exec(line))?(currentFile.newName=values[1],currentFile.isCopy=!0):(values=renameFrom.exec(line))?(currentFile.oldName=values[1],currentFile.isRename=!0):(values=renameTo.exec(line))?(currentFile.newName=values[1],currentFile.isRename=!0):(values=similarityIndex.exec(line))?currentFile.unchangedPercentage=values[1]:(values=dissimilarityIndex.exec(line))?currentFile.changedPercentage=values[1]:(values=index.exec(line))?(currentFile.checksumBefore=values[1],currentFile.checksumAfter=values[2],values[2]&&(currentFile.mode=values[3])):(values=combinedIndex.exec(line))?(currentFile.checksumBefore=[values[2],values[3]],currentFile.checksumAfter=values[1]):(values=combinedMode.exec(line))?(currentFile.oldMode=[values[2],values[3]],currentFile.newMode=values[1]):(values=combinedNewFile.exec(line))?currentFile.newFileMode=values[1]:(values=combinedDeletedFile.exec(line))?currentFile.deletedFileMode=values[1]:currentBlock&&createLine(line)}}),saveBlock(),saveFile(),files},module.exports.DiffParser=new DiffParser}(this)},function(module){!function(){function Utils(){}Utils.prototype.escape=function(str){return str.slice(0).replace(/&/g,"&").replace(//g,">").replace(/\t/g," ")},Utils.prototype.getRandomId=function(prefix){return prefix+"-"+Math.random().toString(36).slice(-3)},Utils.prototype.startsWith=function(str,start){if("object"==typeof start){var result=!1;return start.forEach(function(s){0===str.indexOf(s)&&(result=!0)}),result}return 0===str.indexOf(start)},Utils.prototype.valueOrEmpty=function(value){return value?value:""},module.exports.Utils=new Utils}(this)},function(module,exports,__webpack_require__){!function(){function FileListPrinter(){}var printerUtils=__webpack_require__(4).PrinterUtils,utils=__webpack_require__(2).Utils;FileListPrinter.prototype.generateFileList=function(diffFiles){var hideId=utils.getRandomId("d2h-hide"),showId=utils.getRandomId("d2h-show");return'
\n
Files changed ('+diffFiles.length+')  
\n +\n -\n
\n \n'+diffFiles.map(function(file){return' \n \n \n \n \n"}).join("\n")+"
\n +'+file.addedLines+'\n \n -'+file.deletedLines+'\n  '+printerUtils.getDiffName(file)+"
\n"},module.exports.FileListPrinter=new FileListPrinter}(this)},function(module,exports,__webpack_require__){!function(){function PrinterUtils(){}function isDeletedName(name){return"dev/null"===name}function removeIns(line){return line.replace(/(((.|\n)*?)<\/ins>)/g,"")}function removeDel(line){return line.replace(/(((.|\n)*?)<\/del>)/g,"")}var jsDiff=__webpack_require__(5),utils=__webpack_require__(2).Utils;PrinterUtils.prototype.getHtmlId=function(file){var hashCode=function(text){var i,chr,len,hash=0;if(0==text.length)return hash;for(i=0,len=text.length;len>i;i++)chr=text.charCodeAt(i),hash=(hash<<5)-hash+chr,hash|=0;return hash};return"d2h-"+hashCode(this.getDiffName(file)).toString().slice(-6)},PrinterUtils.prototype.getDiffName=function(file){var oldFilename=file.oldName,newFilename=file.newName;return oldFilename&&newFilename&&oldFilename!==newFilename&&!isDeletedName(newFilename)?oldFilename+" -> "+newFilename:newFilename&&!isDeletedName(newFilename)?newFilename:oldFilename?oldFilename:"Unknown filename"},PrinterUtils.prototype.diffHighlight=function(diffLine1,diffLine2,config){var lineStart1,lineStart2,prefixSize=1;config.isCombined&&(prefixSize=2),lineStart1=diffLine1.substr(0,prefixSize),lineStart2=diffLine2.substr(0,prefixSize),diffLine1=diffLine1.substr(prefixSize),diffLine2=diffLine2.substr(prefixSize);var diff;diff=config.charByChar?jsDiff.diffChars(diffLine1,diffLine2):jsDiff.diffWordsWithSpace(diffLine1,diffLine2);var highlightedLine="";return diff.forEach(function(part){var elemType=part.added?"ins":part.removed?"del":null,escapedValue=utils.escape(part.value);highlightedLine+=null!==elemType?"<"+elemType+">"+escapedValue+"":escapedValue}),{first:{prefix:lineStart1,line:removeIns(highlightedLine)},second:{prefix:lineStart2,line:removeDel(highlightedLine)}}},module.exports.PrinterUtils=new PrinterUtils}(this)},function(module,exports,__webpack_require__){"use strict";function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{"default":obj}}exports.__esModule=!0;var _diffBase=__webpack_require__(6),_diffBase2=_interopRequireDefault(_diffBase),_diffCharacter=__webpack_require__(7),_diffWord=__webpack_require__(8),_diffLine=__webpack_require__(10),_diffSentence=__webpack_require__(11),_diffCss=__webpack_require__(12),_diffJson=__webpack_require__(13),_patchApply=__webpack_require__(14),_patchParse=__webpack_require__(15),_patchCreate=__webpack_require__(17),_convertDmp=__webpack_require__(18),_convertXml=__webpack_require__(19);exports.Diff=_diffBase2["default"],exports.diffChars=_diffCharacter.diffChars,exports.diffWords=_diffWord.diffWords,exports.diffWordsWithSpace=_diffWord.diffWordsWithSpace,exports.diffLines=_diffLine.diffLines,exports.diffTrimmedLines=_diffLine.diffTrimmedLines,exports.diffSentences=_diffSentence.diffSentences,exports.diffCss=_diffCss.diffCss,exports.diffJson=_diffJson.diffJson,exports.structuredPatch=_patchCreate.structuredPatch,exports.createTwoFilesPatch=_patchCreate.createTwoFilesPatch,exports.createPatch=_patchCreate.createPatch,exports.applyPatch=_patchApply.applyPatch,exports.applyPatches=_patchApply.applyPatches,exports.parsePatch=_patchParse.parsePatch,exports.convertChangesToDMP=_convertDmp.convertChangesToDMP,exports.convertChangesToXML=_convertXml.convertChangesToXML,exports.canonicalize=_diffJson.canonicalize},function(module,exports){"use strict";function Diff(){}function buildValues(diff,components,newString,oldString,useLongestToken){for(var componentPos=0,componentLen=components.length,newPos=0,oldPos=0;componentLen>componentPos;componentPos++){var component=components[componentPos];if(component.removed){if(component.value=oldString.slice(oldPos,oldPos+component.count).join(""),oldPos+=component.count,componentPos&&components[componentPos-1].added){var tmp=components[componentPos-1];components[componentPos-1]=components[componentPos],components[componentPos]=tmp}}else{if(!component.added&&useLongestToken){var value=newString.slice(newPos,newPos+component.count);value=value.map(function(value,i){var oldValue=oldString[oldPos+i];return oldValue.length>value.length?oldValue:value}),component.value=value.join("")}else component.value=newString.slice(newPos,newPos+component.count).join("");newPos+=component.count,component.added||(oldPos+=component.count)}}var lastComponent=components[componentLen-1];return(lastComponent.added||lastComponent.removed)&&diff.equals("",lastComponent.value)&&(components[componentLen-2].value+=lastComponent.value,components.pop()),components}function clonePath(path){return{newPos:path.newPos,components:path.components.slice(0)}}exports.__esModule=!0,exports["default"]=Diff,Diff.prototype={diff:function(oldString,newString){function done(value){return callback?(setTimeout(function(){callback(void 0,value)},0),!0):value}function execEditLength(){for(var diagonalPath=-1*editLength;editLength>=diagonalPath;diagonalPath+=2){var basePath=void 0,addPath=bestPath[diagonalPath-1],removePath=bestPath[diagonalPath+1],_oldPos=(removePath?removePath.newPos:0)-diagonalPath;addPath&&(bestPath[diagonalPath-1]=void 0);var canAdd=addPath&&addPath.newPos+1=0&&oldLen>_oldPos;if(canAdd||canRemove){if(!canAdd||canRemove&&addPath.newPos=newLen&&_oldPos+1>=oldLen)return done(buildValues(self,basePath.components,newString,oldString,self.useLongestToken));bestPath[diagonalPath]=basePath}else bestPath[diagonalPath]=void 0}editLength++}var options=arguments.length<=2||void 0===arguments[2]?{}:arguments[2],callback=options.callback;"function"==typeof options&&(callback=options,options={}),this.options=options;var self=this;oldString=this.castInput(oldString),newString=this.castInput(newString),oldString=this.removeEmpty(this.tokenize(oldString)),newString=this.removeEmpty(this.tokenize(newString));var newLen=newString.length,oldLen=oldString.length,editLength=1,maxEditLength=newLen+oldLen,bestPath=[{newPos:-1,components:[]}],oldPos=this.extractCommon(bestPath[0],newString,oldString,0);if(bestPath[0].newPos+1>=newLen&&oldPos+1>=oldLen)return done([{value:newString.join(""),count:newString.length}]);if(callback)!function exec(){setTimeout(function(){return editLength>maxEditLength?callback():void(execEditLength()||exec())},0)}();else for(;maxEditLength>=editLength;){var ret=execEditLength();if(ret)return ret}},pushComponent:function(components,added,removed){var last=components[components.length-1];last&&last.added===added&&last.removed===removed?components[components.length-1]={count:last.count+1,added:added,removed:removed}:components.push({count:1,added:added,removed:removed})},extractCommon:function(basePath,newString,oldString,diagonalPath){for(var newLen=newString.length,oldLen=oldString.length,newPos=basePath.newPos,oldPos=newPos-diagonalPath,commonCount=0;newLen>newPos+1&&oldLen>oldPos+1&&this.equals(newString[newPos+1],oldString[oldPos+1]);)newPos++,oldPos++,commonCount++;return commonCount&&basePath.components.push({count:commonCount}),basePath.newPos=newPos,oldPos},equals:function(left,right){return left===right},removeEmpty:function(array){for(var ret=[],i=0;ifuzzFactor))return!1;toPos++}}return!0}var options=arguments.length<=2||void 0===arguments[2]?{}:arguments[2];if("string"==typeof uniDiff&&(uniDiff=_parse.parsePatch(uniDiff)),Array.isArray(uniDiff)){if(uniDiff.length>1)throw new Error("applyPatch only works with a single input.");uniDiff=uniDiff[0]}for(var lines=source.split("\n"),hunks=uniDiff.hunks,compareLine=options.compareLine||function(lineNumber,line,operation,patchContent){return line===patchContent},errorCount=0,fuzzFactor=options.fuzzFactor||0,minLine=0,offset=0,removeEOFNL=void 0,addEOFNL=void 0,i=0;i=start+localOffset)return localOffset;forwardExhausted=!0}if(backwardExhausted);else{if(forwardExhausted||(wantForward=!0),start-localOffset>=minLine)return-localOffset++;backwardExhausted=!0,_again=!0}}}},module.exports=exports["default"]},function(module,exports,__webpack_require__){"use strict";function _toConsumableArray(arr){if(Array.isArray(arr)){for(var i=0,arr2=Array(arr.length);i0?contextLines(prev.lines.slice(-options.context)):[],oldRangeStart-=curRange.length,newRangeStart-=curRange.length)}(_curRange=curRange).push.apply(_curRange,_toConsumableArray(lines.map(function(entry){return(current.added?"+":"-")+entry}))),current.added?newLine+=lines.length:oldLine+=lines.length}else{if(oldRangeStart)if(lines.length<=2*options.context&&i=diff.length-2&&lines.length<=options.context){var oldEOFNewline=/\n$/.test(oldStr),newEOFNewline=/\n$/.test(newStr);0!=lines.length||oldEOFNewline?oldEOFNewline&&newEOFNewline||curRange.push("\\ No newline at end of file"):curRange.splice(hunk.oldLines,0,"\\ No newline at end of file")}hunks.push(hunk),oldRangeStart=0,newRangeStart=0,curRange=[]}oldLine+=lines.length,newLine+=lines.length}},i=0;i"):change.removed&&ret.push(""),ret.push(escapeHTML(change.value)),change.added?ret.push(""):change.removed&&ret.push("")}return ret.join("")}function escapeHTML(s){var n=s;return n=n.replace(/&/g,"&"),n=n.replace(//g,">"),n=n.replace(/"/g,""")}exports.__esModule=!0,exports.convertChangesToXML=convertChangesToXML},function(module,exports,__webpack_require__){!function(){function HtmlPrinter(){}var lineByLinePrinter=__webpack_require__(21).LineByLinePrinter,sideBySidePrinter=__webpack_require__(22).SideBySidePrinter;HtmlPrinter.prototype.generateLineByLineJsonHtml=lineByLinePrinter.generateLineByLineJsonHtml,HtmlPrinter.prototype.generateSideBySideJsonHtml=sideBySidePrinter.generateSideBySideJsonHtml,module.exports.HtmlPrinter=new HtmlPrinter}(this)},function(module,exports,__webpack_require__){!function(){function LineByLinePrinter(){}function generateFileHtml(file,config){return file.blocks.map(function(block){ 2 | for(var lines='\n \n
'+utils.escape(block.header)+"
\n\n",oldLines=[],newLines=[],processedOldLines=[],processedNewLines=[],i=0;inewLines.length)newLines.push(line);else{var oldLine,newLine,j=0;if(oldLines.length===newLines.length){for(j=0;j");var htmlContent="";return content&&(htmlContent=''+content+""),'\n
'+utils.valueOrEmpty(oldNumber)+'
'+utils.valueOrEmpty(newNumber)+'
\n
'+htmlPrefix+htmlContent+"
\n\n"}function generateEmptyDiff(){return'\n
File without changes
\n\n'}var diffParser=__webpack_require__(1).DiffParser,printerUtils=__webpack_require__(4).PrinterUtils,utils=__webpack_require__(2).Utils;LineByLinePrinter.prototype.generateLineByLineJsonHtml=function(diffFiles,config){return'
\n'+diffFiles.map(function(file){var diffs;return diffs=file.blocks.length?generateFileHtml(file,config):generateEmptyDiff(),'
\n
\n
\n +'+file.addedLines+'\n \n -'+file.deletedLines+'\n \n
\n
'+printerUtils.getDiffName(file)+'
\n
\n
\n
\n \n \n '+diffs+" \n
\n
\n
\n
\n"}).join("\n")+"
\n"},module.exports.LineByLinePrinter=new LineByLinePrinter}(this)},function(module,exports,__webpack_require__){!function(){function SideBySidePrinter(){}function generateSideBySideFileHtml(file,config){var fileHtml={};return fileHtml.left="",fileHtml.right="",file.blocks.forEach(function(block){fileHtml.left+='\n \n
'+utils.escape(block.header)+"
\n\n",fileHtml.right+='\n \n
\n\n';for(var oldLines=[],newLines=[],tmpHtml="",i=0;inewLines.length)newLines.push(line);else{var oldLine,newLine,j=0;if(oldLines.length===newLines.length)for(j=0;j");var htmlContent="";return content&&(htmlContent=''+content+""),'\n '+number+'\n
'+htmlPrefix+htmlContent+"
\n \n"}function generateEmptyDiff(){var fileHtml={};return fileHtml.right="",fileHtml.left='\n
File without changes
\n\n',fileHtml}var diffParser=__webpack_require__(1).DiffParser,printerUtils=__webpack_require__(4).PrinterUtils,utils=__webpack_require__(2).Utils;SideBySidePrinter.prototype.generateSideBySideJsonHtml=function(diffFiles,config){return'
\n'+diffFiles.map(function(file){var diffs;return diffs=file.blocks.length?generateSideBySideFileHtml(file,config):generateEmptyDiff(),'
\n
\n
\n +'+file.addedLines+'\n \n -'+file.deletedLines+'\n \n
\n
'+printerUtils.getDiffName(file)+'
\n
\n
\n
\n
\n \n \n '+diffs.left+' \n
\n
\n
\n
\n
\n \n \n '+diffs.right+" \n
\n
\n
\n
\n
\n"}).join("\n")+"
\n"},module.exports.SideBySidePrinter=new SideBySidePrinter}(this)}]); -------------------------------------------------------------------------------- /diff-static/github.min.css: -------------------------------------------------------------------------------- 1 | .hljs{display:block;overflow-x:auto;padding:0.5em;color:#333;background:#f8f8f8;-webkit-text-size-adjust:none}.hljs-comment,.diff .hljs-header{color:#998;font-style:italic}.hljs-keyword,.css .rule .hljs-keyword,.hljs-winutils,.nginx .hljs-title,.hljs-subst,.hljs-request,.hljs-status{color:#333;font-weight:bold}.hljs-number,.hljs-hexcolor,.ruby .hljs-constant{color:#008080}.hljs-string,.hljs-tag .hljs-value,.hljs-doctag,.tex .hljs-formula{color:#d14}.hljs-title,.hljs-id,.scss .hljs-preprocessor{color:#900;font-weight:bold}.hljs-list .hljs-keyword,.hljs-subst{font-weight:normal}.hljs-class .hljs-title,.hljs-type,.vhdl .hljs-literal,.tex .hljs-command{color:#458;font-weight:bold}.hljs-tag,.hljs-tag .hljs-title,.hljs-rule .hljs-property,.django .hljs-tag .hljs-keyword{color:#000080;font-weight:normal}.hljs-attribute,.hljs-variable,.lisp .hljs-body,.hljs-name{color:#008080}.hljs-regexp{color:#009926}.hljs-symbol,.ruby .hljs-symbol .hljs-string,.lisp .hljs-keyword,.clojure .hljs-keyword,.scheme .hljs-keyword,.tex .hljs-special,.hljs-prompt{color:#990073}.hljs-built_in{color:#0086b3}.hljs-preprocessor,.hljs-pragma,.hljs-pi,.hljs-doctype,.hljs-shebang,.hljs-cdata{color:#999;font-weight:bold}.hljs-deletion{background:#fdd}.hljs-addition{background:#dfd}.diff .hljs-change{background:#0086b3}.hljs-chunk{color:#aaa} -------------------------------------------------------------------------------- /edgar-monitor.js: -------------------------------------------------------------------------------- 1 | 2 | var request = require('request'), 3 | fs = require('fs'), 4 | exec = require('child_process').exec, 5 | Diff2Html = require('diff2html').Diff2Html, 6 | pg = require('pg'), 7 | cheerio = require('cheerio'), 8 | uuid = require('node-uuid'), 9 | aws = require('aws-sdk'), 10 | s3 = new aws.S3({ apiVersion: '2006-03-01', 'region': 'us-east-1' }), 11 | ses = new aws.SES({ apiVersion: '2010-12-01', 'region': 'us-east-1' }); 12 | 13 | function handleError(err) { 14 | if (err) { 15 | var timeStamp = (new Date()).toString(); 16 | console.log(timeStamp, ' Error:'); 17 | console.log(err); 18 | console.log('-------'); 19 | 20 | process.exit(1); 21 | } 22 | } 23 | 24 | var EdgarMonitor = module.exports = function(emitter, config) { 25 | if (config.maxDatabaseConnections) { 26 | pg.defaults.poolSize = config.maxDatabaseConnections; 27 | } 28 | 29 | emitter.on('new-entry', function(entry, feed) { 30 | var entryType = typeFromTitle(entry.title); 31 | 32 | if (config.diffAndNotifyAboutFilingTypes.indexOf(entryType) != -1) { 33 | diffEntry(entry, feed, config, function(err, diffLink) { 34 | handleError(err); 35 | notifyEntry(entry, feed, config, diffLink); 36 | }); 37 | } else if (config.notifyAboutFilingTypes.indexOf(entryType) != -1) { 38 | notifyEntry(entry, feed, config); 39 | } 40 | }); 41 | } 42 | 43 | function notifyEntry(entry, feed, config, extraResource) { 44 | var subject = entry.meta.title + ': ' + entry.title; 45 | 46 | var bodyText = entry.meta.title + '\n\n' + 47 | entry.title + '\n\n' + 48 | entry.date + '\n\n' + 49 | 'Link to filing: ' + entry.link + '\n\n'; 50 | 51 | if (extraResource) { 52 | bodyText += 'Link to diff: ' + extraResource + '\n\n'; 53 | } 54 | 55 | bodyText += entry.guid + '\n' + 56 | entry.categories + '\n\n' + 57 | 'BuzzFeed Open Lab project: this is a Beta release. We think it is pretty cool, but you shouldn\'t depend on it just yet!\nPlease report any errors or unexpected behavior to #openlab-dev or openlab@buzzfeed.com, not to BuzzFeed dev or helpdesk. Thanks!'; 58 | 59 | sendEmail(config.emailSource, config.emails, subject, bodyText, {}, function(err) { 60 | handleError(err); 61 | }); 62 | } 63 | 64 | function diffEntry(entry, feed, config, cb) { 65 | var entryType = typeFromTitle(entry.title); 66 | 67 | findOlderVersionOfEntry(entry, feed, config.databaseUrl, function(err, oldEntry) { 68 | if (err) { return cb(err); } 69 | 70 | if (!oldEntry) { return cb(null, null); } 71 | 72 | getDocLinkFromEntry(entry, function(err, newLink) { 73 | if (err) { return cb(err); } 74 | 75 | getDocLinkFromEntry(oldEntry, function(err, oldLink) { 76 | if (err) { return cb(err); } 77 | 78 | diffDocs(oldLink, newLink, function(err, diff) { 79 | if (err) { return cb(err); } 80 | 81 | var body = Diff2Html.getPrettyHtml(diff, { inputFormat: 'diff' }), 82 | html = buildDiffHtml(body), 83 | filename = entry.guid + '.html'; 84 | 85 | fs.writeFileSync(filename, html); 86 | uploadFileToS3('edgar-diffs', filename, html, { ACL: 'public-read', ContentType: 'text/html' }, function(err, link) { 87 | if (err) { return cb(err); } 88 | 89 | var secondsInAWeek = 604800; 90 | var params = { Bucket: 'edgar-diffs', Key: filename, Expires: secondsInAWeek }; 91 | s3.getSignedUrl('getObject', params, function(err, url) { 92 | cb(err, url); 93 | }); 94 | }); 95 | 96 | }); 97 | 98 | }); 99 | }); 100 | 101 | }); 102 | 103 | } 104 | 105 | function multiGet(urls, cb) { 106 | var results = {}, 107 | countFinished = 0; 108 | 109 | function handler(error, response, body) { 110 | var url = response.request.uri.href; 111 | 112 | results[url] = { error: error, response: response, body: body }; 113 | 114 | if (++countFinished === urls.length) { 115 | cb(results); 116 | } 117 | }; 118 | 119 | for (var i = 0; i < urls.length; ++i) { 120 | request(urls[i], handler); 121 | } 122 | } 123 | 124 | function diffDocs(url1, url2, cb) { 125 | var doc1 = uuid.v4(), 126 | doc2 = uuid.v4(); 127 | 128 | exec('w3m -dump -cols 200 ' + url1 + ' > "' + doc1 + '"', function(err) { 129 | if (err) { return cb(err); } 130 | 131 | exec('w3m -dump -cols 200 ' + url2 + ' > "' + doc2 + '"', function(err) { 132 | if (err) { return cb(err); } 133 | 134 | var oneGigInBytes = 1073741824; 135 | exec('git diff --ignore-all-space --no-index "' + doc1 + '" "' + doc2 + '"', 136 | { maxBuffer: oneGigInBytes - 1 }, function(err, stdout) { 137 | 138 | // git diff returns 1 when it found a difference 139 | if (err && err.code == 1) { 140 | err = null; 141 | } 142 | 143 | fs.unlinkSync(doc1); 144 | fs.unlinkSync(doc2); 145 | 146 | cb(err, stdout); 147 | }); 148 | }); 149 | }); 150 | } 151 | 152 | function buildDiffHtml(body) { 153 | return '' + 154 | '' + 155 | '' + 156 | '' + 157 | 158 | '' + 159 | 160 | '' + 161 | '' + 162 | 163 | '' + 164 | '' + 165 | 166 | body + 167 | 168 | '' + 169 | ''; 170 | } 171 | 172 | function typeFromTitle(title) { 173 | var normTitle = title.toUpperCase().trim(); 174 | 175 | if (normTitle.indexOf('S-1') == 0) { return 'S-1'; } 176 | if (normTitle.indexOf('CT ORDER') == 0) { return 'CT ORDER'; } 177 | if (normTitle.indexOf('S-8') == 0) { return 'S-8'; } 178 | if (normTitle.indexOf('EFFECT') == 0) { return 'EFFECT'; } 179 | if (normTitle.indexOf('CERTNYS') == 0) { return 'CERTNYS'; } 180 | if (normTitle.indexOf('FWP') == 0) { return 'FWP'; } 181 | if (normTitle.indexOf('8-A12B') == 0) { return '8-A12B'; } 182 | if (normTitle.indexOf('DRS') == 0) { return 'DRS'; } 183 | if (normTitle.indexOf('D') == 0) { return 'D'; } 184 | 185 | return 'UNKNOWN'; 186 | } 187 | 188 | function normalizeLink(link) { 189 | return link.replace(/^(https:\/\/)/, "") 190 | .replace(/^(http:\/\/)/, "") 191 | .replace(/^(www\.)/, ""); 192 | } 193 | 194 | function findOlderVersionOfEntry(entry, feed, databaseUrl, cb) { 195 | var entryType = typeFromTitle(entry.title), 196 | entryDate = new Date(entry.date), 197 | entryLink = normalizeLink(entry.link); 198 | 199 | pg.connect(databaseUrl, function(err, client, done) { 200 | if (err) { 201 | cb(err); 202 | return done(client); 203 | } 204 | 205 | // slect all the entries because we don't have a way to filter on filing type 206 | client.query('SELECT * from entries WHERE feed = $1 ORDER BY date DESC', [feed], function(err, result) { 207 | if (err) { 208 | cb(err); 209 | return done(client); 210 | } 211 | 212 | var foundEntry = false; 213 | for (var i = 0; i < result.rows.length; ++i) { 214 | var row = result.rows[i], 215 | rowType = typeFromTitle(row.title), 216 | rowDate = new Date(row.date), 217 | rowLink = normalizeLink(row.link); 218 | 219 | if (rowType == entryType && rowDate < entryDate && rowLink != entryLink) { 220 | foundEntry = true; 221 | cb(null, row); 222 | break; 223 | } 224 | } 225 | 226 | if (!foundEntry) { 227 | cb(null, null); 228 | } 229 | 230 | done(); 231 | }); 232 | }); 233 | } 234 | 235 | function getDocLinkFromEntry(entry, cb) { 236 | request(entry.link, function(err, response, body) { 237 | if (err) { cb(err); } 238 | 239 | var entryType = typeFromTitle(entry.title), 240 | $ = cheerio.load(body); 241 | 242 | var docRow = $('.tableFile tr').filter(function(index) { 243 | var children = $(this).children(); 244 | return $(children[3]).text().indexOf(entryType) != -1; 245 | }); 246 | 247 | var edgarSite = 'https://www.sec.gov', 248 | docLink = edgarSite + $(docRow.children()[2]).find('a').attr('href'); 249 | 250 | cb(null, docLink); 251 | }); 252 | } 253 | 254 | function extend(obj, extensions) { 255 | if (extensions && typeof extensions === 'object') { 256 | var keys = Object.keys(extensions); 257 | 258 | for (var i = 0; i < keys.length; ++i) { 259 | obj[keys[i]] = extensions[keys[i]]; 260 | } 261 | } 262 | 263 | return obj; 264 | } 265 | 266 | function uploadFileToS3(bucket, filename, body, options, cb) { 267 | var fileUpload = { 268 | Bucket: bucket, 269 | Key: filename, 270 | ACL: 'private', 271 | Body: body, 272 | }; 273 | 274 | extend(fileUpload, options); 275 | 276 | s3.putObject(fileUpload, cb); 277 | } 278 | 279 | function sendEmail(from, to, subject, body, options, cb) { 280 | var email = { 281 | Source: from, 282 | 283 | Destination: { 284 | ToAddresses: to 285 | }, 286 | 287 | Message: { 288 | Subject: { 289 | Data: subject 290 | }, 291 | Body: { 292 | Text: { 293 | Data: body 294 | } 295 | } 296 | } 297 | } 298 | 299 | extend(email, options); 300 | 301 | ses.sendEmail(email, cb); 302 | } 303 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | 2 | { 3 | "name": "edgar-monitor", 4 | "version": "0.0.1", 5 | "author": { 6 | "name": "Westley Argentum Hennigh-Palermo" 7 | }, 8 | "dependencies": { 9 | "request": "2.65.x", 10 | "aws-sdk": "2.2.x", 11 | "diff2html": "1.1.x", 12 | "pg": "4.4.x", 13 | "cheerio": "0.19.x", 14 | "node-uuid": "1.4.x" 15 | }, 16 | "engines": { 17 | "node": ">=0.8.0" 18 | }, 19 | "main": "./run.js", 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/buzzfeed-openlab/edgar-monitor" 23 | }, 24 | "bugs": { 25 | "url": "https://github.com/buzzfeed-openlab/edgar-monitor/issues" 26 | } 27 | } 28 | --------------------------------------------------------------------------------