├── test ├── resources │ ├── symlinks │ │ ├── file1 │ │ ├── file2 │ │ └── file3-link │ ├── filenonexistence │ │ ├── test.dmg │ │ ├── test.log │ │ └── rulesFMNE.json │ ├── patterndetector │ │ ├── incorrectRules.json │ │ └── testExclude │ │ │ └── testFile.js │ ├── patternexistence │ │ ├── README.md │ │ ├── rulesFMCP.json │ │ ├── README.txt │ │ └── LICENSE.txt │ ├── patternnonexistence │ │ ├── test_terms.txt │ │ ├── ip_addr_test.txt │ │ ├── pii_test.txt │ │ └── rulesFMNCP.json │ ├── unknownrt │ │ └── rulesUNK.json │ ├── TestLib.js │ ├── sample_results.csv │ └── rulesparser │ │ └── sample.json ├── unit │ ├── TestFilePattern.js │ ├── TestLocalDirWrapper.js │ ├── TestUtils.js │ ├── TestFMEMultipleFiles.js │ ├── TestUnknownRules.js │ ├── TestFileNonExistenceTestlet.js │ ├── TestIOUtils.js │ ├── TestPatternExistenceTestlet.js │ ├── TestGithubUtils.js │ ├── TestPatternNonExistenceTestlet.js │ ├── TestRulesParser.js │ ├── TestPatternDetector.js │ └── TestEvaluationResult.js └── integration │ └── TestRepoReport.js ├── standardly ├── .eslintignore ├── scripts ├── standardly.sh └── build.sh ├── changelog ├── imgs ├── FME.png ├── StandardlyFlow.png └── StandardlyLogo.svg ├── .github ├── codeowners ├── code_of_conduct.md ├── pull_request_template.md └── issue_template │ ├── feature_req_template.md │ └── bug_template.md ├── .gitignore ├── src ├── lib │ ├── common.js │ ├── utils.js │ ├── logUtils.js │ ├── gitHubUtils.js │ ├── ioUtils.js │ └── localDirWrapper.js ├── rules │ ├── Testlet.js │ ├── complianceReporter.js │ ├── TestletFactory.js │ ├── gitDetectWrapper.js │ ├── RulesParser.js │ ├── FileExistenceTestlet.js │ ├── FileNonExistenceTestlet.js │ ├── SecretKeysNonExistenceTestlet.js │ ├── ComplianceCalculator.js │ ├── EvaluationResult.js │ ├── PatternNonExistenceTestlet.js │ ├── PatternExistenceTestlet.js │ └── patternDetector.js └── app.js ├── CHANGELOG.md ├── LICENSE.md ├── docs └── CREATING-RULES.md ├── CONTRIBUTING.md ├── .eslintrc.json ├── package.json ├── .circleci └── config.yml ├── sample └── rules.json └── README.md /test/resources/symlinks/file1: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/resources/symlinks/file2: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /standardly: -------------------------------------------------------------------------------- 1 | node src/app.js "$@" 2 | -------------------------------------------------------------------------------- /test/resources/filenonexistence/test.dmg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/resources/filenonexistence/test.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | coverage 2 | deps 3 | node_modules -------------------------------------------------------------------------------- /scripts/standardly.sh: -------------------------------------------------------------------------------- 1 | node src/app.js "$@" 2 | -------------------------------------------------------------------------------- /test/resources/patterndetector/incorrectRules.json: -------------------------------------------------------------------------------- 1 | ["test"] -------------------------------------------------------------------------------- /test/resources/symlinks/file3-link: -------------------------------------------------------------------------------- 1 | test/resources/symlinks/file1 -------------------------------------------------------------------------------- /test/resources/patternexistence/README.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Company Inc. -------------------------------------------------------------------------------- /changelog: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | 4 | 5 | ## 2019-03-04 6 | Initial version -------------------------------------------------------------------------------- /imgs/FME.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/standardly/HEAD/imgs/FME.png -------------------------------------------------------------------------------- /.github/codeowners: -------------------------------------------------------------------------------- 1 | #List of main code owners 2 | * @snarayanan5 @agrover1 @vpandeti 3 | -------------------------------------------------------------------------------- /test/resources/patternnonexistence/test_terms.txt: -------------------------------------------------------------------------------- 1 | BUNAME xxx 2 | xxx testterm xxx 3 | xxxAppName -------------------------------------------------------------------------------- /imgs/StandardlyFlow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intuit/standardly/HEAD/imgs/StandardlyFlow.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | .project 4 | .settings 5 | .vscode 6 | node_modules 7 | reports 8 | coverage/ 9 | cobertura-coverage.xml 10 | .nyc_output/ 11 | gitdetect.log 12 | tmp/ 13 | package-lock.json -------------------------------------------------------------------------------- /test/resources/unknownrt/rulesUNK.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "UNK": [ 4 | { 5 | "dummy" : "This rule must be ignored", 6 | "ruleID": "UNK-9999" 7 | } 8 | ] 9 | } 10 | ] -------------------------------------------------------------------------------- /test/resources/filenonexistence/rulesFMNE.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "FMNE": [ 4 | { 5 | "fileList": ["*.DMG","*.LOG"], 6 | "description": "File Must Not Exist", 7 | "ruleID": "OOS-0004" 8 | } 9 | ] 10 | } 11 | ] -------------------------------------------------------------------------------- /test/resources/patternnonexistence/ip_addr_test.txt: -------------------------------------------------------------------------------- 1 | 10.255.255.255.10.10.255.255. 2 | 1.2.3.4.5.6 3 | 1.2.3.4 4 | 172.16.255.255 5 | 192.168.255.255 6 | 10.255.255.255 7 | The file id 7161093205057351174. 8 | "172.16.255.255",'192.168.255.255' 9 | (172.16.255.255),{192.168.255.255},[10.255.255.255] -------------------------------------------------------------------------------- /src/lib/common.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const appRoot = require("app-root-path"); 3 | const logger = require("bunyan"); 4 | const log = logger.createLogger({ name: "standardly" }); 5 | 6 | 7 | module.exports = { 8 | appRoot: appRoot, 9 | appPath : appRoot.path, 10 | log : log 11 | }; -------------------------------------------------------------------------------- /.github/code_of_conduct.md: -------------------------------------------------------------------------------- 1 | Thank you for considering contributing your time and brilliance to standardly. 2 | At Intuit, we take our open source code of conduct seriously. Please take a moment to read our [code of conduct](https://opensource.intuit.com/#coc) before you create any pull request or issue. 3 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | # setup env 2 | set -e 3 | 4 | # Install node and modules 5 | # TBD 6 | npm install 7 | 8 | # Version of node and npm 9 | npm -v 10 | node -v 11 | 12 | #run eslint 13 | npx eslint ./src/ 14 | npx eslint ./test/ 15 | 16 | # test 17 | npm test 18 | 19 | # Code coverage 20 | # TBD 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v0.1.2 (Sat Aug 17 2019) 2 | 3 | #### ⚠️ Pushed to master 4 | 5 | - fix version ([@hipstersmoothie](https://github.com/hipstersmoothie)) 6 | - Update README.md ([@hipstersmoothie](https://github.com/hipstersmoothie)) 7 | 8 | #### Authors: 1 9 | 10 | - Andrew Lisowski ([@hipstersmoothie](https://github.com/hipstersmoothie)) -------------------------------------------------------------------------------- /test/resources/patternexistence/rulesFMCP.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "FMCP": [ 4 | { 5 | "ruleID": "OOS-0016", 6 | "pattern": "^(Copyright)(\\s.{1,}\\s){1,}(Company Inc\\.)", 7 | "fileNames": ["LICENSE", "README", "DOESNOTEXIST"], 8 | "patternType": "term", 9 | "description": "company copyright", 10 | "excludeDirs":["node_modules",".git"] 11 | } 12 | ] 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /test/resources/patterndetector/testExclude/testFile.js: -------------------------------------------------------------------------------- 1 | // This file should violate the FMNCP rules, but it shouldn't show up under the scan because it's 2 | // under the directory node_modules. 3 | 4 | let ssn = "876-54-3219"; 5 | let companyEmail = "company@company.com"; 6 | 7 | function testFunc() { 8 | ssn.concat("1234"); 9 | companyEmail.concat("test"); 10 | } 11 | 12 | module.exports = { 13 | testFunc: testFunc 14 | }; -------------------------------------------------------------------------------- /test/unit/TestFilePattern.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const ldw = require("../../src/lib/localDirWrapper.js"); 5 | 6 | describe("Test File Pattern", function() { 7 | it("License.md must contain copyright", () => { 8 | return ldw.validateFilePatternExists(".", "LICENSE.md", "Copyright") 9 | .then((response) => { 10 | expect(response.result).to.be.eql("Pass"); 11 | }); 12 | }); 13 | 14 | }); -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2019 Intuit 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /test/resources/TestLib.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | /** 4 | * Get the rules corresponding to a ruleType 5 | * @param rulesFile 6 | * @param ruleType 7 | * @returns {ruleSet}. 8 | */ 9 | function getRulesSet(rulesFile, ruleType) { 10 | let ruleSets = require(rulesFile); 11 | let ruleSet; 12 | for (let i = 0; i < ruleSets.length; i++) { 13 | if (Object.keys(ruleSets[i])[0] === ruleType) { 14 | ruleSet = ruleSets[i][ruleType]; 15 | break; 16 | } 17 | } 18 | return ruleSet; 19 | } 20 | 21 | module.exports = { 22 | getRulesSet: getRulesSet 23 | }; -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | **Description** 4 | 5 | 6 | 7 | **Issue** 8 | 9 | 10 | 11 | Check the following: 12 | 13 | - [ ] The PR is not huge (it has less than a few files, and very few lines of change overall) 14 | - [ ] Proper tests are added/updated 15 | 16 | 17 | -------------------------------------------------------------------------------- /src/lib/utils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Json2csvParser = require("json2csv").Parser; 3 | 4 | function convertJsonToCsv(fields, data) { 5 | return new Promise(function(resolve, reject) { 6 | try { 7 | const json2csvParser = new Json2csvParser({ fields }); 8 | const csv = json2csvParser.parse(data); 9 | if(csv) { 10 | resolve(csv); 11 | } else { 12 | reject(Error("Error in parsing the json data into csv")); 13 | } 14 | } catch(err) { 15 | reject(Error(err)); 16 | } 17 | }); 18 | } 19 | 20 | module.exports = { 21 | convertJsonToCsv: convertJsonToCsv 22 | }; -------------------------------------------------------------------------------- /test/unit/TestLocalDirWrapper.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const fswrapper = require("../../src/lib/localDirWrapper.js"); 5 | 6 | describe("Test file Dictionary", function() { 7 | it("file dict must include this file", () => { 8 | let files = fswrapper.getDicts(".", true)[0]; 9 | expect(files["TESTLOCALDIRWRAPPER"]).to.exist; 10 | }); 11 | it("file dict must work in dirs with symlink", () => { 12 | let files = fswrapper.getDicts(".", true)[0]; 13 | expect(files["FILE1"]).to.exist; 14 | expect(files["FILE2"]).to.exist; 15 | expect(files["FILE3-LINK"]).to.not.exist; 16 | }); 17 | }); -------------------------------------------------------------------------------- /test/unit/TestUtils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const utils = require("../../src/lib/utils.js"); 5 | 6 | describe("jsonToCsv", function() { 7 | it("Json must convert to csv", () => { 8 | const fields = ["status", "% compliance"]; 9 | const data = [{ "status": "Pass", "% compliance": 30 }, { "status": "Fail", "% compliance": 50 }, { "status": "Unknown", "% compliance": 20 }]; 10 | return utils.convertJsonToCsv(fields, data) 11 | .then((csvData) => { 12 | expect(csvData).to.exist; 13 | expect(csvData).to.be.not.empty; 14 | // TODO check the content of csv 15 | }); 16 | }); 17 | 18 | }); -------------------------------------------------------------------------------- /src/lib/logUtils.js: -------------------------------------------------------------------------------- 1 | const colors = require("colors"); 2 | 3 | /** 4 | * Style the initial command line interface 5 | * Creates beautiful Standardly text 6 | * and intro sentence 7 | **/ 8 | /* eslint-disable no-console */ 9 | function clIntro() { 10 | const figlet = require("figlet"); 11 | console.log(colors.green(figlet.textSync("Standardly", {font: "big"}))); 12 | console.log(colors.bgGreen("DIY tool for Standards Governance\n")); 13 | } 14 | /* eslint-enable no-console */ 15 | 16 | /* 17 | * Custom error message 18 | * @param str 19 | */ 20 | function consoleErr(str) { 21 | console.error(colors.red(str)); // eslint-disable-line no-console 22 | } 23 | 24 | module.exports = { 25 | clIntro : clIntro, 26 | consoleErr : consoleErr 27 | }; -------------------------------------------------------------------------------- /test/resources/patternnonexistence/pii_test.txt: -------------------------------------------------------------------------------- 1 | abc@gmail.com //email 2 | a.bc@gmail.com //email 3 | a.b.c@org.net //email 4 | ;;'\/abc.def@hotmail.com 5 | sbcc@yahoocom //not an email 6 | 9@yahoo.com //email 7 | 567-345-3447 8 | (456)-345-4567 9 | 233467899 10 | ;,.234-34-3456.. 11 | ".564-89-2444" 12 | 569-00-666 13 | node@1.2.3 //the at sign used to result in false positives 14 | npm@1.2 //this is not an email either just because there is an @ 15 | @babel/runtime@7.0.0 //this should not match an email 16 | github.com/xyz/xyz@v1.2 //another non email pattern that should not be picked up 17 | asa.asd 18 | developer.company.com // should not be picked up as internal url 19 | abc.company.com // should be picked up as internal url 20 | xxxx def.company.com // should be picked up as internal url -------------------------------------------------------------------------------- /test/resources/patternexistence/README.txt: -------------------------------------------------------------------------------- 1 | # std-gov-oos 2 | 3 | * * * 4 | 5 | [![Build Status](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/buildStatus/buildIcon?job=EnterpriseArchitecture/std-gov-oos-ci)](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/job/EnterpriseArchitecture/job/std-gov-oos-ci/) [![Code Coverage](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/buildStatus/coverageIcon?job=EnterpriseArchitecture/std-gov-oos-ci)](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/job/EnterpriseArchitecture/job/std-gov-oos-ci/) 6 | 7 | [CI](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/job/EnterpriseArchitecture/job/std-gov-oos-ci/) | [PR](https://pub-bu-cto-jenkins.prod1-ibp.a.company.com/jenkins/job/EnterpriseArchitecture/job/std-gov-oos-pr/) 8 | 9 | * * * 10 | 11 | Outbound Opensource Standards Governance Automation 12 | 13 | Copyright .. -------------------------------------------------------------------------------- /test/unit/TestFMEMultipleFiles.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const assertArrays = require("chai-arrays"); 4 | chai.use(assertArrays); 5 | const expect = chai.expect; 6 | const appRoot = require("app-root-path"); 7 | const ioutils = require("../../src/lib/ioUtils.js"); 8 | const dirWrapper = require("../../src/lib/localDirWrapper.js"); 9 | 10 | const fileDict = dirWrapper.getDicts(appRoot.path, false)[0]; 11 | 12 | describe("Test multiple files for FME", function() { 13 | it("FME File dictionary must contain a list of files", () => { 14 | expect(fileDict["README"]).to.be.array(); 15 | expect(fileDict["README"]).length.above(1); 16 | }); 17 | 18 | it("Checking non empty file exits in a location", () => { 19 | return ioutils.checkNonEmptyFileExists("README", fileDict, "", "/", appRoot.path).then(res =>{ 20 | expect(res).to.be.eql(true); 21 | }); 22 | }); 23 | 24 | }); -------------------------------------------------------------------------------- /src/lib/gitHubUtils.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const shelljs = require("shelljs"); 3 | const common = require("../lib/common"); 4 | const appPath = common.appPath; 5 | 6 | function cloneGitRepo(repository, workingDir) { 7 | return new Promise((resolve, reject) => { 8 | const branch = "master"; // TODO: make this an option - or at least read the default branch from git. 9 | shelljs.cd(workingDir); 10 | // repo clone will be for just the master branch, and only the last revision. tmp dir will get deleted on exit if empty 11 | if (shelljs.exec(`git clone -b ${branch} --single-branch --depth 1 ${repository}`, {"silent": false}).code !== 0) { 12 | reject("Error cloning repository"); 13 | } 14 | shelljs.cd(appPath); 15 | const repoName = path.basename(repository, ".git"); 16 | resolve(path.resolve(workingDir, repoName)); 17 | }); 18 | } 19 | 20 | module.exports = { 21 | cloneGitRepo: cloneGitRepo 22 | }; -------------------------------------------------------------------------------- /test/unit/TestUnknownRules.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const TestletFactory = require("../../src/rules/TestletFactory.js"); 5 | const TestLib = require("../resources/TestLib.js"); 6 | const appPath = require("../../src/lib/common.js").appPath; 7 | const rulesJson = appPath + "/test/resources/unknownrt/rulesUNK.json"; 8 | 9 | describe("Test handling of unknown ruletype", function() { 10 | it("Base Testlet class to return an array of dummy promise ", function() { 11 | const target = {localdir : appPath + "/test/resources/unknownrt"}; 12 | const ruleSet = TestLib.getRulesSet(rulesJson, "UNK"); 13 | const unkTestlet = TestletFactory.createTestlet(target, "UNK", ruleSet); 14 | const promises = unkTestlet.evaluate(); 15 | 16 | return Promise.all(promises).then(response => { 17 | response.forEach(res => { 18 | expect(res).to.exist; 19 | expect(res).to.be.empty; 20 | }); 21 | }); 22 | }); 23 | }); -------------------------------------------------------------------------------- /docs/CREATING-RULES.md: -------------------------------------------------------------------------------- 1 | # The Rules File 2 | 3 | A sample rules file is available [here](../test/resources/rulesparser/sample.json) 4 |

Rules should be written in JSON format. An example of a standard that was turned into a rule is shown below:

5 | FME rule: converted from a standard to json formatted rule 6 | 7 | Currently, Standardly supports the following types of rules: 8 | * FME (File Must Exist) 9 | * FMNE (File Must Not Exist) 10 | * FMCP (File Must Contain Pattern) 11 | * FMNCP (File Must Not Contain Pattern) 12 | 13 | Each ruletype is implemented by a corresponding Testlet. For example the 'File Must Exist' rule 14 | is implemented by the 'FileExistenceTestlet'. Testlets can be found by navigating to the [rules 15 | directory](../src/rules). 16 | 17 | Testlets essentially evaluate whether or not a rule's requirement was fullfilled. Therefore, for any rule type, a corresponding Testlet needs to exist. If you find the need to create a new rule type, we would love to help you. Please log an issue describing the type of rule you are creating and we would be happy to help you create it. 18 | -------------------------------------------------------------------------------- /.github/issue_template/feature_req_template.md: -------------------------------------------------------------------------------- 1 | Before you submit an issue, ensure that the issue is not already found in this list. 2 | 3 | If you are absolutely sure that this issue isn't already open, clearly describe the problem providing the reason why you believe it needs to be solved. If this is a bug please include steps to reproduce the bug, including logs / screenshots as appropriate 4 | 5 | --- 6 | 7 | name: Feature Request 8 | about: You have an idea? We are all ears! 9 | title: '' 10 | labels: bug 11 | assignees: '' 12 | 13 | --- 14 | 15 | **Your Idea** 16 | 17 | 18 | 19 | **Impact** 20 | 21 | 22 | 23 | **Your approach** 24 | 25 | 26 | 27 | **Additional detail** 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/rules/Testlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const log = require("../lib/common.js").log; 3 | /** 4 | * A Testlet evaluates a set of one or more rules of a particular type and captures the results of the evaluation. 5 | */ 6 | class Testlet { 7 | constructor(target, ruleType, ruleSet, ruleFileName, gitUrl, excludeDirs) { 8 | this.target = target; 9 | this.ruleType = ruleType; 10 | this.ruleSet = ruleSet; 11 | this.ruleFileName = ruleFileName; 12 | this.results = []; 13 | this.gitUrl = gitUrl; 14 | this.excludeDirs = excludeDirs; 15 | } 16 | 17 | /** 18 | * The individual Testlets must implement this method 19 | * The method must return an array of promises and must never throw any exception 20 | * All exceptions must be handled in the method 21 | */ 22 | evaluate() { 23 | log.warn("Ignoring rule type " + this.ruleType); 24 | const promise = new Promise(resolve => { 25 | resolve([]); 26 | }); 27 | return [promise]; 28 | } 29 | 30 | /** 31 | * Gets the results of evaluation of this testlet 32 | */ 33 | getEvaluationResults() { 34 | let results = []; 35 | return results; 36 | } 37 | } 38 | 39 | module.exports = Testlet; -------------------------------------------------------------------------------- /test/unit/TestFileNonExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const FileNonExistenceTestlet = require("../../src/rules/FileNonExistenceTestlet.js"); 5 | const TestLib = require("../resources/TestLib.js"); 6 | const appRoot = require("app-root-path"); 7 | const rulesJson =appRoot.path + "/test/resources/filenonexistence/rulesFMNE.json"; 8 | 9 | describe("Test File non existence Testlet", function() { 10 | it("Must have matches for .LOG and .DMG files", function() { 11 | const target = {localdir : appRoot.path + "/test/resources/filenonexistence"}; 12 | const ruleSet = TestLib.getRulesSet(rulesJson, "FMNE"); 13 | const fileNonExistenceTestlet = new FileNonExistenceTestlet(target, ruleSet); 14 | const promises = fileNonExistenceTestlet.evaluate(); 15 | 16 | return Promise.all(promises).then(response => { 17 | expect(response).to.have.length.above(1); 18 | response.forEach(res => { 19 | expect(res).to.exist; 20 | expect(res).to.be.not.empty; 21 | if (res.message.includes(".LOG")||res.message.includes(".DMG")) { 22 | expect(res.result).to.be.eql("Fail"); 23 | } 24 | }); 25 | }); 26 | }); 27 | }); -------------------------------------------------------------------------------- /test/integration/TestRepoReport.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai"); 2 | const expect = chai.expect; 3 | const shelljs = require("shelljs"); 4 | const csv = require("csvtojson"); 5 | const common = require("../../src/lib/common"); 6 | const appPath = common.appPath; 7 | 8 | describe("Tool report is consistent", function() { 9 | it("Report output is consistent", () => { 10 | return new Promise(resolve => { 11 | // TODO: change repo to test standarly once it is public 12 | const toolCmd = "node src/app.js --giturl https://github.com/intuit/saloon.git -r sample/rules.json"; 13 | shelljs.exec(toolCmd, resolve); 14 | }) 15 | .then(() => Promise.all([csv().fromFile(appPath + "/test/resources/sample_results.csv"), csv().fromFile(appPath + "/reports/results.csv")])) 16 | .then(([sample, actual]) => { 17 | // Assert that both files have the same object count 18 | expect(Object.keys(sample).length).to.be.eql(Object.keys(actual).length); 19 | for(let obj in sample) { 20 | // Assert that for each object,the result is the same 21 | expect(sample[obj].result).to.be.eql(actual[obj].result, "obj #"+obj+" with rule "+ sample[obj].ruleID + " has a different result"); 22 | } 23 | 24 | }); 25 | }).timeout(4000); 26 | }); -------------------------------------------------------------------------------- /.github/issue_template/bug_template.md: -------------------------------------------------------------------------------- 1 | Before you submit an issue, ensure that the issue is not already found in this list. 2 | 3 | If you are absolutely sure that this issue isnt already open, clearly describe the problem providing the reason why you believe it needs to be solved. If this is a bug please include steps to reproduce the bug, including logs / screenshots as appropriate 4 | 5 | --- 6 | 7 | name: Bug Report 8 | about: Sorry that you are facing a problem. Thank you for helping us keep those pesky bugs off! 9 | title: '' 10 | labels: bug 11 | assignees: '' 12 | 13 | --- 14 | 15 | **Describe the bug** 16 | 17 | 18 | 19 | **To Reproduce** 20 | 21 | 22 | 23 | **Expected behavior** 24 | 25 | 26 | 27 | **Log or report files** 28 | 29 | 30 | 31 | **Screenshots** 32 | 33 | 34 | 35 | **Desktop (please complete the following information):** 36 | 37 | - OS: 38 | - node version: 39 | - npm version: 40 | - version of this module: 41 | 42 | **Additional detail** 43 | 44 | 45 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for considering contributing your time and brilliance to standardly. We welcome your contributions in the form of issues or code. But before we may accept any of those, we request you to kindly do the following: 2 | 3 | ### Be courteous 4 | At Intuit, we take our engineering open source code of conduct seriously. Please take a moment to read our [code of conduct](https://opensource.intuit.com/#coc) before you create any pull request or issue. 5 | 6 | ### DRY : Please do not repeat yourself. 7 | Make sure you have checked the list of open issues and ensure that the issue is not already covered by an existing one. Our issue and pr templates are not mere suggestions. Please fill in whatever is applicable - if something is not applicable, please say so, rather than just leave it empty - this way we know that you actually care. 8 | 9 | ### Pull Requests 10 | We will review your pull requests and get back to you - our timeline for response will be displayed as a comment in the description area of the pull request when you create one. 11 | - Pull requests without an associated issue could be categorically rejected. So please pay attention that your pr actually solves an real issue. 12 | - They must also describe your approach to the problem and any other information you can give regarding your solution. 13 | - Make sure that your tests are adequate and actually pass. 14 | - Ensure merge conflicts are resolved before you request for a review. 15 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "eslint:recommended", 3 | "parserOptions": { 4 | "ecmaVersion": 2017 5 | }, 6 | "env": { 7 | "es6": true, 8 | "node": true, 9 | "mocha": true 10 | }, 11 | "plugins": [ 12 | "mocha" 13 | ], 14 | "rules": { 15 | "array-bracket-newline": ["error", "never"], 16 | "array-bracket-spacing": ["error", "never"], 17 | "array-element-newline": ["error", "never"], 18 | "block-spacing": ["error", "always"], 19 | "brace-style": ["error", "1tbs"], 20 | "camelcase": "error", 21 | "comma-dangle": ["error", "never"], 22 | "comma-spacing": ["error", {"before": false, "after": true}], 23 | "comma-style": ["error", "last"], 24 | "curly": "error", 25 | "eol-last": ["error", "never"], 26 | "func-call-spacing": ["error", "never"], 27 | "function-paren-newline": ["error", "never"], 28 | "indent": ["error", 4], 29 | "mocha/no-exclusive-tests": "error", 30 | "no-multi-spaces": ["error", {"ignoreEOLComments": true}], 31 | "no-trailing-spaces": ["error", {"skipBlankLines":false}], 32 | "quotes": ["error", "double", { "avoidEscape": true }], 33 | "semi": ["error", "always"], 34 | "semi-style": ["error", "last"], 35 | "space-before-function-paren": ["error", "never"], 36 | "spaced-comment": ["error", "always"], 37 | "space-in-parens": ["error", "never"] 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/unit/TestIOUtils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const iou = require("../../src/lib/ioUtils.js"); 5 | const assertArrays = require("chai-arrays"); 6 | chai.use(assertArrays); 7 | 8 | describe("Test File Exists", function() { 9 | it("LICENSE.md must exist", () => { 10 | return iou.checkFileExists(".", "LICENSE.md") 11 | .then((response) => { 12 | expect(response).to.be.eql(true); 13 | }); 14 | }); 15 | 16 | }); 17 | 18 | describe("Test exclude dir is considered", ()=> { 19 | it("All files must be returned as none in excluded dir", ()=> { 20 | let res = iou.getUnexcludedDirs(["fruit/apple", "pine", "pineapple"], ["tree"]); 21 | expect(res).to.have.length(3); 22 | }); 23 | it("No files must be be returned as all in excluded dir", ()=> { 24 | let res = iou.getUnexcludedDirs(["edible/fruit/tree/apple", "edible/fruit/bush/pineapple", "edible/fruit/pine"], ["FRUIT", "PINE"]); 25 | expect(res).to.be.an.array().that.is.empty; 26 | }); 27 | it("Some files must be returned that are not in excluded dir", ()=> { 28 | let res = iou.getUnexcludedDirs(["edible/fruit/tree/apple", "edible/fruit/bush/pineapple", "edible/fruit/pine"], ["PINE"]); 29 | expect(res).to.be.have.length(2); 30 | }); 31 | it("file without dir name should not match", ()=> { 32 | let res = iou.getUnexcludedDirs(["pine"], ["FRUIT", "PINE"]); 33 | expect(res).to.be.have.length(1); 34 | }); 35 | 36 | }); -------------------------------------------------------------------------------- /test/unit/TestPatternExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const PatternExistenceTestlet = require("../../src/rules/PatternExistenceTestlet.js"); 5 | const TestLib = require("../resources/TestLib.js"); 6 | const appPath = require("../../src/lib/common.js").appPath; 7 | const rulesJson = appPath + "/test/resources/patternexistence/rulesFMCP.json"; 8 | 9 | describe("Test Pattern Existence Testlet", function() { 10 | it("Run Pattern Existence Testlet and Test Object", function() { 11 | const target = {localdir : appPath + "/test/resources/patternexistence"}; 12 | const ruleSet = TestLib.getRulesSet(rulesJson, "FMCP"); 13 | 14 | const patternExistenceTestlet = new PatternExistenceTestlet(target, ruleSet); 15 | const promises = patternExistenceTestlet.evaluate(); 16 | 17 | return Promise.all(promises).then(response => { 18 | expect(response).to.have.length.above(1); 19 | response.forEach(res => { 20 | expect(res).to.exist; 21 | expect(res).to.be.not.empty; 22 | if (res.message.includes("LICENSE")) { 23 | expect(res.result).to.be.eql("Pass"); 24 | } else if (res.message.includes("README")) { 25 | expect(res.result).to.be.eql("Fail"); 26 | } else if (res.message.includes("DOESNOTEXIST")) { 27 | expect(res.result).to.be.eql("Error"); 28 | } 29 | }); 30 | }); 31 | }); 32 | }); -------------------------------------------------------------------------------- /test/unit/TestGithubUtils.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai"); 2 | const expect = chai.expect; 3 | const gitHubUtils = require("../../src/lib/gitHubUtils.js"); 4 | const path = require("path"); 5 | const shelljs = require("shelljs"); 6 | const common = require("../../src/lib/common"); 7 | const appPath = common.appPath; 8 | const tmpDir = path.join(appPath, "/tmp"); 9 | const fse = require("fs-extra"); 10 | 11 | let logger = require("bunyan"); 12 | let log = logger.createLogger({ name: "standardly" }); 13 | 14 | describe("Test GitHubUtils Clone ", function() { 15 | it("Sucessfully clones a repo", () => { // skipping until we get a github service account. 16 | return fse.emptyDir(tmpDir).then(() => { 17 | gitHubUtils.cloneGitRepo("https://github.com/intuit/saloon.git", tmpDir) 18 | .then((response) => { 19 | expect(response).to.be.not.empty; 20 | expect(path.basename(response, ".git")).to.eql("saloon"); 21 | }) 22 | .finally(() => { 23 | shelljs.rm("-rf", tmpDir); 24 | }); 25 | }); 26 | }).timeout(5000); 27 | 28 | it("Fails when repo url is not correct", () => { 29 | // Wait no longer than the timeout specified for the incorrect url clone to fail 30 | setTimeout(() => { 31 | log.info("Ending negative git url clone on timeout"); 32 | }, 2000); 33 | return gitHubUtils.cloneGitRepo("https://github.com/intuit/saloon.gitXXX", tmpDir) 34 | .catch((response) => { 35 | expect(response).to.have.string("Error cloning repository"); 36 | }); 37 | }).timeout(2500); 38 | 39 | }); -------------------------------------------------------------------------------- /src/rules/complianceReporter.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const ioutil = require("../lib/ioUtils"); 3 | const EvaluationResult = require("./EvaluationResult"); 4 | const ComplianceCalculator = require("./ComplianceCalculator"); 5 | const detailFields = "ruleID,error,detail\n"; 6 | 7 | /** 8 | * Reports compliance 9 | * @param {*} finalResults 10 | * @param {*} resultsfile 11 | */ 12 | function reportCompliance(finalResults, resultsfile) { 13 | return new Promise(resolve => { 14 | let finalResultsString = ""; 15 | finalResults.forEach(result => { 16 | finalResultsString += serializeEvaluationResults(result); 17 | }); 18 | if (finalResults.length > 0) { 19 | new ComplianceCalculator(finalResults).getComplianceSummary() 20 | .then(statistics => { 21 | ioutil.writeFile(resultsfile, statistics + "\n\n" + detailFields + finalResultsString) 22 | .then((result)=>{ 23 | resolve(result); 24 | }); 25 | }); 26 | } else { 27 | return resolve(false); 28 | } 29 | }); 30 | } 31 | 32 | /** 33 | * Serializes rule evaulation results 34 | * @param {*} result 35 | */ 36 | function serializeEvaluationResults(result) { 37 | let str = ""; 38 | if(result instanceof EvaluationResult) { 39 | str += result.getAsString() + "\n"; 40 | } else if (result instanceof Array) { 41 | result.forEach(evaluationResult => { 42 | str += evaluationResult.getAsString(); 43 | }); 44 | } 45 | return str; 46 | } 47 | 48 | module.exports = { 49 | reportCompliance : reportCompliance 50 | }; -------------------------------------------------------------------------------- /src/rules/TestletFactory.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const FileExistenceTestlet = require("./FileExistenceTestlet"); 3 | const FileNonExistenceTestlet = require("./FileNonExistenceTestlet"); 4 | const PatternNonExistenceTestlet = require("./PatternNonExistenceTestlet"); 5 | const PatternExistenceTestlet = require("./PatternExistenceTestlet"); 6 | const Testlet = require("./Testlet.js"); 7 | const log = require("../lib/common.js").log; 8 | 9 | /** 10 | * Types of rules 11 | */ 12 | const RuleTypeEnum = { 13 | FME: "FME", 14 | FMNE: "FMNE", 15 | FMCP: "FMCP", 16 | FMNCP: "FMNCP", 17 | SECRETKEYS: "SECRETKEYS" 18 | }; 19 | 20 | /** 21 | * Creates the testlet based on the ruleType 22 | * @param {*} target 23 | * @param {*} ruleType 24 | * @param {*} ruleset 25 | * @param {*} rulesfile 26 | * @param {*} excludeDirs 27 | */ 28 | function createTestlet(target, ruleType, ruleset, rulesfile, excludeDirs) { 29 | if (ruleType == RuleTypeEnum.FME) { 30 | return new FileExistenceTestlet(target, ruleset, rulesfile, excludeDirs); 31 | } else if (ruleType == RuleTypeEnum.FMNE) { 32 | return new FileNonExistenceTestlet(target, ruleset, rulesfile, excludeDirs); 33 | } else if (ruleType == RuleTypeEnum.FMNCP) { 34 | return new PatternNonExistenceTestlet(target, ruleset, rulesfile, excludeDirs); 35 | } else if (ruleType == RuleTypeEnum.FMCP) { 36 | return new PatternExistenceTestlet(target, ruleset, rulesfile, excludeDirs); 37 | } else { 38 | log.warn("Unknown rule type " + ruleType + " will be ignored"); 39 | return new Testlet(target, null, ruleset, rulesfile, excludeDirs); 40 | } 41 | } 42 | 43 | module.exports = { 44 | createTestlet: createTestlet, 45 | RuleTypeEnum: RuleTypeEnum 46 | }; -------------------------------------------------------------------------------- /test/unit/TestPatternNonExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const PatternNonExistenceTestlet = require("../../src/rules/PatternNonExistenceTestlet.js"); 5 | const TestLib = require("../resources/TestLib.js"); 6 | const appRoot = require("app-root-path"); 7 | const rulesJson =appRoot.path + "/test/resources/patternnonexistence/rulesFMNCP.json"; 8 | 9 | describe("Test Pattern non existence Testlet", function() { 10 | it("Must have matches for email, phone and ssn", function() { 11 | const target = {localdir : appRoot.path + "/test/resources/patternnonexistence"}; 12 | const ruleSet = TestLib.getRulesSet(rulesJson, "FMNCP"); 13 | const patternNonExistenceTestlet = new PatternNonExistenceTestlet(target, ruleSet, rulesJson); 14 | const promises = patternNonExistenceTestlet.evaluate(); 15 | 16 | return promises.then(response => { 17 | expect(response).to.have.length.above(1); 18 | expect(response.filter(res => res.message.includes("internal term"))).length(2); 19 | expect(response.filter(res => res.message.includes("internal urls"))).length(2); 20 | response.forEach(res => { 21 | expect(res).to.exist; 22 | expect(res).to.be.not.empty; 23 | if (res.message.includes("email address")) { 24 | expect(res.detail).length(4); 25 | } else if (res.message.includes("phone number")) { 26 | expect(res.detail).length(2); 27 | } else if (res.message.includes("SSN")) { 28 | expect(res.detail).length(2); 29 | } else if (res.message.includes("internal ip address")) { 30 | expect(res.detail).length(8); 31 | } 32 | }); 33 | }); 34 | }); 35 | }); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@intuit/standardly", 3 | "version": "0.1.2", 4 | "description": "DIY framework for Standards Governance", 5 | "main": "index.js", 6 | "directories": { 7 | "lib": "lib" 8 | }, 9 | "scripts": { 10 | "test": "nyc --reporter=cobertura mocha './test/unit/*.js'", 11 | "test:integration": "mocha './test/integration/*.js'", 12 | "lint": "node_modules/.bin/eslint src test", 13 | "release": "auto shipit" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/intuit/standardly.git" 18 | }, 19 | "keywords": [ 20 | "standards", 21 | "governance", 22 | "compliance" 23 | ], 24 | "author": "Andrew Lisowski ", 25 | "license": "Apache-2.0", 26 | "dependencies": { 27 | "app-root-path": "2.1.0", 28 | "bunyan": "1.8.12", 29 | "chai-arrays": "2.0.0", 30 | "colors": "1.3.3", 31 | "commander": "2.19.0", 32 | "figlet": "1.2.1", 33 | "fs-extra": "7.0.1", 34 | "js-yaml": "3.13.1", 35 | "json2csv": "4.2.1", 36 | "request": "2.88.0" 37 | }, 38 | "devDependencies": { 39 | "auto": "^7.3.0", 40 | "babel-eslint": "^10.0.2", 41 | "chai": "4.2.0", 42 | "chai-as-promised": "7.1.1", 43 | "csvtojson": "2.0.8", 44 | "eslint": "^6.1.0", 45 | "eslint-config-airbnb": "^18.0.1", 46 | "eslint-config-airbnb-base": "^14.0.0", 47 | "eslint-plugin-import": "^2.18.2", 48 | "eslint-plugin-jsx-a11y": "^6.2.3", 49 | "eslint-plugin-mocha": "5.2.0", 50 | "eslint-plugin-react": "^7.14.3", 51 | "eslint-plugin-react-hooks": "^1.7.0", 52 | "husky": "1.3.1", 53 | "mocha": "5.2.0", 54 | "nyc": "14.1.1", 55 | "shelljs": "0.8.3" 56 | }, 57 | "husky": { 58 | "hooks": { 59 | "pre-commit": "npm run lint -- --fix && npm test && npm run test:integration" 60 | } 61 | }, 62 | "bin": { 63 | "standardly": "./standardly" 64 | }, 65 | "engines": { 66 | "node": ">=10.16.0" 67 | }, 68 | "auto": { 69 | "plugins": [ 70 | "npm", 71 | "released" 72 | ] 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/rules/gitDetectWrapper.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const appRoot = require("app-root-path"); 4 | // TODO, once gitdetect is available replace this definition - for now pointing script to a noop 5 | // script should be set to appRoot.path + "/deps/gitdetect/gitdetect"; 6 | const script = ":"; 7 | // TODO - move the above script config to a config file 8 | const gitDetectConfig = appRoot.path + "/deps/gitdetect/gitdetect.conf.yaml"; 9 | const exec = require("child_process").exec; 10 | const ioUtils = require("../lib/ioUtils"); 11 | const gitDetectResultsDir = appRoot.path + "/reports/"; 12 | const gitDetectOutputFile = "defectReport.yaml"; 13 | 14 | let logger = require("bunyan"); 15 | let log = logger.createLogger({ name: "standardly" }); 16 | 17 | /** 18 | * Runs git detect tool's executable to find secret keys. 19 | * @param {*} repoOwner 20 | * @param {*} repoName 21 | * @param {*} gitToken 22 | * @returns {Promise} 23 | */ 24 | function runGitDetect(repoOwner, repoName, gitToken) { 25 | return new Promise(resolve => { 26 | try { 27 | const cmd = script + " -access-token " + gitToken + " -config " + gitDetectConfig + " -output " + 28 | gitDetectResultsDir + " -repo-name " + repoOwner + "/" + repoName; 29 | exec(cmd, {stdio: "inherit"}, 30 | (error, stderr) => { 31 | if (error) { 32 | log.error(stderr); 33 | resolve(false); 34 | } 35 | ioUtils 36 | .readFile(gitDetectResultsDir + gitDetectOutputFile) 37 | .then(data => { 38 | if (data) { 39 | resolve(data); 40 | } else { 41 | resolve("[]"); 42 | } 43 | }); 44 | }); 45 | } catch (ex) { 46 | log.error(ex); 47 | resolve("[]"); 48 | } 49 | }); 50 | } 51 | 52 | module.exports = { 53 | runGitDetect: runGitDetect 54 | }; -------------------------------------------------------------------------------- /test/unit/TestRulesParser.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const RulesParser = require("../../src/rules/RulesParser.js"); 5 | const path = require("path"); 6 | const common = require("../../src/lib/common.js"); 7 | 8 | describe("Test the Rules Parser", () => { 9 | const rulesFile = path.join(common.appPath, "/test/resources/rulesparser/sample.json"); 10 | const outputdir = path.join(common.appPath, "reports"); 11 | // TODO Initialize when bugs are fixed for exludedDirs 12 | const excludeDirs = ""; 13 | it("Validate the rules in the given array", () => { 14 | const target = {localdir : path.join(common.appPath, "/test/resources/"), 15 | gitUrl : "https://github.com/intuit/saloon"}; 16 | 17 | const rp = new RulesParser(target, rulesFile, outputdir, excludeDirs); 18 | 19 | // TODO write a more controlled rules.json so results can be scrutinized 20 | // TODO Timeout might need to be added when we add more data in the test resources folder. 21 | return rp.parse().then(results => { 22 | expect(results).to.have.lengthOf.above(0); 23 | }); 24 | }); 25 | }); 26 | 27 | /** 28 | * Test the sample rules file - yes, the purpose of this is different from 29 | * the purpose of the tests agains the rules parser. This one tests that any 30 | * changes in the sample file is still parseable and valid with our Rules Parser, 31 | * whereas the rules parser tests validate that any changes to the rules parser 32 | * dont break our test rules. There was a brief consideration of only testing the 33 | * samples, but that doesnt make sense as we need to do robust ruleparser testing 34 | * going forward 35 | */ 36 | describe("Test the sample rules", () => { 37 | const rulesFile = path.join(common.appPath, "test/resources/rulesparser/sample.json"); 38 | const outputdir = path.join(common.appPath, "reports"); 39 | const excludeDirs = ""; 40 | it("validate that rules in the sample pass through rulesparser fine", () => { 41 | const target = {localdir : path.join(common.appPath, "/test/resources/"), 42 | gitUrl : "https://github.com/intuit/saloon"}; 43 | 44 | const rp = new RulesParser(target, rulesFile, outputdir, excludeDirs); 45 | return rp.parse().then(results => { 46 | expect(results).to.have.lengthOf.above(0); 47 | }); 48 | }); 49 | }); -------------------------------------------------------------------------------- /src/rules/RulesParser.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const TestletFactory = require("./TestletFactory.js"); 3 | const common = require("../lib/common.js"); 4 | const ioutils = require("../lib/ioUtils.js"); 5 | const log = common.log; 6 | 7 | /** 8 | * The parser that reads through the rules.json and creates testlets 9 | */ 10 | class RulesParser { 11 | constructor(target, rulesfile, excludeDirs) { 12 | this.target = target; 13 | this.rulesfile = rulesfile; 14 | this.testlets = []; 15 | this.excludeDirs = excludeDirs; 16 | } 17 | 18 | parse() { 19 | // TODO surround with outer try/catch? 20 | return new Promise(resolve => { 21 | ioutils.readFile(this.rulesfile) 22 | .then((data) => { 23 | const ruleSets = JSON.parse(data); 24 | log.info("Read the rulesets - found " + ruleSets.length + " of them!"); 25 | let evalPromises = []; 26 | ruleSets.forEach(ruleSet => { 27 | let ruleType = Object.keys(ruleSet); 28 | log.info("parsing " + ruleType); 29 | let testlet = TestletFactory.createTestlet(this.target, ruleType, ruleSet[ruleType], 30 | this.rulesfile, this.excludeDirs); 31 | let temp = testlet.evaluate(); 32 | evalPromises = evalPromises.concat(temp); 33 | }); 34 | let finalResults = []; 35 | return Promise.all(evalPromises) 36 | .then(evalResults => { 37 | log.info("Inspecting results - received " + evalResults.length + " of them"); 38 | evalResults.forEach(result => { 39 | finalResults = finalResults.concat(result); 40 | }); 41 | return resolve(finalResults); 42 | }).catch(exception => { 43 | log.error(JSON.stringify(exception)); 44 | return resolve([]); 45 | }); 46 | }).catch(exception => { 47 | log.error("Error validating the rules " + exception); 48 | return resolve([]); 49 | }); 50 | }); 51 | } 52 | } 53 | 54 | module.exports = RulesParser; -------------------------------------------------------------------------------- /imgs/StandardlyLogo.svg: -------------------------------------------------------------------------------- 1 | Standardly_small -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | defaults: &defaults 5 | working_directory: ~/standardly 6 | docker: 7 | - image: circleci/node:latest-browsers 8 | 9 | jobs: 10 | install: 11 | <<: *defaults 12 | steps: 13 | - checkout 14 | - restore_cache: 15 | keys: 16 | # Find a cache corresponding to this specific package.json checksum 17 | # when this file is changed, this key will fail 18 | - standardly-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum ".circleci/config.yml" }} 19 | - standardly-{{ .Branch }}-{{ checksum "package-lock.json" }} 20 | - standardly-{{ .Branch }} 21 | # Find the most recent cache used from any branch 22 | - standardly-master 23 | - standardly- 24 | - run: 25 | name: Install Dependencies 26 | command: yarn install --frozen-lockfile 27 | - save_cache: 28 | key: standardly-{{ .Branch }}-{{ checksum "package-lock.json" }}-{{ checksum ".circleci/config.yml" }} 29 | paths: 30 | - node_modules 31 | - ~/.cache/yarn 32 | - persist_to_workspace: 33 | root: . 34 | paths: 35 | - . 36 | 37 | test: 38 | <<: *defaults 39 | steps: 40 | - attach_workspace: 41 | at: ~/standardly 42 | - run: 43 | name: Test 44 | command: yarn test 45 | 46 | lint: 47 | <<: *defaults 48 | steps: 49 | - attach_workspace: 50 | at: ~/standardly 51 | - run: 52 | name: Lint 53 | command: yarn lint 54 | 55 | # integration: 56 | # <<: *defaults 57 | # steps: 58 | # - attach_workspace: 59 | # at: ~/standardly 60 | # - run: 61 | # name: Integration Tests 62 | # command: yarn test:integration 63 | 64 | release: 65 | <<: *defaults 66 | steps: 67 | - attach_workspace: 68 | at: ~/standardly 69 | - run: mkdir ~/.ssh/ && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > ~/.ssh/config 70 | - run: 71 | name: Release 72 | command: yarn release 73 | 74 | workflows: 75 | version: 2 76 | build_and_test: 77 | jobs: 78 | - install 79 | 80 | - test: 81 | requires: 82 | - install 83 | 84 | - lint: 85 | requires: 86 | - install 87 | 88 | # - integration: 89 | # requires: 90 | # - install 91 | 92 | - release: 93 | requires: 94 | - lint 95 | - test 96 | # - integration 97 | 98 | -------------------------------------------------------------------------------- /src/rules/FileExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Testlet = require("./Testlet.js"); 3 | const dirWrapper = require("../lib/localDirWrapper.js"); 4 | const EvaluationResult = require("./EvaluationResult.js"); 5 | const ResultEnum = EvaluationResult.ResultEnum; 6 | 7 | class FileExistenceTestlet extends Testlet { 8 | /** 9 | * Creates the FileExistenceTestlet 10 | * @param {*} target - the target object, for example a directory where this rule is executed 11 | * @param {*} ruleSet 12 | * @param {*} ruleJson 13 | * @param {*} excludeDirs 14 | */ 15 | constructor(target, ruleSet, ruleJson, excludeDirs) { 16 | super(target, "FME", ruleSet, ruleJson, "", excludeDirs); 17 | } 18 | 19 | /** 20 | * Evaluates the ruleSet 21 | * Returns an array of promises that resolve to a TestletOutput object 22 | */ 23 | evaluate() { 24 | let promises = []; 25 | let fileDict = dirWrapper.getDicts(this.target.localdir, false)[0]; 26 | this.ruleSet.forEach(rule => { 27 | let exclude = []; 28 | if (this.excludeDirs){ 29 | exclude.concat(this.excludeDirs); 30 | } 31 | if("excludeDirs" in Object.keys(rule) && rule.excludeDirs){ 32 | exclude.concat(rule.excludeDirs); 33 | } 34 | promises.push(this.validateNonEmptyFileExists(rule, fileDict, exclude, this.target.localdir)); 35 | }); 36 | return promises; 37 | } 38 | 39 | /** 40 | * Validates if non-empty file exists 41 | * @param {*} rule 42 | * @param {*} fileDict 43 | * @param {*} excludeDirs 44 | * @param {*} localdir 45 | */ 46 | validateNonEmptyFileExists(rule, fileDict, excludeDirs, localdir) { 47 | let vResult; 48 | return new Promise(resolve => { 49 | let location = "location" in Object.keys(rule) ? rule.location : ""; 50 | dirWrapper.validateNonEmptyFileExists(rule.fileName, fileDict, excludeDirs, location, localdir) 51 | .then(exists => { 52 | const message = (exists ? "Found " : "Not found ") + "non-zero length " + 53 | rule.fileName + " in " + 54 | (this.target.giturl ? this.target.giturl : this.target.localdir); 55 | const result = exists ? ResultEnum.PASS : ResultEnum.FAIL; 56 | vResult = new EvaluationResult(rule.ruleID, result, message); 57 | resolve(vResult); 58 | }).catch(exception => { 59 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, "Error evaluating rule ", JSON.stringify(exception)); 60 | resolve(vResult); 61 | }); 62 | }); 63 | } 64 | } 65 | 66 | module.exports = FileExistenceTestlet; -------------------------------------------------------------------------------- /test/unit/TestPatternDetector.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai"); 2 | const expect = chai.expect; 3 | const patternDetector = require("../../src/rules/patternDetector.js"); 4 | const appRoot = require("app-root-path"); 5 | const repo = appRoot.path + "/src/rules"; 6 | const TestLib = require("../resources/TestLib.js"); 7 | const rulesJson = appRoot.path + "/test/resources/patternnonexistence/rulesFMNCP.json"; 8 | const ruleSet = TestLib.getRulesSet(rulesJson, "FMNCP"); 9 | 10 | describe("Test Pattern Detector - JavaScript", function() { 11 | it("Test with rulesfile containing only FMNCP in directory with 0 matches", function() { 12 | return patternDetector.processPatterns(repo, ruleSet).then(response => { 13 | expect(response).to.exist; 14 | expect(response).to.be.not.empty; 15 | expect(response).to.have.lengthOf(1); 16 | }); 17 | }); 18 | it("Test undefined files directory", function() { 19 | return patternDetector.processPatterns(undefined, ruleSet).then(() => Promise.reject(new Error("Expected undefined directory input to result in reject.")), 20 | err => chai.assert.instanceOf(err, Error)); 21 | }); 22 | it("Test incorrect files directory", function() { 23 | return patternDetector.processPatterns(appRoot.path + "/src/nonexistentdirectory", ruleSet).then(() => Promise.reject(new Error("Expected incorrect files directory input to result in reject.")), 24 | err => chai.assert.instanceOf(err, Error)); 25 | }); 26 | it("Testing files that should be excluded", function() { 27 | return patternDetector.processPatterns(appRoot.path + "/test/resources/patterndetector/testExclude", ruleSet).then(response => { 28 | expect(response).to.exist; 29 | expect(response).to.be.not.empty; 30 | expect(response).to.have.lengthOf(1); 31 | }); 32 | }); 33 | it("Testing files that should contain patterns", function() { 34 | return patternDetector.processPatterns(appRoot.path + "/test/resources/patternnonexistence", ruleSet).then(response => { 35 | expect(response).to.exist; 36 | expect(response).to.be.not.empty; 37 | expect(response).to.have.lengthOf(19); // double check this 38 | }); 39 | }); 40 | it("Testing single exclude", function() { 41 | return patternDetector.processPatterns(repo, ruleSet, "rules").then(response => { 42 | expect(response).to.exist; 43 | expect(response).to.be.not.empty; 44 | expect(response).to.have.lengthOf(1); 45 | }); 46 | }); 47 | it("Testing multiple exclude (comma-separated)", function() { 48 | return patternDetector.processPatterns(appRoot.path + "/test/resources/patternnonexistence", ruleSet, "pii_test.txt,ip_addr_test.txt").then(response => { 49 | expect(response).to.exist; 50 | expect(response).to.be.not.empty; 51 | expect(response).to.have.lengthOf(5); 52 | }); 53 | }); 54 | }); -------------------------------------------------------------------------------- /src/rules/FileNonExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Testlet = require("./Testlet.js"); 3 | const dirWrapper = require("../lib/localDirWrapper.js"); 4 | const EvaluationResult = require("./EvaluationResult.js"); 5 | const ioUtils = require("../lib/ioUtils.js"); 6 | const ResultEnum = EvaluationResult.ResultEnum; 7 | const log = require("../lib/common.js").log; 8 | 9 | class FileNonExistenceTestlet extends Testlet { 10 | /** 11 | * Creates the FileExistenceTestlet 12 | * @param {*} target - the target object, for example a directory where this rule is executed 13 | * @param {*} ruleSet 14 | * @param {*} ruleJson 15 | */ 16 | constructor(target, ruleSet, ruleJson, excludeDirs) { 17 | super(target, "FMNE", ruleSet, ruleJson, "", excludeDirs); 18 | } 19 | 20 | /** 21 | * Evaluates the ruleSet 22 | * Returns an array of promises that resolve to a TestletOutput object 23 | */ 24 | evaluate() { 25 | let promises = []; 26 | let dicts = dirWrapper.getDicts(this.target.localdir, true); 27 | let fileDict = dicts[0]; 28 | let fileExtensionDict = dicts[1]; 29 | this.ruleSet.forEach(rule => { 30 | rule.fileList.forEach(fileName =>{ 31 | let exclude = (this.excludeDirs ? rule.excludeDirs.concat(this.excludeDirs) : rule.excludeDirs); 32 | if (fileName.charAt(0)==="*") { 33 | promises.push(this.validateFileExists(rule, fileName.substr(1), fileExtensionDict, exclude)); 34 | } else { 35 | promises.push(this.validateFileExists(rule, fileName, fileDict, exclude)); 36 | } 37 | }); 38 | }); 39 | log.info("Reporting from " + this.ruleType + " resolving results"); 40 | return promises; 41 | } 42 | 43 | /** 44 | * Validates if file exists 45 | * @param {*} rule - the rule 46 | * @param {*} fileName - the file to validate 47 | * @param {*} dict - the dictionary of filenames 48 | * @param {*} excludeDirs 49 | */ 50 | validateFileExists(rule, fileName, dict, excludeDirs) { 51 | let vResult; 52 | return new Promise(resolve => { 53 | try { 54 | let files = (fileName in dict) ? dict[fileName] : []; 55 | if (excludeDirs) { 56 | files = ioUtils.getUnexcludedDirs(files, excludeDirs); 57 | } 58 | const message = (files.length > 0 ? "Found " : "Not found ") + 59 | fileName + " file in " + 60 | (this.target.giturl ? this.target.giturl : this.target.localdir); 61 | const result = files.length > 0 ? ResultEnum.FAIL : ResultEnum.PASS; 62 | const fileList = files.length > 0 ? " Found: " + files : ""; 63 | vResult = new EvaluationResult(rule.ruleID, result, message + fileList); 64 | resolve(vResult); 65 | } catch(exception) { 66 | log.warn("From Testlet for " + this.ruleType + " error evaluating rule"); 67 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, "Error evaluating rule ", JSON.stringify(exception)); 68 | resolve(vResult); 69 | } 70 | }); 71 | } 72 | 73 | } 74 | 75 | module.exports = FileNonExistenceTestlet; -------------------------------------------------------------------------------- /sample/rules.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "FME": [ 4 | { 5 | "fileName": "README", 6 | "description": "Readme file", 7 | "ruleID": "OOS-FME-0001", 8 | "location": "/" 9 | }, 10 | { 11 | "fileName": "LICENSE", 12 | "description": "License file", 13 | "ruleID": "OOS-FME-0002", 14 | "excludeDirs":["node_modules",".git"] 15 | }, 16 | { 17 | "fileName": "CONTRIBUTING", 18 | "description": "Github 'Contributing' file", 19 | "ruleID": "OOS-FME-0003", 20 | "excludeDirs":["node_modules",".git"] 21 | } 22 | ] 23 | }, 24 | { 25 | "FMNE": [ 26 | { 27 | "fileList": [".IDEA", ".ECLIPSE","NODE_MODULES","*.DMG", "*.EXE","*.EXEC", "*.CMD", "*.BIN", "*.COM","*.CLASS","*.PYC","*.JAR","*.WAR","*.DS_STORE","*.GIT","*.LOG"], 28 | "description": "Project files, executables, compiled binaries", 29 | "ruleID": "OOS-FMNE-0001", 30 | "excludeDirs":[".git"] 31 | } 32 | ] 33 | }, 34 | { 35 | "FMNCP": [ 36 | { 37 | "pattern": "(^|[\"'({\\[]|\\s)(10(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){3}|((172\\.(1[6-9]|2[0-9]|3[01]))|192.168)(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){2})([\"',)\\]}]|$|\\s)", 38 | "failureType": "Warning", 39 | "flags": { "ignoreCase": "True" }, 40 | "ruleID": "OOS-FMNCP-0001", 41 | "patternType": "ip address", 42 | "description": "internal ip address(es)", 43 | "excludeDirs":["node_modules",".git"] 44 | }, 45 | { 46 | "pattern": "(^|\\s)TestTerm($|\\s)", 47 | "failureType": "Warning", 48 | "flags": { "ignoreCase": "True" }, 49 | "ruleID": "OOS-FMNCP-0004", 50 | "patternType": "term", 51 | "description": "restricted term", 52 | "excludeDirs":["node_modules",".git"] 53 | }, 54 | { 55 | "pattern": "(^|\\s|\\W)([a-zA-z]{1,})([a-zA-Z0-9._-]{0,})@[a-zA-Z0-9][a-zA-Z0-9-]{1,61}[a-zA-Z0-9](?:\\.[a-zA-Z]{2,})+$", 56 | "failureType": "Warning", 57 | "flags": { "ignoreCase": "True" }, 58 | "ruleID": "OOS-FMNCP-0010", 59 | "patternType": "email", 60 | "description": "email address", 61 | "excludeDirs":["node_modules",".git"] 62 | }, 63 | { 64 | "pattern": "(^|\\s|\\W)(\\(\\d{3}\\)(-|\\s|)|\\d{3}-)\\d{3}-\\d{4}($|\\s|\\W)", 65 | "failureType": "Warning", 66 | "flags": { "ignoreCase": "True" }, 67 | "ruleID": "OOS-FMNCP-0011", 68 | "patternType": "phone", 69 | "description": "phone number", 70 | "excludeDirs":["node_modules",".git"] 71 | }, 72 | { 73 | "pattern": "(^|\\s|\\W)(?!219-09-9999|078-05-1120)(?!666|000|9\\d{2})\\d{3}-(?!00)\\d{2}-(?!0{4})\\d{4}($|\\s|\\W)", 74 | "failureType": "Warning", 75 | "flags": { "ignoreCase": "True" }, 76 | "ruleID": "OOS-FMNCP-0012", 77 | "patternType": "ssn", 78 | "description": "PII information such as SSN", 79 | "excludeDirs":["node_modules",".git"] 80 | } 81 | ] 82 | }, 83 | { 84 | "FMCP": [ 85 | { 86 | "ruleID": "OOS-FMCP-0001", 87 | "pattern": "^(Copyright)(\\s.{1,}\\s){1,}(Company Inc\\.)", 88 | "fileNames": ["LICENSE", "README"], 89 | "patternType": "term", 90 | "description": "Company copyright", 91 | "excludeDirs":["node_modules",".git"] 92 | } 93 | ] 94 | } 95 | ] 96 | -------------------------------------------------------------------------------- /src/rules/SecretKeysNonExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Testlet = require("./Testlet.js"); 3 | const gitDetectWrapper = require("./gitDetectWrapper.js"); 4 | const EvaluationResult = require("./EvaluationResult.js"); 5 | const yaml = require("js-yaml"); 6 | const appRoot = require("app-root-path"); 7 | const gitDetectResults = appRoot.path + "/reports/defectReport.yaml"; 8 | const ResultEnum = EvaluationResult.ResultEnum; 9 | const log = require("../lib/common.js").log; 10 | 11 | class SecretKeysNonExistenceTestlet extends Testlet { 12 | /** 13 | * Creates the FileExistenceTestlet 14 | * @param {*} target - the target object, for example a directory where this rule is executed 15 | * @param {*} ruleSet 16 | * @param {*} ruleJson 17 | */ 18 | constructor(gitUrl, ruleSet) { 19 | super("", "SECRETKEYS", ruleSet, "", gitUrl); 20 | } 21 | 22 | /** 23 | * Evaluates the ruleSet 24 | * Returns an array of promises that resolve to a TestletOutput object 25 | */ 26 | evaluate() { 27 | let promises = []; 28 | let gitToken = process.env.gitToken; 29 | this.ruleSet.forEach(rule => { 30 | promises.push(this.validateDefectsFile(rule, this.target.giturl, gitToken)); 31 | }); 32 | return promises; 33 | } 34 | 35 | /** 36 | * Validates the defects file from the secrets scan 37 | * @param {*} rule 38 | * @param {*} repoUrl 39 | * @param {*} gitToken 40 | */ 41 | validateDefectsFile(rule, repoUrl, gitToken) { 42 | return new Promise(resolve => { 43 | let vResult = []; 44 | 45 | if (!repoUrl || !gitToken) { 46 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, "Error finding secret keys", 47 | JSON.stringify("Please check the github url and token provided.")); 48 | log.info("Reporting from " + this.ruleType + " no git info - returning"); 49 | return resolve(vResult); 50 | } 51 | 52 | let urlTokens = repoUrl.split("/").filter(token => { 53 | return token.trim() != ""; 54 | }); 55 | let repoOwner = urlTokens[urlTokens.length - 2]; 56 | let repoName = urlTokens[urlTokens.length - 1]; 57 | 58 | gitDetectWrapper 59 | .runGitDetect(repoOwner, repoName, gitToken) 60 | .then(data => { 61 | if (data) { 62 | let obj = yaml.load(data); 63 | if (Object.keys(obj["repos"]).length === 0) { 64 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.PASS, 65 | "Not found secret keys in " + repoUrl); 66 | } else { 67 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.FAIL, 68 | "Found Secret keys in " + this.target + ". Check details: " + gitDetectResults); 69 | } 70 | log.info("Reporting from " + this.ruleType + " resolving results"); 71 | resolve(vResult); 72 | } 73 | }) 74 | .catch(exception => { 75 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, 76 | "Error reading defect yaml", JSON.stringify(exception)); 77 | log.info("Reporting from " + this.ruleType + " resolving errors"); 78 | resolve(vResult); 79 | }); 80 | }); 81 | } 82 | } 83 | module.exports = SecretKeysNonExistenceTestlet; -------------------------------------------------------------------------------- /src/rules/ComplianceCalculator.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const utils = require("../lib/utils"); 3 | let logger = require("bunyan"); 4 | let log = logger.createLogger({ name: "standardly" }); 5 | 6 | /** 7 | * The compliance calculator 8 | */ 9 | class ComplianceCalculator { 10 | 11 | constructor(evaluationResults) { 12 | this.evaluationResults = evaluationResults; 13 | this.passed = 0; 14 | this.failed = 0; 15 | this.error = 0; 16 | this.total = 0; 17 | } 18 | 19 | /** 20 | * Calculates compliance statistics (how many rules are passed and failed, and couldn't be evaluated (error)) 21 | */ 22 | calculateComplianceStatistics() { 23 | this.total = this.evaluationResults.length; 24 | this.countPassFailRules(); 25 | this.error = this.total - this.passed - this.failed; 26 | } 27 | 28 | /** 29 | * Calculates the percentage out of total 30 | * @param {*} value 31 | */ 32 | calculatePercentage(value) { 33 | if (this.total === 0) { 34 | return 0; 35 | } 36 | return parseFloat(Number((value * 100) / this.total).toFixed(2)); 37 | } 38 | 39 | /** 40 | * Counts passed and failed rules 41 | */ 42 | countPassFailRules() { 43 | this.evaluationResults.forEach((result) => { 44 | if (result.result.toLowerCase() === "pass") { 45 | ++this.passed; 46 | } else if (result.result.toLowerCase() === "fail") { 47 | ++this.failed; 48 | } 49 | }); 50 | } 51 | 52 | /** 53 | * Formats the compliance summary into (#Passed --> <#Passed> (<#Passed in percentage>)) 54 | * @param {*} label 55 | * @param {*} count 56 | * @param {*} percentage 57 | */ 58 | formatComplianceSummary(label, count, percentage) { 59 | let csvData = label + " --> "; 60 | return csvData.concat(count) 61 | .concat(" (") 62 | .concat(percentage) 63 | .concat("%)\n"); 64 | } 65 | 66 | /** 67 | * Gets the compliance summary in CSV format 68 | */ 69 | getComplianceSummary() { 70 | return new Promise(resolve => { 71 | utils.convertJsonToCsv(this.getHeaderFields(), this.evaluationResults).then((csvData) => { 72 | resolve(csvData.concat(this.getRuleComplianceStatisticsInCSV())); 73 | }).catch(exception => { 74 | log.error(exception); 75 | return resolve([]); 76 | }); 77 | }); 78 | } 79 | 80 | /** 81 | * Returns list of fields as headers for summay section in resuts.csv 82 | */ 83 | getHeaderFields() { 84 | let fields = Object.getOwnPropertyNames(this.evaluationResults[0]); 85 | fields = fields.filter(function(item) { 86 | return item != "detail"; // Details are shown below the statistics section 87 | }); 88 | return fields; 89 | } 90 | 91 | /** 92 | * Converts the compliance statistics into csv 93 | */ 94 | getRuleComplianceStatisticsInCSV() { 95 | let csvData = "\n\n"; 96 | this.calculateComplianceStatistics(); 97 | return csvData.concat(this.formatComplianceSummary("#Passed", this.passed, this.calculatePercentage(this.passed))) 98 | .concat(this.formatComplianceSummary("#Failed", this.failed, this.calculatePercentage(this.failed))) 99 | .concat(this.formatComplianceSummary("#Error", this.error, this.calculatePercentage(this.error))); 100 | } 101 | } 102 | 103 | module.exports = ComplianceCalculator; -------------------------------------------------------------------------------- /src/rules/EvaluationResult.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const detailFieldName = "detail"; 3 | const detailFields = ["ruleID", "error", "detail"]; 4 | 5 | /** 6 | * The data structure representing the Evaluation Result 7 | */ 8 | class EvaluationResult { 9 | constructor(ruleID, result, message, error, detail) { 10 | this.ruleID = ruleID; 11 | this.result = result; 12 | this.message = message; 13 | this.error = error; 14 | this.detail = detail; 15 | } 16 | 17 | /** 18 | * Surrounds the value with quotes if it contains ',' to append it to CSV string 19 | * @param {*} value 20 | */ 21 | encodeValue(value) { 22 | if(value.indexOf(",") > -1) { 23 | value = "\"" + value + "\""; 24 | } 25 | return value; 26 | } 27 | 28 | /** 29 | * Gets the object as string 30 | * Retruns an array of strings 31 | */ 32 | getAsString() { 33 | let result = ""; 34 | if (this[detailFieldName] instanceof Array && this[detailFieldName].length > 0) { 35 | // we need to serialize each item of the detail as a line item 36 | // the line item contains the rest of the 'this' object as a prefix before appending the details 37 | const str = this.serialize(this, false); 38 | result = this.serializeDetailObjects(str, this[detailFieldName]); 39 | } else { 40 | result = this.serialize(this, false); 41 | } 42 | return result; 43 | } 44 | 45 | /** 46 | * Serialize details array 47 | * @param {*} str - string to include before serializing the details object 48 | * @param detail - the detail object to serialize 49 | * @returns serialized string 50 | */ 51 | serializeDetailObjects(str, detail) { 52 | let result = ""; 53 | detail.forEach(element => { 54 | result += str + "," + this.serialize(element, true) + "\n"; 55 | }); 56 | return result; 57 | } 58 | 59 | /** 60 | * Serializes an object 61 | * @param {*} object - the object to serialize 62 | * @param {*} includeKeyName - pass true if the field name should be serialized 63 | * @returns serialized string 64 | */ 65 | serialize(object, includeKeyName) { 66 | let str = ""; 67 | let separator = ""; 68 | let array; 69 | if (object instanceof EvaluationResult) { 70 | array = detailFields; 71 | } else { 72 | array = Object.keys(object); 73 | } 74 | array.forEach(element => { 75 | if(includeKeyName) { 76 | str += separator + element + " : " + object[element]; 77 | separator = "; "; 78 | } else { 79 | if (element !== detailFieldName) { 80 | str += separator + (this[element] === undefined? "" : this.encodeValue(this[element])); 81 | } 82 | separator = ","; 83 | } 84 | }); 85 | if(includeKeyName) { 86 | str = this.encodeValue(str); 87 | } 88 | return str; 89 | } 90 | 91 | /** 92 | * Gets the object property names as a header 93 | * Returns a string 94 | */ 95 | getFieldsAsString() { 96 | let fields = Object.getOwnPropertyNames(this); 97 | return fields.toString(); 98 | } 99 | 100 | } 101 | /** 102 | * Possible values of the actual result 103 | */ 104 | EvaluationResult.ResultEnum = { 105 | PASS: "Pass", 106 | WARN: "Warning", 107 | FAIL: "Fail", 108 | UNKNOWN: "Unknown", 109 | ERROR: "Error" 110 | }; 111 | 112 | module.exports = EvaluationResult; -------------------------------------------------------------------------------- /test/resources/sample_results.csv: -------------------------------------------------------------------------------- 1 | \"ruleID","result","message","error" 2 | "OOS-FME-0001","Pass","Found non-zero length README in https://github.com/intuit/saloon.git", 3 | "OOS-FME-0002","Pass","Found non-zero length LICENSE in https://github.com/intuit/saloon.git", 4 | "OOS-FME-0003","Fail","Not found non-zero length CONTRIBUTING in https://github.com/intuit/saloon.git", 5 | "OOS-FMNE-0001","Pass","Not found .IDEA file in https://github.com/intuit/saloon.git", 6 | "OOS-FMNE-0001","Pass","Not found .ECLIPSE file in https://github.com/intuit/saloon.git", 7 | "OOS-FMNE-0001","Pass","Not found NODE_MODULES file in https://github.com/intuit/saloon.git", 8 | "OOS-FMNE-0001","Pass","Not found .DMG file in https://github.com/intuit/saloon.git", 9 | "OOS-FMNE-0001","Pass","Not found .EXE file in https://github.com/intuit/saloon.git", 10 | "OOS-FMNE-0001","Pass","Not found .EXEC file in https://github.com/intuit/saloon.git", 11 | "OOS-FMNE-0001","Pass","Not found .CMD file in https://github.com/intuit/saloon.git", 12 | "OOS-FMNE-0001","Pass","Not found .BIN file in https://github.com/intuit/saloon.git", 13 | "OOS-FMNE-0001","Pass","Not found .COM file in https://github.com/intuit/saloon.git", 14 | "OOS-FMNE-0001","Pass","Not found .CLASS file in https://github.com/intuit/saloon.git", 15 | "OOS-FMNE-0001","Pass","Not found .PYC file in https://github.com/intuit/saloon.git", 16 | "OOS-FMNE-0001","Pass","Not found .JAR file in https://github.com/intuit/saloon.git", 17 | "OOS-FMNE-0001","Pass","Not found .WAR file in https://github.com/intuit/saloon.git", 18 | "OOS-FMNE-0001","Pass","Not found .DS_STORE file in https://github.com/intuit/saloon.git", 19 | "OOS-FMNE-0001","Pass","Not found .GIT file in https://github.com/intuit/saloon.git", 20 | "OOS-FMNE-0001","Pass","Not found .LOG file in https://github.com/intuit/saloon.git", 21 | "OOS-FMNCP-0011","Fail","Possible phone number found","Warning" 22 | "OOS-FMNCP-0012","Fail","Possible PII information such as SSN found","Warning" 23 | "OOS-FMCP-0001","Fail","Company copyright not found in at least one LICENSE file. Check detail.","" 24 | "OOS-FMCP-0001","Fail","Company copyright not found in at least one README file. Check detail.","" 25 | 26 | #Passed --> 18 (78.26%) 27 | #Failed --> 5 (21.74%) 28 | #Error --> 0 (0%) 29 | 30 | 31 | ruleID,error,detail 32 | OOS-FME-0001, 33 | OOS-FME-0002, 34 | OOS-FME-0003, 35 | OOS-FMNE-0001, 36 | OOS-FMNE-0001, 37 | OOS-FMNE-0001, 38 | OOS-FMNE-0001, 39 | OOS-FMNE-0001, 40 | OOS-FMNE-0001, 41 | OOS-FMNE-0001, 42 | OOS-FMNE-0001, 43 | OOS-FMNE-0001, 44 | OOS-FMNE-0001, 45 | OOS-FMNE-0001, 46 | OOS-FMNE-0001, 47 | OOS-FMNE-0001, 48 | OOS-FMNE-0001, 49 | OOS-FMNE-0001, 50 | OOS-FMNE-0001, 51 | OOS-FMNCP-0011,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/examples/standard/personas/homer_simpson.json; line : 8; col : 22 52 | OOS-FMNCP-0011,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/src/__tests__/__snapshots__/main.js.md; line : 124; col : 18 53 | 54 | OOS-FMNCP-0012,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/examples/standard/personas/homer_simpson.json; line : 24; col : 36 55 | OOS-FMNCP-0012,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/examples/standard/personas/homer_simpson.json; line : 43; col : 36 56 | OOS-FMNCP-0012,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/src/__tests__/__snapshots__/main.js.md; line : 69; col : 24 57 | OOS-FMNCP-0012,Warning,fileName : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/src/__tests__/__snapshots__/main.js.md; line : 86; col : 24 58 | 59 | OOS-FMCP-0001,,detail : Company copyright not found in LICENSE file : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/LICENSE 60 | 61 | OOS-FMCP-0001,,detail : Company copyright not found in README file : /Users/abcdxz/Documents/devrepos/standardly/tmp/saloon/README.md 62 | 63 | -------------------------------------------------------------------------------- /src/rules/PatternNonExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Testlet = require("./Testlet.js"); 3 | const patternDetector = require("./patternDetector.js"); 4 | const EvaluationResult = require("./EvaluationResult.js"); 5 | const ResultEnum = EvaluationResult.ResultEnum; 6 | const log = require("../lib/common.js").log; 7 | 8 | class PatternNonExistenceTestlet extends Testlet { 9 | 10 | /** 11 | * Creates the PatternNonExistenceTestlet 12 | * @param {*} target - the target object, for example a directory where this rule is executed 13 | * @param {*} ruleSet 14 | * @param {*} ruleFileName 15 | */ 16 | constructor(target, ruleSet, ruleFileName, excludeDirs) { 17 | super(target, "FMNCP", ruleSet, ruleFileName, "", excludeDirs); 18 | } 19 | 20 | /** 21 | * Evaluates the ruleSet 22 | * Returns an array of promises that resolve to a TestletOutput object 23 | */ 24 | evaluate() { 25 | return new Promise(resolve => { 26 | let patternResults = []; 27 | patternDetector.processPatterns(this.target.localdir, this.ruleSet, this.excludeDirs) 28 | .then(obj => { 29 | if ((Object.keys(obj).length === 1) && (obj[0]["evaluationStatus"] === "Pass")) { 30 | const vResult = new EvaluationResult("", obj[0]["evaluationStatus"], obj[0]["evaluationMessage"] + " in " + this.target); 31 | patternResults.push(vResult); 32 | } else { 33 | const ruleIDSet = new Set(this.ruleSet.map(value => value.ruleID)); 34 | patternResults = []; 35 | ruleIDSet.forEach(ruleID => { 36 | const result = this.validatePatternExists(obj, ruleID); 37 | if (result) { 38 | patternResults.push(result); 39 | } 40 | }); 41 | } 42 | log.info("Reporting from " + this.ruleType + " resolving results " + patternResults); 43 | return resolve(patternResults); 44 | }).catch(ex => { 45 | const vResult = new EvaluationResult(this.ruleID, ResultEnum.ERROR, "Error evaluating rule ", JSON.stringify(ex)); 46 | patternResults.push(vResult); 47 | return resolve(patternResults); 48 | }); 49 | }); 50 | } 51 | 52 | /** 53 | * Evaluates if patterns exist for different ruleID's 54 | * @param {*} jsonObj 55 | * @param {*} ruleID 56 | * @returns {Promise} 57 | */ 58 | validatePatternExists(jsonObj, ruleID) { 59 | let vResult; 60 | try { 61 | let rules = jsonObj.filter(rule => rule.ruleID === ruleID); 62 | if (rules.length == 0) { 63 | vResult = new EvaluationResult(ruleID, ResultEnum.PASS, "No matches found for ruleID " + ruleID); 64 | } else { 65 | let details = []; 66 | rules.forEach(function(rule) { 67 | details.push({ 68 | fileName: rule.fileName, 69 | line: rule.line, 70 | col: rule.col 71 | }); 72 | }); 73 | vResult = new EvaluationResult(ruleID, ResultEnum.FAIL, 74 | "Possible " + rules[0].description + " found", rules[0].evaluationStatus, details); 75 | return vResult; 76 | } 77 | } catch (exception) { 78 | log.warn("From Testlet for " + this.ruleType + " error evaluating rule"); 79 | vResult = new EvaluationResult(ruleID, ResultEnum.ERROR, "Error evaluating rule ", JSON.stringify(exception)); 80 | return vResult; 81 | } 82 | } 83 | } 84 | 85 | module.exports = PatternNonExistenceTestlet; -------------------------------------------------------------------------------- /test/resources/patternnonexistence/rulesFMNCP.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "FMNCP": [ 4 | { 5 | "pattern": "(^|[\"'({\\[]|\\s)(10(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){3}|((172\\.(1[6-9]|2[0-9]|3[01]))|192.168)(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){2})([\"',)\\]}]|$|\\s)", 6 | "failureType": "Warning", 7 | "flags": { "ignoreCase": "True" }, 8 | "ruleID": "OOS-FMNCP-0001", 9 | "patternType": "ip address", 10 | "description": "internal ip address", 11 | "excludeDirs":["node_modules",".git"] 12 | }, 13 | { 14 | "pattern": "@company.com", 15 | "failureType": "Error", 16 | "flags": { "ignoreCase": "True" }, 17 | "ruleID": "OOS-FMNCP-0002", 18 | "patternType": "url", 19 | "description": "internal urls", 20 | "excludeDirs":["node_modules",".git","testExclude"] 21 | }, 22 | { 23 | "pattern": "((? 0) { 27 | displayHelp(true); 28 | } 29 | 30 | if (!app.giturl && !app.localdir) { 31 | consoleErr("At least one of giturl or localdir must be provided"); // eslint-disable-line no-console 32 | displayHelp(true); 33 | } 34 | 35 | if (app.giturl && app.localdir) { 36 | consoleErr("Only one of giturl or localdir must be provided"); // eslint-disable-line no-console 37 | displayHelp(true); 38 | } 39 | 40 | if (app.localdir) { 41 | target.localdir = app.localdir; 42 | } 43 | 44 | if (!app.rulesfile) { 45 | consoleErr("A Rules json file must be provided"); // eslint-disable-line no-console 46 | displayHelp(true); 47 | } 48 | 49 | if (app.excludeDirs) { 50 | if (app.excludeDirs.indexOf(",") > 0) { 51 | app.excludeDirs = app.excludeDirs.split(","); 52 | } else { 53 | app.excludeDirs = [app.excludeDirs]; 54 | } 55 | } 56 | 57 | if (app.giturl) { 58 | target.giturl = app.giturl; 59 | return fse.emptyDir(tmpDir) 60 | .then(() => { 61 | return gitHubUtils.cloneGitRepo(app.giturl, tmpDir) 62 | .then((response) => { 63 | target.localdir = response; 64 | return validate(target, app.rulesfile, app.excludeDirs); 65 | }) 66 | .catch((err) => { 67 | log.err(err); 68 | }) 69 | .finally(() => { 70 | shelljs.rm("-rf", tmpDir); // may not always have a tmpdir, but no harm in trying to del 71 | }); 72 | }); 73 | } else { 74 | return validate(target, app.rulesfile, app.excludeDirs); 75 | } 76 | } 77 | 78 | /** 79 | * Validate the target against the given set of rules 80 | * @param {*} target 81 | * @param {*} rulesfile 82 | * @param {*} excludeDirs any directories that may need to be excluded from the scan 83 | */ 84 | function validate(target, rulesfile, excludeDirs) { 85 | return new Promise((resolve, reject) => { 86 | const rp = new RulesParser(target, rulesfile, excludeDirs); 87 | return rp.parse() 88 | .then((results) => { 89 | mkDirIfNotExists(app.outputdir); 90 | const resultsfile = path.join(app.outputdir, "results.csv"); 91 | return reporter.reportCompliance(results, resultsfile) 92 | .then(() => { 93 | log.info("Printed results into " + resultsfile); 94 | resolve(true); 95 | }).catch((err) => { 96 | log.error(err); 97 | reject(err); 98 | }); 99 | }); 100 | }); 101 | } 102 | 103 | /** 104 | * Sets the command line options 105 | */ 106 | function setCLOptions() { 107 | app.option("-l, --localdir ", "Local Directory to scan. Only one of Local Directory or Git URL must be provided."); 108 | app.option("-g, --giturl ", "Git URL to scan. Only one of Local Directory or Git URL must be provided."); 109 | app.option("-r, --rulesfile ", "Rules.json file to use for the scan"); 110 | app.option("-o, --outputdir ", "Results.csv file to output the scan results to", path.join(appPath, "reports")); 111 | app.option("-e, --excludeDirs ", "Comma seprated list of directories to be excluded from scanning"); 112 | } 113 | 114 | /** 115 | * Displays help 116 | * @param {*} exit - flag to indicate if we should exit - typically when not all mandatory inputs are given 117 | */ 118 | function displayHelp(exit) { 119 | app.outputHelp(); 120 | if (exit) { 121 | process.exit(1); 122 | } 123 | } 124 | 125 | doRun(); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |
3 | StandardlyLogo 4 |

5 | Enforcing standards, one resource at a time! 6 | 7 |
8 |

CircleCI

9 |

10 | 11 | ![npm](https://img.shields.io/badge/node-v10.16.0-blue.svg) 12 | 13 | *** 14 | 15 | 25 | 26 | 27 | ## Introduction 28 |

Do you find code, documentation, or any other resource veering off standards?

29 |

Is it hard to enforce standards on your team when an upcoming deadline is just around the corner?

30 |

Want to automate part of the code review process?

31 |

Well have no fear, Standardly is here! Standardly is a DIY automation tool for Standards Governance.

32 | 33 |

So how does it work?

34 |

1. You establish standards

35 |

2. You translate standards into rules

36 |

- Rules should be in JSON format

37 |

3. To run the tool, you pass Standardly the following input:

38 |

- Resource to be assessed

39 |

- Rules

40 |

- Location to output the results

41 |

4. You view the results!

42 |

Makes sense? Below is a high-level flow chart of how Standardly works!

43 | Flow of how Standardly works 44 | 45 |

Standardly was designed in mind to work on a number of different resources: code, databases, documents, you name it! For example, for a code resource, rules can revolve around coding standards. For a database resource, rules might be that certain types of data must be encrypted, or data older than a particular date should be archived. Currently, Standardly only supports resources that are files; files that are on your filesystem or on a github repo. Okay enough chit chat, let's get to analysing your resource!

46 | 47 | ## Prerequisites For Installing 48 | * node 10.16.0 49 | * npm 50 | * git 51 | 52 | ## How To Use 53 | 54 | Download/clone this repository, to clone: 55 | 56 | ``` 57 | git clone https://github.com/intuit/standardly.git 58 | ``` 59 | 60 | Change your working directory so that you are inside the Standardly repository: 61 | 62 | ``` 63 | cd standardly 64 | ``` 65 | 66 | Before you can run the tool first install the dependencies it needs: 67 | 68 | ``` 69 | npm install 70 | ``` 71 | 72 | Now Standardly is equipped to start scanning your resource! 73 |

To scan a local directory, run:

74 | 75 | ``` 76 | standardly --localdir --rulesfile 77 | ``` 78 | 79 | or 80 | 81 | ``` 82 | standardly -l -r 83 | ``` 84 | 85 | or if running in a bash shell, simply run 86 | 87 | ``` 88 | ./standardly -l -r 89 | ``` 90 | 91 | 92 | To scan a github repo, run: 93 | 94 | ``` 95 | standardly --giturl --rulesfile 96 | ``` 97 | 98 | or 99 | 100 | ``` 101 | standardly -g -r 102 | ``` 103 | 104 | The output is created as a results.csv file in a folder named 'reports' under the current directory. If you would like to 105 | change the location of the results.csv file pass a --outputdir (or simply -o) parameter to output where you want the 106 | results.csv file to be. Below is an example of explicitly specifying the outputdir. 107 | 108 | ``` 109 | standardly -g https://github.com/argoproj/argo -r /Users/standardlyRocks/Desktop/standardly/sample/rules.json -o /Users/standardlyRocks/Desktop/reports 110 | ``` 111 | 112 | When this command is executed, a results.csv file will be created in the ```/Users/standardlyRocks/Desktop/reports``` directory 113 | 114 | ## Running Tests 115 | ### Unit tests 116 | To run the unit tests in the Standardly repo, in the base directory of the repo run: 117 | 118 | ``` 119 | npm test 120 | ``` 121 | ### Integration tests 122 | 123 | ``` 124 | npm run test:integration 125 | ``` 126 | 127 | ## Extending Standardly To Support New Rules 128 | See [CREATING-RULES.md](docs/CREATING-RULES.md) 129 | 130 | ## How To Contribute 131 | See [CONTRIBUTING.md](CONTRIBUTING.md) 132 | -------------------------------------------------------------------------------- /src/rules/PatternExistenceTestlet.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const Testlet = require("./Testlet.js"); 3 | const EvaluationResult = require("./EvaluationResult.js"); 4 | const dirWrapper = require("../lib/localDirWrapper.js"); 5 | const ioUtils = require("../lib/ioUtils"); 6 | const ResultEnum = EvaluationResult.ResultEnum; 7 | const log = require("../lib/common.js").log; 8 | 9 | class PatternExistenceTestlet extends Testlet { 10 | /** 11 | * Creates the PatternNonExistenceTestlet 12 | * @param {*} target - the target object, for example a directory where this rule is executed 13 | * @param {*} ruleSet 14 | * @param {*} ruleFileName 15 | */ 16 | constructor(target, ruleSet, ruleFileName, excludeDirs) { 17 | super(target, "FMCP", ruleSet, ruleFileName, "", excludeDirs); 18 | } 19 | 20 | 21 | /** 22 | * Evaluates the ruleSet 23 | * Returns an array of promises that resolve to a TestletOutput object 24 | */ 25 | evaluate() { 26 | let promises = []; 27 | let fileDict = dirWrapper.getDicts(this.target.localdir, false)[0]; 28 | this.ruleSet.forEach(rule => { 29 | let files = rule.fileNames; 30 | let exclude = (this.excludeDirs ? rule.excludeDirs.concat(this.excludeDirs) : rule.excludeDirs); 31 | files.forEach(fileName => { 32 | promises.push(this.validatePatternExists(rule, fileDict, fileName, exclude)); 33 | }); 34 | }); 35 | log.info("Reporting from " + this.ruleType + " resolving results"); 36 | return promises; 37 | } 38 | 39 | /** 40 | * Evaluates if patterns exist for different ruleID's 41 | * @param {*} rule 42 | * @param {*} fileDict 43 | * @param {*} fileName 44 | * @param {*} excludeDirs 45 | * @returns {Promise} 46 | */ 47 | validatePatternExists(rule, fileDict, fileName, excludeDirs) { 48 | let vResult; 49 | let regex = rule.pattern; 50 | return new Promise(resolve => { 51 | let exists = (fileName in fileDict); 52 | if (exists) { 53 | let files = fileDict[fileName]; 54 | if (excludeDirs) { 55 | files = ioUtils.getUnexcludedDirs(files, excludeDirs); 56 | } 57 | if (!files) { 58 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, fileName + " file not found; so rule could not be evaluated."); 59 | return resolve(vResult); 60 | } else { 61 | let patternPromises = []; 62 | files.forEach(filePath => { 63 | patternPromises.push(this.checkFilePattern(rule, fileName, filePath, regex)); 64 | }); 65 | Promise.all(patternPromises).then(res => { 66 | let patternExists = res.map(obj => obj.found).every(function(el) { 67 | return el !== null; 68 | }); 69 | let details = []; 70 | res.forEach(obj => { 71 | details.push({detail : obj.detail}); 72 | }); 73 | const result = patternExists ? ResultEnum.PASS : ResultEnum.FAIL; 74 | const message = patternExists ? rule.description + " found in " + fileName + " file. Check detail." : rule.description + " not found in at least one " + fileName + " file. Check detail."; 75 | vResult = new EvaluationResult(rule.ruleID, result, message, "", details); 76 | resolve(vResult); 77 | }); 78 | } 79 | } else { 80 | vResult = new EvaluationResult(rule.ruleID, ResultEnum.ERROR, fileName + " file not found; so rule could not be evaluated."); 81 | resolve(vResult); 82 | } 83 | }); 84 | } 85 | 86 | /** 87 | * Checks if given regex pattern exists in a given file 88 | * @param {*} rule 89 | * @param {*} fileName 90 | * @param {*} filePath 91 | * @param {*} regex 92 | * @returns {Promise} 93 | */ 94 | checkFilePattern(rule, fileName, filePath, regex) { 95 | return new Promise(resolve => { 96 | let found; 97 | let detail; 98 | ioUtils.readFile(filePath).then(data => { 99 | let lines = data.toString().split("\n").filter(element => { 100 | if (element) { 101 | return element; 102 | } 103 | }); 104 | for (let i = 0; i < lines.length && !(found); i++) { 105 | found = lines[i].trim().match(regex); 106 | } 107 | detail = rule.description + (found ? " found in " : " not found in ") + fileName + " file : " + filePath; 108 | let result = {found : found, detail: detail}; 109 | resolve(result); 110 | }).catch(exception => { 111 | const detail = "Error evalutaing rule " + JSON.stringify(exception); 112 | let result = {found: false, detail: detail}; 113 | resolve(result); 114 | }); 115 | }); 116 | } 117 | } 118 | 119 | 120 | module.exports = PatternExistenceTestlet; -------------------------------------------------------------------------------- /test/unit/TestEvaluationResult.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const chai = require("chai"); 3 | const expect = chai.expect; 4 | const EvaluationResult = require("../../src/rules/EvaluationResult"); 5 | 6 | const erOneDetArray = new EvaluationResult("ruleID", "result", "message", "error", [{ "file": "app.js", "col": "10", "line": "10" }]); 7 | const erNoDetArray = new EvaluationResult("OOS-FMNCP-0010", "Fail", "Possible internal github found", "Error", []); 8 | 9 | describe("Test Evaulation Result", function() { 10 | 11 | describe("Test getAsString method", function() { 12 | it("Gets the object as a string when the details message exists", () => { 13 | expect(erOneDetArray.getAsString()).to.be.equal("ruleID,error,file : app.js; col : 10; line : 10\n"); 14 | }); 15 | 16 | it("Gets the object as a string when the details message does not exist", () => { 17 | expect(erNoDetArray.getAsString()).to.be.equal("OOS-FMNCP-0010,Error"); 18 | }); 19 | 20 | }); 21 | 22 | describe("Check ResultEnum is accessible", () => { 23 | it("Checks ResultEnum pass", () => { 24 | expect(EvaluationResult.ResultEnum.PASS).to.equal("Pass"); 25 | }); 26 | 27 | it("Checks ResultEnum fail", () => { 28 | expect(EvaluationResult.ResultEnum.FAIL).to.equal("Fail"); 29 | }); 30 | 31 | it("Checks ResultEnum unknown", () => { 32 | expect(EvaluationResult.ResultEnum.UNKNOWN).to.equal("Unknown"); 33 | }); 34 | 35 | it("Checks ResultEnum error", () => { 36 | expect(EvaluationResult.ResultEnum.ERROR).to.equal("Error"); 37 | }); 38 | 39 | it("Checks ResultEnum warning", () => { 40 | expect(EvaluationResult.ResultEnum.WARN).to.equal("Warning"); 41 | }); 42 | }); 43 | 44 | describe("Test encodeValue method", () => { 45 | it("encodes a value that contains a comma at the end of the string", () => { 46 | const stringWithComma = "stringWithComma,"; 47 | expect(erOneDetArray.encodeValue(stringWithComma)).to.equal("\"stringWithComma,\""); 48 | }); 49 | 50 | it("encodes a value that contains only a comma", () => { 51 | const stringWithComma = ","; 52 | expect(erOneDetArray.encodeValue(stringWithComma)).to.equal("\",\""); 53 | }); 54 | 55 | it("encodes a value that contains no comma", () => { 56 | const stringWithoutComma = "HI"; 57 | expect(erOneDetArray.encodeValue(stringWithoutComma)).to.equal("HI"); 58 | }); 59 | 60 | it("encodes a value that contains the empty string", () => { 61 | const emptyString = ""; 62 | expect(erOneDetArray.encodeValue(emptyString)).to.equal(""); 63 | }); 64 | }); 65 | 66 | describe("Test getFieldsAsString method", () => { 67 | it("Returns all of an evaluationResults objects property names as a string", () => { 68 | expect(erOneDetArray.getFieldsAsString()).to.equal("ruleID,result,message,error,detail"); 69 | expect(erNoDetArray.getFieldsAsString()).to.equal("ruleID,result,message,error,detail"); 70 | }); 71 | }); 72 | 73 | describe("Test serializeDetailObjects method", () => { 74 | const detArrayTwoObjects = [{"a" : "valuea", "b": "valueb"}, {"a" : "valuea1", "b": "valueb1"}]; 75 | const detArrayOneObject = [{"a" : "valuea", "b": "valueb"}]; 76 | const detArrayEmpty = []; 77 | const pref = "somestring"; 78 | const emptyPref = ""; 79 | const erWithDetArray = new EvaluationResult("ruleid1", "Fail", "some failure message", "Error", [{"a" : "valuea", "b": "valueb"}, {"a" : "valuea1", "b": "valueb1"}]); 80 | it("Add the generated details message with the an evaluationResults object details data", () => { 81 | expect(erOneDetArray.serializeDetailObjects(pref, detArrayTwoObjects)).to.equal("somestring,a : valuea; b : valueb\nsomestring,a : valuea1; b : valueb1\n"); 82 | expect(erWithDetArray.serializeDetailObjects(pref, detArrayOneObject)).to.equal("somestring,a : valuea; b : valueb\n"); 83 | }); 84 | it("Add a zero length details message with the erWithDetArray details data", () => { 85 | expect(erWithDetArray.serializeDetailObjects(emptyPref, detArrayEmpty)).to.equal(""); 86 | }); 87 | }); 88 | 89 | describe("Test serialize method", () => { 90 | const detArrayOneString = ["fileName /Users/sampleUser/nameOfProject"]; 91 | const detArrayMultStrings = ["fileName /Users/sampleUser/nameOfProject", "Some other message"]; 92 | it("Checks if details message is in new string format when details message contains one message", () => { 93 | expect(erOneDetArray.serialize(detArrayOneString, true)).to.equal("0 : fileName /Users/sampleUser/nameOfProject"); 94 | }); 95 | 96 | it("Checks if details message is in new string format when details message contains multiple messages", () => { 97 | expect(erNoDetArray.serialize(detArrayMultStrings, true)).to.equal("0 : fileName /Users/sampleUser/nameOfProject; 1 : Some other message"); 98 | }); 99 | 100 | it("Checks if entire block is in new string method", () => { 101 | expect(erNoDetArray.serialize(erNoDetArray, false)).to.equal("OOS-FMNCP-0010,Error"); 102 | }); 103 | }); 104 | }); -------------------------------------------------------------------------------- /test/resources/rulesparser/sample.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "FME": [ 4 | { 5 | "fileName": "README", 6 | "description": "File Must Exist", 7 | "ruleID": "OOS-FME-0001", 8 | "location": "/" 9 | }, 10 | { 11 | "fileName": "LICENSE", 12 | "description": "File Must Exist", 13 | "ruleID": "OOS-FME-0002", 14 | "excludeDirs":["node_modules",".git"] 15 | }, 16 | { 17 | "fileName": "CONTRIBUTING", 18 | "description": "File Must Exist", 19 | "ruleID": "OOS-FME-0003", 20 | "excludeDirs":["node_modules",".git"] 21 | } 22 | ] 23 | }, 24 | { 25 | "FMNE": [ 26 | { 27 | "fileList": [".IDEA", ".ECLIPSE","NODE_MODULES","*.DMG", "*.EXE","*.EXEC", "*.CMD", "*.BIN", "*.COM","*.CLASS","*.PYC","*.JAR","*.WAR","*.DS_STORE","*.GIT","*.LOG"], 28 | "description": "File Must Not Exist", 29 | "ruleID": "OOS-FMNE-0001", 30 | "excludeDirs":[".git"] 31 | } 32 | ] 33 | }, 34 | 35 | { 36 | "FMNCP": [ 37 | { 38 | "pattern": "(^|[\"'({\\[]|\\s)(10(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){3}|((172\\.(1[6-9]|2[0-9]|3[01]))|192.168)(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{1,2}|[0-9]{1,2})){2})([\"',)\\]}]|$|\\s)", 39 | "failureType": "Warning", 40 | "flags": { "ignoreCase": "True" }, 41 | "ruleID": "OOS-FMNCP-0001", 42 | "patternType": "ip address", 43 | "description": "internal ip address", 44 | "excludeDirs":["node_modules",".git"] 45 | }, 46 | { 47 | "pattern": "@company.com", 48 | "failureType": "Error", 49 | "flags": { "ignoreCase": "True" }, 50 | "ruleID": "OOS-FMNCP-0002", 51 | "patternType": "url", 52 | "description": "internal urls", 53 | "excludeDirs":["node_modules",".git"] 54 | }, 55 | { 56 | "pattern": "((? { 18 | file = directoryPath + "/" + file; 19 | var stat = fs.statSync(file); 20 | if (stat && stat.isDirectory()) { 21 | results.push(...walk(file)); 22 | } else { 23 | results.push(fs.realpathSync(file)); 24 | } 25 | }); 26 | return results; 27 | } 28 | 29 | /** 30 | * Finds all the matches for each file in the directory path inputted. 31 | * 32 | * @param {String} directoryPath 33 | * @param {Array} ruleSet 34 | * @param {Array} output 35 | * @param {Array[String]} excludeDirs 36 | */ 37 | function findAllMatches(directoryPath, ruleSet, output, excludeDirs, callback) { 38 | let files = walk(directoryPath); 39 | files.forEach((file) => { 40 | let exclude = false; 41 | excludeDirs.forEach((d) => { 42 | if ((file.includes("/") && (file.toUpperCase().split("/").includes(d.toUpperCase()))) || d.toUpperCase() == file.toUpperCase()) { 43 | exclude = true; 44 | return; 45 | } 46 | }); 47 | if (exclude) { 48 | return; 49 | } 50 | 51 | let fileData = fs.readFileSync(file).toString().split("\n"); 52 | for (var i = 0; i < fileData.length; i++) { 53 | output = matchPatterns(i, fileData[i], file, ruleSet, output); 54 | } 55 | }); 56 | callback(output); 57 | } 58 | 59 | /** 60 | * Searches for regex patterns for the text (specific row of a file) selected. 61 | * 62 | * @param {int} row 63 | * @param {String} text 64 | * @param {String} filename 65 | * @param {Array} ruleSet 66 | * @param {Array} output 67 | * @param {Array} excludeDirs 68 | * 69 | * @returns {Array} output with list of new patterns found concatenated on it 70 | */ 71 | function matchPatterns(row, text, filename, ruleSet, output) { 72 | ruleSet.forEach((p) => { 73 | // Skips if filename is listed in excludeDirs for that specific pattern. 74 | let exclude = false; 75 | if ("excludeDirs" in p) { 76 | p["excludeDirs"].forEach((d) => { 77 | if ((filename.includes("/") && (filename.toUpperCase().split("/").includes(d.toUpperCase()))) || d.toUpperCase() == filename.toUpperCase()) { 78 | exclude = true; 79 | return; 80 | } 81 | }); 82 | if (exclude) { 83 | return; 84 | } 85 | } 86 | 87 | // Regex matching - allows for multiple matching within the same line. 88 | let regex = ((p["flags"]["ignoreCase"] == "True") ? new RegExp(p["pattern"], "gi") : new RegExp(p["pattern"], "g")); 89 | let failureType = p["failureType"]; 90 | 91 | let match = regex.exec(text); 92 | while (match) { 93 | let col = match.index; 94 | output.push({ 95 | "fileName": filename, 96 | "pattern": regex.toString(), 97 | "line": (row + 1).toString(), 98 | "col": (col + 1).toString(), 99 | "evaluationStatus": failureType, 100 | "patternType": p["patternType"], 101 | "description": p["description"], 102 | "evaluationMessage": "Pattern " + regex.toString() + " found.", 103 | "ruleID": p["ruleID"] 104 | }); 105 | 106 | match = regex.exec(text); 107 | } 108 | }); 109 | 110 | return output; 111 | } 112 | 113 | /** 114 | * Finds patterns 115 | * 116 | * @param {*} repo Repo to scan for patterns 117 | * @param {*} ruleSet Rules file to use to scan 118 | * @param {*} excludeInput Files/directories to exclude 119 | * @returns {Promise} 120 | */ 121 | function processPatterns(repo, ruleSet, excludeInput) { 122 | return new Promise((resolve, reject) => { 123 | try { 124 | if (!repo) { 125 | return reject(new Error("Please provide code directory path where patterns have to be found.")); 126 | } else if (!fs.existsSync(repo)) { 127 | return reject(new Error("Path " + repo + " for repo to scan not found.")); 128 | } 129 | 130 | let excludeDirs = []; 131 | if (excludeInput) { 132 | excludeDirs = (excludeInput.includes(",")) ? excludeInput.split(",") : [excludeInput]; 133 | } 134 | 135 | let output = []; 136 | 137 | // Load rules/patterns list 138 | if (!ruleSet) { 139 | output.push({"evaluationStatus": "Pass", 140 | "evaluationMessage": "No key found for rule type " + ruleType + " patterns in rule set."}); 141 | log.info("Reporting from FMNCP resolving results"); 142 | return resolve(output); 143 | } 144 | 145 | // Find all patterns 146 | findAllMatches(repo, ruleSet, output, excludeDirs, (output) => { 147 | if (output.length == 0) { 148 | output.push({"evaluationStatus": "Pass", 149 | "evaluationMessage": "No matches found for rule type " + ruleType + "."}); 150 | } 151 | log.info("Reporting from FMNCP resolving results"); 152 | return resolve(output); 153 | }); 154 | } catch (ex) { 155 | return reject(new Error("ERROR WITH FINDING PATTERNS: " + ex)); 156 | } 157 | }); 158 | } 159 | 160 | module.exports = { 161 | processPatterns: processPatterns 162 | }; -------------------------------------------------------------------------------- /src/lib/ioUtils.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const fs = require("fs"); 3 | 4 | let logger = require("bunyan"); 5 | let log = logger.createLogger({ name: "standardly" }); 6 | 7 | /** 8 | * Promisified function to write to a file asynchronously 9 | * @param {*} filePath 10 | * @param {*} data 11 | * @returns {Promise} 12 | */ 13 | function writeFile(fileName, data) { 14 | return new Promise(function(resolve) { 15 | fs.writeFile(fileName, data, function(err) { 16 | if (err) { 17 | log.error(err); 18 | resolve(false); 19 | } else { 20 | log.debug("Data is written to " + fileName); 21 | resolve(true); 22 | } 23 | }); 24 | }); 25 | } 26 | 27 | /** 28 | * 29 | * @param {*} filePath 30 | * @returns {Promise} 31 | */ 32 | function checkFileExists(filePath) { 33 | return new Promise(function(resolve) { 34 | if (!fs.existsSync(filePath)) { 35 | resolve(false); 36 | } 37 | resolve(true); 38 | }); 39 | } 40 | 41 | /** 42 | * 43 | * @param {*} filePath - the path of the file to read. 44 | * @param defaultEncoding - true if no special encoding needed 45 | * @returns {Promise} 46 | */ 47 | function readFile(filePath, defaultEncoding) { 48 | return new Promise(function(resolve, reject) { 49 | if (!fs.existsSync(filePath)) { 50 | reject(filePath + " doesn't exist"); 51 | } 52 | let opts = defaultEncoding ? {} : {encoding: "utf-8" }; 53 | 54 | fs.readFile(filePath, opts, function(err, data) { 55 | if (err) { 56 | reject(err); 57 | } 58 | resolve(data); 59 | }); 60 | }); 61 | } 62 | 63 | /** 64 | * Checks if file name exists in fileDict and the file is nonempty 65 | * @param {} fileName 66 | * @param {} fileDict 67 | * @param {} excludeDirs 68 | * @param {} location 69 | * @param {} localdir 70 | * @returns true if a non-empty file exists with given fileName, and location or excludeDirs, else @returns false 71 | */ 72 | function checkNonEmptyFileExists(fileName, fileDict, excludeDirs, location, localdir) { 73 | return new Promise(resolve => { 74 | let exists = (fileName in fileDict); 75 | if (exists) { 76 | let files = fileDict[fileName]; 77 | if (excludeDirs) { 78 | files = getUnexcludedDirs(files, excludeDirs); 79 | if (!files){ 80 | return resolve(false); 81 | } 82 | } 83 | if (location && location.length >= 1) { 84 | for (let i = 0; i < files.length; i++){ 85 | let file = files[i]; 86 | let filedir = file.slice(0, file.lastIndexOf("/")); 87 | if (localdir.substr(-1) === "/"){ 88 | localdir = localdir.slice(0, -1); 89 | } 90 | if ((localdir.toUpperCase() == filedir.toUpperCase() && location=="/") || (localdir+"/"+location).toUpperCase() == filedir.toUpperCase()) { 91 | return checkNonEmptyFile(file).then(res => { 92 | return resolve(res); 93 | }); 94 | } 95 | } 96 | resolve(false); 97 | } else { 98 | let res = checkAnyFileNonEmpty(fileDict[fileName]); 99 | return resolve(res); 100 | } 101 | 102 | } else { 103 | resolve(exists); 104 | } 105 | }); 106 | } 107 | 108 | /** 109 | * Checks if atleast one of the files in the list is non-empty 110 | * @param {} fileList - input file list 111 | * @returns true as soon as one file in the list has length > 0 else @returns false 112 | */ 113 | function checkAnyFileNonEmpty(fileList) { 114 | let results = []; 115 | let exists = false; 116 | fileList.forEach(filePath => { 117 | results.push(checkNonEmptyFile(filePath)); 118 | }); 119 | return Promise.all(results).then(results => { 120 | for (let i = 0; i < results.length; i++) { 121 | exists = exists || results[i]; 122 | if (exists) { 123 | return exists; 124 | } 125 | } 126 | return false; 127 | }); 128 | } 129 | 130 | /** 131 | * Checks if a file is non-empty 132 | * @param {} file - input file 133 | * @returns true if the given file has length > 0 else @returns false 134 | */ 135 | function checkNonEmptyFile(file){ 136 | return new Promise(resolve => { 137 | fs.readFile(file, {encoding: "utf-8"}, function(err, data) { 138 | if (err){ 139 | resolve(false); 140 | }else if (data.length > 0) { 141 | resolve(true); 142 | } else { 143 | resolve(false); 144 | } 145 | }); 146 | }); 147 | } 148 | 149 | /** 150 | * Checks if file belongs to an excluded directory 151 | * @param {} fileList - the list of files to check in the excludeDirs 152 | * @param {} excludeDirs - the directory list to be excluded in scans/pattern matches 153 | * @returns outputArray which is a filtered list of files that do not lie in the excluded dirs 154 | */ 155 | function getUnexcludedDirs(fileList, excludeDirs) { 156 | let fileArray = Array.isArray(fileList) ? fileList : [fileList]; 157 | let outputArray = []; 158 | for (let i = 0; i < fileArray.length; i++) { 159 | let found = false; 160 | for (let j = 0; j < excludeDirs.length; j++) { 161 | if (fileArray[i].includes("/") && fileArray[i].toUpperCase().split("/").includes(excludeDirs[j].toUpperCase())) { 162 | found = true; 163 | break; 164 | } 165 | } 166 | if (!found){ 167 | outputArray.push(fileArray[i]); 168 | } 169 | } 170 | return outputArray; 171 | } 172 | 173 | /* 174 | * Creates the folder that is passed in 175 | */ 176 | function mkDirIfNotExists(folder) { 177 | fs.mkdir(folder, {recursive: true}, (err) => { 178 | if (err) { 179 | throw err; 180 | } 181 | }); 182 | } 183 | 184 | module.exports = { 185 | checkNonEmptyFileExists: checkNonEmptyFileExists, 186 | writeFile: writeFile, 187 | checkFileExists: checkFileExists, 188 | readFile: readFile, 189 | getUnexcludedDirs: getUnexcludedDirs, 190 | mkDirIfNotExists: mkDirIfNotExists 191 | }; -------------------------------------------------------------------------------- /src/lib/localDirWrapper.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | const path = require("path"); 3 | const ioUtils = require("./ioUtils"); 4 | const notfound = "404: Not Found"; 5 | const fs = require("fs"); 6 | let logger = require("bunyan"); 7 | let log = logger.createLogger({ name: "standardly" }); 8 | 9 | /** 10 | * Validates if the file exists 11 | * @param {*} dir 12 | * @param {*} filename 13 | */ 14 | function validateFileExists(dir, filename) { 15 | const filePath = getFullFileName(dir, filename); 16 | return new Promise((resolve, reject) => { 17 | ioUtils 18 | .readFile(filePath) 19 | .then(function(content) { 20 | if (!content) { 21 | reject("File " + filename + " not found"); 22 | } 23 | resolve(content); 24 | }) 25 | .catch(function(err) { 26 | log.error(err); 27 | reject(err); 28 | }); 29 | }); 30 | // TODO if fileName starts with *, it should be validated recursively in the subfolders. e.g. */.DS_Store 31 | } 32 | 33 | /** 34 | * Gets a requested file for github 35 | * @param filename - the name of the file to retrieve 36 | * @param org - the org under which this file is 37 | * @param repo - the repo to retrieve from 38 | * @param branch - the branch to retrieve the file from, if null, "master" is assumed 39 | * @returns 40 | */ 41 | function validateFileNotExists(dir, filename) { 42 | const filePath = getFullFileName(dir, filename); 43 | return new Promise((resolve, reject) => { 44 | ioUtils 45 | .checkFileExists(filePath) 46 | .then(function(result) { 47 | if (result === false) { 48 | resolve("ok"); 49 | } 50 | reject("File " + filename + " found"); 51 | }) 52 | .catch(function(err) { 53 | reject(err); 54 | log.error(err); 55 | }); 56 | }); 57 | 58 | // TODO if fileName starts with *, it should be validated recursively in the subfolders. e.g. */.DS_Store 59 | } 60 | 61 | /** 62 | * Gets a requested file for github 63 | * @param filename - the name of the file to retrieve 64 | * @param org - the org under which this file is 65 | * @param repo - the repo to retrieve from 66 | * @param branch - the branch to retrieve the file from, if null, "master" is assumed 67 | * @param expression - regular expression to check the pattern 68 | * @returns 69 | */ 70 | function validateFilePatternExists(dir, filename, ruleExpression) { 71 | const filePath = getFullFileName(dir, filename); 72 | return new Promise((resolve, reject) => { 73 | ioUtils 74 | .readFile(filePath) 75 | .then(content => { 76 | if (content.startsWith(notfound)) { 77 | reject("File " + filename + " not found"); 78 | } 79 | let expression = new RegExp(ruleExpression, "g"); 80 | let check = {}; 81 | if (content.match(expression)) { 82 | check.result = "Pass"; 83 | check.error = ""; 84 | } else { 85 | check.result = "Fail"; 86 | check.error = 87 | "Did not find [" + ruleExpression + "] in file " + filename; 88 | } 89 | resolve(check); 90 | }) 91 | .catch(err => { 92 | reject(err); 93 | }); 94 | }); 95 | // TODO if fileName starts with *, it should be validated recursively in the subfolders. e.g. */.DS_Store 96 | } 97 | 98 | /** 99 | * Get the full file name - dir and fileName concatenated 100 | * @param dir - the directory where file is expected to be 101 | * @param fileName - name of the file 102 | * @returns concatenated directory and fileName, adding path.sep if needed. 103 | */ 104 | function getFullFileName(dir, fileName) { 105 | dir = dir || "."; 106 | let fullFileName; 107 | 108 | if (dir.charAt(dir.length - 1) === path.sep) { 109 | fullFileName = dir + fileName; 110 | } else { 111 | fullFileName = dir + path.sep + fileName; 112 | } 113 | return fullFileName; 114 | } 115 | 116 | /** 117 | * Validates if the file exists and is not empty 118 | * @param {*} fileName 119 | * @param {*} fileDict 120 | * @param {*} excludeDirs 121 | * @returns true if non empty file found, false if not 122 | */ 123 | function validateNonEmptyFileExists(fileName, fileDict, excludeDirs, location, localdir) { 124 | return ioUtils.checkNonEmptyFileExists(fileName, fileDict, excludeDirs, location, localdir); 125 | } 126 | 127 | /** 128 | * Recursively finds all the files in a directory synchronously 129 | * @param {*} dir 130 | * @param {*} fileList 131 | * @param {*} includeDirs 132 | * @returns list of files 133 | */ 134 | function getfileListRecursive(dir, fileList, includeDirs) { 135 | let files = fs.readdirSync(dir); 136 | fileList = fileList || []; 137 | files.forEach(function(file) { 138 | let absFileName = path.join(dir, file); 139 | let stats = fs.lstatSync(absFileName); 140 | if (!stats.isSymbolicLink()) { // skip on symlinks 141 | if (stats.isDirectory()) { 142 | if (includeDirs) { 143 | fileList.push(absFileName); 144 | } 145 | fileList = getfileListRecursive(absFileName, fileList, includeDirs); 146 | } else { 147 | fileList.push(absFileName); 148 | } 149 | } 150 | }); 151 | return fileList; 152 | } 153 | 154 | /** 155 | * Returns an array of two dictionary objects with filename and extension as the key respectively 156 | * and the corresponding file path as the values 157 | * @param {*} dir 158 | * @param {*} includeDirs 159 | * @returns dictionary objects 160 | */ 161 | function getDicts(dir, includeDirs) { 162 | let fileList = getfileListRecursive(dir, [], includeDirs); 163 | let fileDict = {}; 164 | let fileExtensionDict = {}; 165 | fileList.forEach(filePath => { 166 | let fileName = path.basename(filePath); 167 | let index = fileName.indexOf("."); 168 | if (index > 0 && !fs.lstatSync(filePath).isDirectory()){ 169 | let file = fileName.split(".")[0].toUpperCase(); 170 | if (file in fileDict) { 171 | fileDict[file].push(filePath); 172 | } else { 173 | fileDict[file] = [filePath]; 174 | } 175 | let fileExtension = fileName.substring(index).toUpperCase(); 176 | if (fileExtension in fileExtensionDict) { 177 | fileExtensionDict[fileExtension].push(filePath); 178 | } else { 179 | fileExtensionDict[fileExtension] = [filePath]; 180 | } 181 | } else { 182 | let file = fileName.toUpperCase(); 183 | if (file in fileDict) { 184 | fileDict[file].push(filePath); 185 | } else { 186 | fileDict[file] = [filePath]; 187 | } 188 | } 189 | }); 190 | return [fileDict, fileExtensionDict]; 191 | } 192 | 193 | module.exports = { 194 | validateNonEmptyFileExists: validateNonEmptyFileExists, 195 | validateFileExists: validateFileExists, 196 | validateFileNotExists: validateFileNotExists, 197 | validateFilePatternExists: validateFilePatternExists, 198 | getfileListRecursive: getfileListRecursive, 199 | getDicts: getDicts 200 | }; -------------------------------------------------------------------------------- /test/resources/patternexistence/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Company Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------