├── .nvmrc ├── OSSMETADATA ├── .npmrc ├── OutputFiles ├── candidate-converted.json ├── whitesource-due-diligence-converted.json ├── githubMarkdown.md ├── confluenceMarkdown.md ├── outputData.json └── Issues.html ├── InputFiles ├── inputData.json ├── issueLabels.json ├── candidate.json └── whitesource-due-diligence-sample.json ├── package.json ├── LICENSE ├── .github └── workflows │ └── main.yml ├── Utilities ├── generateGitHubMarkdown.js ├── generateConfluenceMarkdown.js ├── convertFromDepsCloud.js ├── generateHtml.js └── convertFromWhiteSourceDueDiligence.js ├── .gitignore ├── findIssues.js ├── INVENTORY_SOURCES.md ├── CODE_OF_CONDUCT.md └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | lts/erbium -------------------------------------------------------------------------------- /OSSMETADATA: -------------------------------------------------------------------------------- 1 | osslifecycle=active 2 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | registry = "https://registry.npmjs.org" 2 | -------------------------------------------------------------------------------- /OutputFiles/candidate-converted.json: -------------------------------------------------------------------------------- 1 | { 2 | "indeedeng/k8dash": 25, 3 | "fasterxml/jackson": 20, 4 | "indeedeng/starfish": 15, 5 | "depscloud/depscloud": 10, 6 | "indeedeng/mariner": 5 7 | } -------------------------------------------------------------------------------- /InputFiles/inputData.json: -------------------------------------------------------------------------------- 1 | { 2 | "skooner-k8s/skooner": 25, 3 | "fasterxml/jackson": 20, 4 | "indeedeng/starfish": 15, 5 | "indeedeng/mariner-issue-collector": 10, 6 | "indeedeng/mariner": 10, 7 | "freeCodeCamp/freeCodeCamp": 5 8 | } 9 | -------------------------------------------------------------------------------- /OutputFiles/whitesource-due-diligence-converted.json: -------------------------------------------------------------------------------- 1 | { 2 | "indeedeng/k8dash": 1, 3 | "fasterxml/jackson": 1, 4 | "indeedeng/mariner": 1, 5 | "indeedeng/starfish": 1, 6 | "depscloud/depscloud": 1, 7 | "indeedeng/mariner-issue-collector": 1 8 | } -------------------------------------------------------------------------------- /InputFiles/issueLabels.json: -------------------------------------------------------------------------------- 1 | [ 2 | "hacktoberfest", 3 | "good+first+issue", 4 | "good first issue", 5 | "documentation", 6 | "help+wanted", 7 | "help wanted", 8 | "first+timers+only", 9 | "first timers only", 10 | "up+for+grabs", 11 | "up for grabs" 12 | ] 13 | -------------------------------------------------------------------------------- /OutputFiles/githubMarkdown.md: -------------------------------------------------------------------------------- 1 | ## Updated: July 14, 2023 at 4:02 PM UTC 2 | 3 | 4 | ### skooner-k8s/skooner 5 | |**Title**|**Age**| 6 | |:----|:----| 7 | 8 | 9 | ### indeedeng/starfish 10 | |**Title**|**Age**| 11 | |:----|:----| 12 | 13 | 14 | ### indeedeng/Mariner-Issue-Collector 15 | |**Title**|**Age**| 16 | |:----|:----| -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Mariner-Issue-Collector", 3 | "version": "0.0.1-alpha", 4 | "description": "A demo app of oss-mariner that can run as a GitHub Action", 5 | "author": "Indeed, Inc", 6 | "license": "MIT", 7 | "dependencies": { 8 | "luxon": "^1.25.0", 9 | "oss-mariner": "^0.3.1-alpha" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/indeedeng/Mariner-Issue-Collector.git" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /InputFiles/candidate.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "repository_url": "https://github.com/skooner-k8s/skooner", 4 | "score": 25 5 | }, 6 | { 7 | "repository_url": "https://github.com/fasterxml/jackson", 8 | "score": 20 9 | }, 10 | { 11 | "repository_url": "https://github.com/indeedeng/starfish", 12 | "score": 15 13 | }, 14 | { 15 | "repository_url": "https://github.com/indeedeng/mariner", 16 | "score": 10 17 | }, 18 | { 19 | "repository_url": "https://github.com/indeedeng/mariner-issue-collector", 20 | "score": 10 21 | }, 22 | { 23 | "repository_url": "https://github.com/freeCodeCamp/freeCodeCamp", 24 | "score": 5 25 | } 26 | ] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Indeed, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Collect Latest Issues 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: '0 */8 * * *' 6 | 7 | jobs: 8 | findIssues: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout Mariner Repo 12 | uses: actions/checkout@v2 13 | - name: Run Install 14 | id: mariner_install 15 | run: | 16 | npm install 17 | - name: Query For Issues 18 | id: mariner_issue_collector 19 | run: | 20 | node findIssues.js 21 | env: 22 | MARINER_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 23 | MARINER_INPUT_FILE_PATH: "./InputFiles/inputData.json" 24 | MARINER_LABELS_FILE_PATH: "./InputFiles/issueLabels.json" 25 | MARINER_OUTPUT_FILE_PATH: "./OutputFiles/outputData.json" 26 | MARINER_MARKDOWN_FILE_PATH: "./OutputFiles/githubMarkdown.md" 27 | MARINER_MAX_ISSUES_AGE: "90" 28 | - name: Format Markdown 29 | id: mariner_markdown_formatter 30 | run: | 31 | node Utilities/generateGitHubMarkdown.js 32 | - name: Commit And Push Issues Output 33 | run: | 34 | git config --local user.email "opensource@indeed.com" 35 | git config --local user.name "Mariner Issue Collector" 36 | git add ./OutputFiles/outputData.json 37 | git add ./OutputFiles/githubMarkdown.md 38 | git commit -m "Mariner auto-generating a fresh issues list" 39 | git push 40 | -------------------------------------------------------------------------------- /Utilities/generateGitHubMarkdown.js: -------------------------------------------------------------------------------- 1 | const { DateTime } = require('luxon') 2 | fs = require('fs') 3 | path = require('path') 4 | 5 | const outputFilePath = 6 | process.env.MARINER_OUTPUT_FILE_PATH || 7 | path.join(__dirname, '..', 'OutputFiles', 'outputData.json'); 8 | const markdownFilePath = 9 | process.env.MARINER_MARKDOWN_FILE_PATH || 10 | path.join(__dirname, '..', 'OutputFiles', 'githubMarkdown.md'); 11 | 12 | const maxIssuesAge = 13 | process.env.MARINER_MAX_ISSUES_AGE || 14 | 30 15 | 16 | const now = DateTime.local() 17 | 18 | var dependencies = {} 19 | markdownArray = [] 20 | 21 | function generateMarkdown() { 22 | markdownArray.push(`## Updated: ${now.toLocaleString(DateTime.DATETIME_FULL)}`) 23 | for(dependency in dependencies) { 24 | if (!dependencies[dependency].length) continue 25 | markdownArray.push('\n') 26 | markdownArray.push(`### ${dependency}`) 27 | markdownArray.push('|**Title**|**Age**|') 28 | markdownArray.push('|:----|:----|') 29 | for(issue in dependencies[dependency]) { 30 | var issueAge = Math.round(now.diff(DateTime.fromISO(dependencies[dependency][issue].createdAt), 'days').days) 31 | if (issueAge < maxIssuesAge) { 32 | markdownArray.push(`|[${dependencies[dependency][issue].title}](${dependencies[dependency][issue].url})|${issueAge} days|`); 33 | } 34 | } 35 | } 36 | } 37 | 38 | fs.exists(outputFilePath, (exists) => { 39 | if (exists) { 40 | const contents = fs.readFileSync(outputFilePath, { 41 | encoding: 'utf8', 42 | }) 43 | dependencies = JSON.parse(contents) 44 | generateMarkdown() 45 | fs.writeFileSync(markdownFilePath, markdownArray.join('\n')) 46 | } else { 47 | console.error("Input file does not exist", outputFilePath) 48 | return 49 | } 50 | }) 51 | -------------------------------------------------------------------------------- /Utilities/generateConfluenceMarkdown.js: -------------------------------------------------------------------------------- 1 | const { DateTime } = require('luxon') 2 | fs = require('fs') 3 | path = require('path') 4 | 5 | const outputFilePath = 6 | process.env.MARINER_OUTPUT_FILE_PATH || 7 | path.join(__dirname, '..', 'OutputFiles', 'outputData.json'); 8 | const markdownFilePath = 9 | process.env.MARINER_MARKDOWN_FILE_PATH || 10 | path.join(__dirname, '..', 'OutputFiles', 'confluenceMarkdown.md'); 11 | 12 | const maxIssuesAge = 13 | process.env.MARINER_MAX_ISSUES_AGE || 14 | 30 15 | 16 | const now = DateTime.local() 17 | 18 | var dependencies = {} 19 | markdownArray = [] 20 | 21 | function generateMarkdown() { 22 | markdownArray.push(`h2. Updated: ${now.toLocaleString(DateTime.DATETIME_FULL)}`) 23 | for(dependency in dependencies) { 24 | if (!dependencies[dependency].length) continue 25 | markdownArray.push('\n') 26 | markdownArray.push(`h3. ${dependency}`) 27 | markdownArray.push('||*Title*||*Age*||') 28 | for(issue in dependencies[dependency]) { 29 | var issueAge = Math.round(now.diff(DateTime.fromISO(dependencies[dependency][issue].createdAt), 'days').days) 30 | if (issueAge < maxIssuesAge) { 31 | markdownArray.push(`|[${dependencies[dependency][issue].title}|${dependencies[dependency][issue].url}]|${issueAge} days|`); 32 | } 33 | } 34 | } 35 | } 36 | 37 | var cleanMarkdown // This may need to be an array ~Damon 38 | generateCleanMarkdown = ()=> { 39 | cleanMarkdown = markdownArray.replace(/{|}/g,''); 40 | } 41 | 42 | fs.exists(outputFilePath, (exists) => { 43 | if (exists) { 44 | const contents = fs.readFileSync(outputFilePath, { 45 | encoding: 'utf8', 46 | }) 47 | dependencies = JSON.parse(contents) 48 | generateMarkdown() 49 | generateCleanMarkdown() 50 | fs.writeFileSync(markdownFilePath, cleanMarkdown.join('\n')) 51 | } else { 52 | console.error("Input file does not exist", outputFilePath) 53 | return 54 | } 55 | }) 56 | -------------------------------------------------------------------------------- /OutputFiles/confluenceMarkdown.md: -------------------------------------------------------------------------------- 1 | h2. Updated: October 25, 2020 5:14 PM 2 | 3 | 4 | h3. indeedeng/k8dash 5 | ||*Title*||*Age*|| 6 | |["Filtering" should support permalinks|https://github.com/indeedeng/k8dash/issues/153]|21 days| 7 | |[Fix styling of dropdown's in dark mode|https://github.com/indeedeng/k8dash/issues/152]|21 days| 8 | |[Use "prefers-color-scheme" to automatically switch to Dark Mode|https://github.com/indeedeng/k8dash/issues/144]|23 days| 9 | 10 | 11 | h3. indeedeng/starfish 12 | ||*Title*||*Age*|| 13 | |[Add configuration options to allow filtering of self-owned repositories|https://github.com/indeedeng/starfish/issues/65]|9 days| 14 | |[replace Moment.js with Luxon|https://github.com/indeedeng/starfish/issues/60]|11 days| 15 | |[Allow filtering based on action types within event types|https://github.com/indeedeng/starfish/issues/58]|12 days| 16 | 17 | 18 | h3. depscloud/depscloud 19 | ||*Title*||*Age*|| 20 | |[Add support for jsonnet-bundler files|https://github.com/depscloud/depscloud/issues/115]|12 days| 21 | |[Add support for prometheus push gateway to indexer|https://github.com/depscloud/depscloud/issues/108]|16 days| 22 | 23 | 24 | h3. indeedeng/Mariner-Issue-Collector 25 | ||*Title*||*Age*|| 26 | |[Catalog potential sources of dependency information.|https://github.com/indeedeng/Mariner-Issue-Collector/issues/19]|3 days| 27 | |[Create a convertFromWhiteSource.js utility|https://github.com/indeedeng/Mariner-Issue-Collector/issues/18]|3 days| 28 | |[Create a generateConfluenceMarkdown.js utility|https://github.com/indeedeng/Mariner-Issue-Collector/issues/17]|3 days| 29 | |[Create a simple generateHtml.js utility|https://github.com/indeedeng/Mariner-Issue-Collector/issues/16]|3 days| 30 | |[Update gh-pages landing page to show found issues.|https://github.com/indeedeng/Mariner-Issue-Collector/issues/15]|3 days| 31 | |[Update the readme to close documentation gaps.|https://github.com/indeedeng/Mariner-Issue-Collector/issues/2]|25 days| -------------------------------------------------------------------------------- /Utilities/convertFromDepsCloud.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | path = require('path') 3 | url = require('url') 4 | 5 | const depsInputPath = 6 | process.env.MARINER_DEPSCLOUD_INPUT || 7 | path.join(__dirname, '..', 'InputFiles', 'candidate.json'); 8 | const depsOutputPath = 9 | process.env.MARINER_DEPSCLOUD_OUTPUT || 10 | path.join(__dirname, '..', 'OutputFiles', 'candidate-converted.json'); 11 | 12 | function convert(deps) { 13 | result = {}; 14 | for(var entry in deps) { 15 | const depsURL = url.parse(deps[entry].repository_url); 16 | const host = depsURL.host 17 | const fullPath = depsURL.pathname; 18 | // Using contains here to give a chance to each URL containing 'github.com' 19 | if (host.indexOf("github.com") >= 0) { 20 | var repoScore = deps[entry].score; 21 | // Count the slashes 22 | var slashCount = (fullPath.match(/\//ig) || []).length; 23 | // Check if pathname contains at least two slashes 24 | if (slashCount >= 2) { 25 | // We follow the slashes to get owner and repo 26 | repoPath = fullPath.split("/"); 27 | var owner = repoPath[1]; 28 | var repo = repoPath[2]; 29 | if (repo.endsWith(".git")) { 30 | repo = repo.replace(/\.git$/, ''); 31 | } 32 | repoIdentifier = owner + "/" + repo; 33 | console.log("** REPO: " + repoIdentifier); 34 | result[repoIdentifier] = repoScore; 35 | } else { 36 | console.log("** REPO: " + fullPath + " Skipped - No OWNER REPO pair"); 37 | } 38 | } else { 39 | console.log("** REPO: " + fullPath + " Skipped - Not a GitHub URL"); 40 | } 41 | } 42 | //Final JSON Product 43 | return result 44 | } 45 | 46 | //Read file, send it out,and get it back 47 | function gather() { 48 | const depsInput = fs.readFileSync(depsInputPath, { 49 | encoding: 'utf8', 50 | }) 51 | deps = JSON.parse(depsInput) 52 | builtJSON = convert(deps) 53 | fs.writeFileSync(depsOutputPath, JSON.stringify(builtJSON, null, 4)) 54 | } 55 | 56 | gather(); 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | # Jetbrains 107 | .idea/ 108 | -------------------------------------------------------------------------------- /InputFiles/whitesource-due-diligence-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "licenses" : [ 3 | { "type" : "Open Source", 4 | "riskScore" : 50, 5 | "library" : "sample-library-name", 6 | "reference" : "License File (sample text)", 7 | "copyright" : "sample copyright holder", 8 | "homepage" : 9 | "https://api.github.com/repos/indeedeng/k8dash/zipball/v2.0.0", 10 | "author" : "sample author", 11 | "projectName" : "sample project name", 12 | "productName" :"sample product name", 13 | "reference_type" : "License File", 14 | "name:" : "ISC" }, 15 | { "type" : "Open Source", 16 | "riskScore" : 50, 17 | "library" : "sample-library-name", 18 | "copyright" : "sample copyright holder", 19 | "homepage" : "https://github.com/fasterxml/jackson/", 20 | "projectName" : "sample project name", 21 | "productName" :"sample product name", 22 | "reference_type" : "POM file", 23 | "name:" : "MIT" }, 24 | { "type" : "Open Source", 25 | "riskScore" : 50, 26 | "library" : "sample-library-name", 27 | "reference" : "https://www.npmjs.org/package/oss-mariner", 28 | "copyright" : "sample copyright holder", 29 | "homepage" : "http://indeedeng.github.io/mariner", 30 | "projectName" : "sample project name", 31 | "productName" :"sample product name", 32 | "reference_type" : "NPM (details available in Node Package Manager)", 33 | "name:" : "Apache 2.0" }, 34 | { "type" : "Open Source", 35 | "riskScore" : 50, 36 | "library" : "sample-library-name", 37 | "reference" : "https://github.com/indeedeng/starfish", 38 | "projectName" : "sample project name", 39 | "productName" :"sample product name", 40 | "reference_type" : "Project home page", 41 | "name:" : "Apache 2.0" }, 42 | { "type" : "Open Source", 43 | "riskScore" : 50, 44 | "library" : "sample-library-name", 45 | "reference" : 46 | "https://repo.maven.apache.org/maven2/com/github/depscloud/depscloud/0.5.0/depscloud-0.5.0.pom", 47 | "homepage" : "http://deps.cloud", 48 | "projectName" : "sample project name", 49 | "productName" :"sample product name", 50 | "reference_type" : "POM file", 51 | "name:" : "Apache 2.0" }, 52 | { "type" : "Open Source", 53 | "riskScore" : 50, 54 | "library" : "sample-library-name", 55 | "reference" : 56 | "License File (https://github.com/indeedeng/mariner-issue-collector/blob/v0.9.0/LICENSE)", 57 | "homepage" : "https://opensource.indeed.com", 58 | "author" : "sample author", 59 | "projectName" : "sample project name", 60 | "productName" :"sample product name", 61 | "reference_type" : "License File", 62 | "name:" : "Apache 2.0" } 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /findIssues.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | path = require('path') 3 | mariner = require('oss-mariner') 4 | 5 | function getFromEnvOrThrow(configField) { 6 | const value = process.env[configField]; 7 | if (!value) { 8 | throw new Error(`${configField} is required`); 9 | } 10 | 11 | return value; 12 | } 13 | 14 | const token = getFromEnvOrThrow('MARINER_GITHUB_TOKEN'); 15 | const inputFilePath = 16 | process.env.MARINER_INPUT_FILE_PATH || 17 | path.join(__dirname, 'InputFiles', 'inputData.json'); 18 | const outputFilePath = 19 | process.env.MARINER_OUTPUT_FILE_PATH || 20 | path.join(__dirname, 'OutputFiles', 'outputData.json'); 21 | 22 | class FancyLogger { 23 | info(message) { 24 | console.log('***INFO: ' + message); 25 | } 26 | error(message) { 27 | console.log('***ERROR: ' + message); 28 | } 29 | } 30 | 31 | const logger = new FancyLogger(); 32 | 33 | logger.info(`Input: ${inputFilePath}`); 34 | logger.info(`Output: ${outputFilePath}`); 35 | 36 | const contents = fs.readFileSync(inputFilePath, { 37 | encoding: 'utf8', 38 | }); 39 | const countsByLibrary = JSON.parse(contents); 40 | const repositoryIdentifiers = Object.keys(countsByLibrary); 41 | 42 | const finder = new mariner.IssueFinder(logger); 43 | 44 | function convertToRecord(issues) { 45 | const record = {}; 46 | repositoryIdentifiers 47 | .sort((a, b) => countsByLibrary[b] - countsByLibrary[a]) 48 | .map(a => record[a] = []) 49 | issues.forEach((issuesForRepo, repo) => { 50 | record[repo] = issuesForRepo; 51 | }); 52 | const jsonFile = outputToJson(record); 53 | 54 | return jsonFile; 55 | } 56 | 57 | function outputToJson(record) { 58 | const noReplacer = undefined; 59 | const indent = 2; 60 | const jsonResults = JSON.stringify(record, noReplacer, indent); 61 | const data = fs.writeFileSync(outputFilePath, jsonResults); 62 | 63 | return data; 64 | } 65 | 66 | function formatLabels(labels) { 67 | return labels.map((label) => label.replace(/\s/g, '+')); 68 | } 69 | 70 | const labelsFilePath = 71 | process.env.MARINER_LABELS_FILE_PATH || 72 | path.join(__dirname, 'InputFiles', 'issueLabels.json'); 73 | 74 | logger.info(`Labels: ${labelsFilePath}`); 75 | 76 | const labelsJSON = fs.readFileSync(labelsFilePath, { 77 | encoding: 'utf8', 78 | }); 79 | const formattedLabels = formatLabels(JSON.parse(labelsJSON)); 80 | 81 | finder 82 | .findIssues(token, formattedLabels, repositoryIdentifiers) 83 | .then((issues) => { 84 | let issueCount = 0; 85 | issues.forEach((issuesForRepo) => { 86 | issueCount += issuesForRepo.length; 87 | }); 88 | 89 | convertToRecord(issues); 90 | logger.info(`Found ${issueCount} issues in ${issues.size} projects\n`); 91 | logger.info(`Saved issue results to: ${outputFilePath}`); 92 | }) 93 | .catch((err) => { 94 | logger.error(err.message); 95 | console.log(err); 96 | }); 97 | -------------------------------------------------------------------------------- /INVENTORY_SOURCES.md: -------------------------------------------------------------------------------- 1 | ## Potential sources of dependency inventory and their current status 2 | 3 | ## Supported Dependency Sources 4 | 5 | These sources are supported via scripts in the Utilities directory. 6 | 7 | |**Source Name**|**Source Description**|**Link for more info**| 8 | |:----|:----|:----| 9 | |deps.cloud|deps.cloud creates a dependency graph for your environment|[deps.cloud/hacktoberfest](https://github.com/depscloud/hacktoberfest#identifying-contribution-candidates)| 10 | 11 | ## Planned Dependency Sources 12 | 13 | These sources are not currently supported, but are planned. 14 | 15 | |**Source Name**|**Source Description**|**Link for more info**| 16 | |:----|:----|:----| 17 | |WhiteSource|WhiteSource is a Source Composition Analysis tool that provides license and vulnerability information about your dependencies|[WhiteSource Documentation](https://whitesource.atlassian.net/wiki/spaces/WD/pages/33915056/Due+Diligence+Report)| 18 | 19 | ## Unsupported Dependency Sources 20 | 21 | These sources are not currently supported. If you would like to propose a Utility that would support one of these sources, please open an issue. 22 | 23 | |**Source Name**|**Source Description**|**Link for more info**| 24 | |:----|:----|:----| 25 | |Debricked|Automates identification of open source vulnerabilities|[Debricked Documentation](https://debricked.com/documentation/solve-your-first-vulnerability/solve-a-vulnerability.html)| 26 | |Dependency-Check|Scans Java and .NET applications to identify the use of known vulnerable components|[Dependency-Check Documentation](https://jeremylong.github.io/DependencyCheck/general/internals.html)
[CLI Usage](https://jeremylong.github.io/DependencyCheck/dependency-check-cli/index.html)| 27 | |Fossa|Automates open source dependency management and security|[Fossa Documentation](https://docs.fossa.com/docs/running-a-scan)| 28 | |GitHub|Dependency graph information for a project on GitHub|[GitHub Preview](https://developer.github.com/v4/previews/#access-to-a-repositories-dependency-graph)| 29 | |GitLab|Find security vulnerabilities in dependencies for a project on GitLab|[GitLab Dependency Scanning](https://docs.gitlab.com/ee/user/application_security/dependency_scanning/)| 30 | |Nexus Repository Manager|Artifact repository that manages dependencies for open source software|[Repo Manager Documentation](https://help.sonatype.com/repomanager3/user-interface/browsing-repositories-and-repository-groups)| 31 | |OSS Review Toolkit|Verify compliance with FOSS licenses|[ORT Documentation](https://github.com/oss-review-toolkit/ort#introduction)| 32 | |ReadyAPI|Software Composition Analysis tool that automates API testing and security|[ReadyAPI Documentation](https://support.smartbear.com/readyapi/docs/general-info/index.html)| 33 | |Snyk|Finds and fixes vulnerabilities in open source projects|[Snyk Documentation](https://support.snyk.io/hc/en-us/articles/360004002558-Reports-overview)
[Reports using JSON](https://snyk.io/blog/getting-the-most-out-of-snyk-test/)| 34 | |Veracode Software Composition Analysis|Detects open source vulnerabilities|[Veracode Scan Documentation](https://help.veracode.com/reader/hHHR3gv0wYc2WbCclECf_A/GZDAOnKxrPhWGbsR4k06sg)
[CLI Usage](https://help.veracode.com/reader/hHHR3gv0wYc2WbCclECf_A/YS3uDX~cMHD5zgAQQjx7qg)| 35 | -------------------------------------------------------------------------------- /Utilities/generateHtml.js: -------------------------------------------------------------------------------- 1 | const { DateTime } = require('luxon') 2 | fs = require('fs') 3 | path = require('path') 4 | 5 | const outputFilePath = 6 | process.env.MARINER_OUTPUT_FILE_PATH || 7 | path.join(__dirname, '..', 'OutputFiles', 'outputData.json'); 8 | const htmlFilePath = 9 | process.env.MARINER_HTML_FILE_PATH || 10 | path.join(__dirname, '..', 'OutputFiles', 'Issues.html'); 11 | 12 | const maxIssuesAge = 13 | process.env.MARINER_MAX_ISSUES_AGE || 14 | 30 15 | 16 | const now = DateTime.local() 17 | 18 | var dependencies = {} 19 | htmlContent = "" 20 | 21 | 22 | function generateHTML() { 23 | htmlContent += ` 24 | 25 | 26 | Issues by Mariner-Issue-Collector 27 | 53 | 54 | 55 | `; 56 | 57 | htmlContent += `

58 | 59 | Updated: ${now.toLocaleString(DateTime.DATETIME_FULL)} 60 |

61 | `; 62 | 63 | for(dependency in dependencies) { 64 | if (!dependencies[dependency].length) continue 65 | htmlContent += `

66 | ${dependency} 67 |

68 | `; 69 | 70 | htmlContent += ` 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | `; 79 | 80 | for(issue in dependencies[dependency]) { 81 | var issueAge = Math.round(now.diff(DateTime.fromISO(dependencies[dependency][issue].createdAt), 'days').days) 82 | if (issueAge < maxIssuesAge) { 83 | htmlContent += ` 84 | 89 | 90 | `; 91 | } 92 | } 93 | htmlContent += ` 94 |
TitleAge
85 | 86 | ${dependencies[dependency][issue].title} 87 | 88 | ${issueAge} days
95 | `; 96 | } 97 | 98 | htmlContent += ` 99 | ` 100 | } 101 | 102 | 103 | fs.exists(outputFilePath, (exists) => { 104 | if (exists) { 105 | const contents = fs.readFileSync(outputFilePath, { 106 | encoding: 'utf8', 107 | }) 108 | dependencies = JSON.parse(contents) 109 | generateHTML() 110 | fs.writeFileSync(htmlFilePath, htmlContent) 111 | } else { 112 | console.error("Input file does not exist", outputFilePath) 113 | return 114 | } 115 | }) 116 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [opensource@indeed.com](mailto:opensource@indeed.com). All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mariner Issue Collector 2 | 3 | ![OSS Lifecycle](https://img.shields.io/osslifecycle/indeedeng/mariner-issue-collector.svg) 4 | 5 | Mariner Issue Collector is a live demo that implements the [Mariner](https://github.com/indeedeng/mariner) library. This demo is configured to run automatically as a GitHub Action. 6 | 7 | Mariner helps you identify recently opened issues in a list of GitHub repositories that you provide. We use Mariner at Indeed to identify beginner-friendly issues that were recently opened in open source projects that we depend on. 8 | 9 | ## Getting Started 10 | 11 | Running the Mariner Issue Collector requires a few steps. 12 | 13 | ### Step 1 : Update the list of repos 14 | 15 | Update the [inputData](./InputFiles/inputData.json) file with repositories and the [issueLabels](./InputFiles/issueLabels.json) file with issue labels you're interested in. We have successfully tested this file with nearly 10,000 repos, which ran in about 15 minutes. 16 | 17 | The inputData.json file should be a JSON object with "owner/repo" as key, and a numeric value that represents the "weight" of the repository. 18 | 19 | The issueLabels.json file should be a JSON array containing issue labels as strings. 20 | 21 | The numerical weights assigned to each repo in the inputData.json file determine the order in which results are listed. Weight should be assigned according to importance to you (e.g. how often a dependency is used in your organization). Results are returned in descending order with the greatest weight listed first. If all weights are equal the results will be listed in the order they appear in the inputData.json object. 22 | 23 | ### Step 2 : Set environment variables 24 | 25 | At a minimum, the demo app will expect you to have set a MARINER_GITHUB_TOKEN environment variable, containing an auth token for the GitHub API. Other environment variables, and their default values, are shown below. 26 | 27 | ``` 28 | MARINER_GITHUB_TOKEN: Your Auth Token 29 | MARINER_LABELS_FILE_PATH: "./InputFiles/issueLabels.json" 30 | MARINER_INPUT_FILE_PATH: "./InputFiles/inputData.json" 31 | MARINER_OUTPUT_FILE_PATH: "./OutputFiles/outputData.json" 32 | MARINER_MARKDOWN_FILE_PATH: "./OutputFiles/githubMarkdown.md" 33 | MARINER_MAX_ISSUES_AGE: "30" 34 | ``` 35 | 36 | MARINER_MAX_ISSUES_AGE sets a limit on the age of issues to be returned, measured in days. You can change it to reflect your desired scope. 37 | 38 | ### Step 3 : Run Mariner 39 | 40 | Start by downloading dependencies: 41 | 42 | ```bash 43 | npm ci 44 | ``` 45 | 46 | Run the findIssues.js script: 47 | 48 | ```bash 49 | node findIssues.js 50 | ``` 51 | 52 | This will update the [outputData](./OutputFiles/outputData.json) file with any issues that Mariner finds. 53 | 54 | The query will return only the data that you specify in the input files. GitHub will automatically paginate results in sets of 30. Mariner will then walk through the paginated results. 55 | 56 | Optionally, generate markdown based on the new set of issues: 57 | 58 | ```bash 59 | node Utilities/generateGitHubMarkdown.js 60 | ``` 61 | 62 | This will parse the outputData.json file and update the [githubMarkdown](./OutputFiles/githubMarkdown.md) with a list of issues that can be easily viewed on GitHub. 63 | 64 | Or generate markdown for use in Confluence/Jira: 65 | 66 | ```bash 67 | node Utilities/generateConfluenceMarkdown.js 68 | ``` 69 | 70 | In Jira, simply copy and paste the contents of [confluenceMarkdown](./OutputFiles/confluenceMarkdown.md) into the text mode of the editor. 71 | 72 | Also, generate static HTML file that shows the new issues: 73 | 74 | ```bash 75 | node Utilities/generateHtml.js 76 | ``` 77 | 78 | This will parse the outputData.json file and update the [Issues.html](./OutputFiles/Issues.html) with the list of issues. 79 | 80 | ## Mariner Issue Collector As A GitHub Action 81 | 82 | Mariner ships with a default GitHub Action that runs every 8 hours to generate a fresh issue list, 83 | and commit that issue list back into the GitHub repository. 84 | Any fork of this repository will automatically include this action, 85 | as it is triggered by the existence of the [action YAML file](./.github/workflows/main.yml). 86 | 87 | ## Use Mariner Issue Collector In A Private Repo 88 | 89 | To use the demo app in a private repo, you will need to create a bare clone of the public project. 90 | 91 | ``` 92 | git clone --bare https://github.com/indeedeng/Mariner-Issue-Collector 93 | ``` 94 | 95 | Next, you will need to move into the clone directory and mirror-push to your private repository. 96 | 97 | ``` 98 | cd Mariner-Issue-Collector.git 99 | git push --mirror https://github.com/YourUserName/YourPrivateRepo 100 | ``` 101 | 102 | You can then remove the public clone. 103 | 104 | ``` 105 | cd .. 106 | rm -rf Mariner-Issue-Collector.git 107 | ``` 108 | 109 | Finally, clone the private repo on your local machine to begin using. 110 | 111 | ``` 112 | git clone https://github.com/YourUserName/YourPrivateRepo 113 | ``` 114 | 115 | ## Getting Help 116 | 117 | If you need help or find bugs, please open an issue. 118 | 119 | ## How To Contribute 120 | 121 | If you see an open issue that you would like to work on, please make sure the issue has the "unclaimed" tag, and that there are no open pull requests for the issue. 122 | 123 | If an issue has the "unclaimed" tag, and you would like to claim it, comment on the issue. A maintainer will add the "claimed" tag to the issue and indicate to whom it has been assigned. 124 | 125 | If you want to propose a new feature, please open an issue and tell us about the contribution you'd like to make. 126 | 127 | **Regarding Hacktoberfest:** We will only tag pull requests with "hacktoberfest accepted" if: 128 | * The pull request references a claimed issue with the Hacktoberfest label 129 | * The pull request was submitted by the person who claimed the issue 130 | 131 | ## Project Maintainers 132 | 133 | This project is primarily maintained by @DuaneOBrien. 134 | 135 | ## Code of Conduct 136 | This project is governed by the [Contributor Covenant v 1.4.1](CODE_OF_CONDUCT.md) 137 | 138 | ## License 139 | This project uses the [MIT](LICENSE) license. 140 | -------------------------------------------------------------------------------- /OutputFiles/outputData.json: -------------------------------------------------------------------------------- 1 | { 2 | "skooner-k8s/skooner": [ 3 | { 4 | "title": "[Feature] Add button to force refresh, and set custom refresh interval", 5 | "createdAt": "2022-08-11T08:57:12Z", 6 | "repositoryNameWithOwner": "skooner-k8s/skooner", 7 | "url": "https://github.com/skooner-k8s/skooner/issues/347" 8 | }, 9 | { 10 | "title": "Sort not working", 11 | "createdAt": "2022-06-28T12:08:22Z", 12 | "repositoryNameWithOwner": "skooner-k8s/skooner", 13 | "url": "https://github.com/skooner-k8s/skooner/issues/338" 14 | }, 15 | { 16 | "title": "CronJobs and Jobs support", 17 | "createdAt": "2022-04-06T13:22:04Z", 18 | "repositoryNameWithOwner": "skooner-k8s/skooner", 19 | "url": "https://github.com/skooner-k8s/skooner/issues/318" 20 | }, 21 | { 22 | "title": "[Feature[ Disable health check success logs", 23 | "createdAt": "2021-07-16T14:01:55Z", 24 | "repositoryNameWithOwner": "skooner-k8s/skooner", 25 | "url": "https://github.com/skooner-k8s/skooner/issues/248" 26 | }, 27 | { 28 | "title": "Use Prometheus for the job number in the workload", 29 | "createdAt": "2021-07-11T17:23:01Z", 30 | "repositoryNameWithOwner": "skooner-k8s/skooner", 31 | "url": "https://github.com/skooner-k8s/skooner/issues/245" 32 | }, 33 | { 34 | "title": "Use Prometheus for the pie charts in the ReplicaSet page", 35 | "createdAt": "2021-07-11T17:20:55Z", 36 | "repositoryNameWithOwner": "skooner-k8s/skooner", 37 | "url": "https://github.com/skooner-k8s/skooner/issues/244" 38 | }, 39 | { 40 | "title": "Display failed to load text instead of loading component", 41 | "createdAt": "2021-07-01T17:50:01Z", 42 | "repositoryNameWithOwner": "skooner-k8s/skooner", 43 | "url": "https://github.com/skooner-k8s/skooner/issues/242" 44 | }, 45 | { 46 | "title": "UI: It would be nice if boxes/frames had labels", 47 | "createdAt": "2020-12-03T14:29:31Z", 48 | "repositoryNameWithOwner": "skooner-k8s/skooner", 49 | "url": "https://github.com/skooner-k8s/skooner/issues/177" 50 | }, 51 | { 52 | "title": "Add additional data to the Pod view", 53 | "createdAt": "2020-12-03T14:28:26Z", 54 | "repositoryNameWithOwner": "skooner-k8s/skooner", 55 | "url": "https://github.com/skooner-k8s/skooner/issues/176" 56 | }, 57 | { 58 | "title": "Fix styling of dropdown's in dark mode", 59 | "createdAt": "2020-10-04T17:11:37Z", 60 | "repositoryNameWithOwner": "skooner-k8s/skooner", 61 | "url": "https://github.com/skooner-k8s/skooner/issues/152" 62 | }, 63 | { 64 | "title": "Use \"prefers-color-scheme\" to automatically switch to Dark Mode", 65 | "createdAt": "2020-10-02T14:09:12Z", 66 | "repositoryNameWithOwner": "skooner-k8s/skooner", 67 | "url": "https://github.com/skooner-k8s/skooner/issues/144" 68 | }, 69 | { 70 | "title": "No logs in dashboard", 71 | "createdAt": "2020-01-27T09:30:38Z", 72 | "repositoryNameWithOwner": "skooner-k8s/skooner", 73 | "url": "https://github.com/skooner-k8s/skooner/issues/61" 74 | }, 75 | { 76 | "title": "Feature Request: Add support for Replication Controllers in Workloads view", 77 | "createdAt": "2020-01-14T10:29:54Z", 78 | "repositoryNameWithOwner": "skooner-k8s/skooner", 79 | "url": "https://github.com/skooner-k8s/skooner/issues/57" 80 | } 81 | ], 82 | "fasterxml/jackson": [], 83 | "indeedeng/starfish": [ 84 | { 85 | "title": "Feature: Allow excluding projects with non-OSI approved licenses", 86 | "createdAt": "2021-10-26T00:33:43Z", 87 | "repositoryNameWithOwner": "indeedeng/starfish", 88 | "url": "https://github.com/indeedeng/starfish/issues/126" 89 | }, 90 | { 91 | "title": "only ignore self-owned events if the project isn't used by others", 92 | "createdAt": "2021-03-02T01:46:48Z", 93 | "repositoryNameWithOwner": "indeedeng/starfish", 94 | "url": "https://github.com/indeedeng/starfish/issues/104" 95 | }, 96 | { 97 | "title": "Add configuration options to allow filtering of employer-owned repositories", 98 | "createdAt": "2020-10-16T01:24:21Z", 99 | "repositoryNameWithOwner": "indeedeng/starfish", 100 | "url": "https://github.com/indeedeng/starfish/issues/66" 101 | }, 102 | { 103 | "title": "Use conditional requests to avoid rate limiting problems", 104 | "createdAt": "2020-06-08T22:24:25Z", 105 | "repositoryNameWithOwner": "indeedeng/starfish", 106 | "url": "https://github.com/indeedeng/starfish/issues/44" 107 | } 108 | ], 109 | "indeedeng/mariner-issue-collector": [], 110 | "indeedeng/mariner": [], 111 | "freeCodeCamp/freeCodeCamp": [], 112 | "indeedeng/Mariner-Issue-Collector": [ 113 | { 114 | "title": "Outputs should either not list projects with no issues, or list projects and say there are no issues", 115 | "createdAt": "2021-12-15T19:43:06Z", 116 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 117 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/40" 118 | }, 119 | { 120 | "title": "Branch Protection conflicts with GitHub Action", 121 | "createdAt": "2021-05-25T22:42:47Z", 122 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 123 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/39" 124 | }, 125 | { 126 | "title": "Add Utilities/README.md", 127 | "createdAt": "2020-10-29T21:18:46Z", 128 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 129 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/30" 130 | }, 131 | { 132 | "title": "Add Mariner Screenshot", 133 | "createdAt": "2020-10-29T21:14:34Z", 134 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 135 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/29" 136 | }, 137 | { 138 | "title": "Catalog potential sources of dependency information.", 139 | "createdAt": "2020-10-22T21:59:51Z", 140 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 141 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/19" 142 | }, 143 | { 144 | "title": "Update gh-pages landing page to show found issues.", 145 | "createdAt": "2020-10-22T21:44:48Z", 146 | "repositoryNameWithOwner": "indeedeng/Mariner-Issue-Collector", 147 | "url": "https://github.com/indeedeng/Mariner-Issue-Collector/issues/15" 148 | } 149 | ] 150 | } -------------------------------------------------------------------------------- /OutputFiles/Issues.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Issues by Mariner-Issue-Collector 5 | 31 | 32 | 33 |

34 | 35 | Updated: October 24, 2020 12:53 AM 36 |

37 |

38 | indeedeng/k8dash 39 |

40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 54 | 55 | 56 | 61 | 62 | 63 | 68 | 69 | 70 |
TitleAge
50 | 51 | "Filtering" should support permalinks 52 | 53 | 18 days
57 | 58 | Fix styling of dropdown's in dark mode 59 | 60 | 19 days
64 | 65 | Use "prefers-color-scheme" to automatically switch to Dark Mode 66 | 67 | 21 days
71 |

72 | indeedeng/starfish 73 |

74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 88 | 89 | 90 | 95 | 96 | 97 | 102 | 103 | 104 |
TitleAge
84 | 85 | Add configuration options to allow filtering of self-owned repositories 86 | 87 | 7 days
91 | 92 | replace Moment.js with Luxon 93 | 94 | 9 days
98 | 99 | Allow filtering based on action types within event types 100 | 101 | 10 days
105 |

106 | depscloud/depscloud 107 |

108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 122 | 123 | 124 | 129 | 130 | 131 |
TitleAge
118 | 119 | Add support for jsonnet-bundler files 120 | 121 | 10 days
125 | 126 | Add support for prometheus push gateway to indexer 127 | 128 | 14 days
132 |

133 | indeedeng/Mariner-Issue-Collector 134 |

135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 149 | 150 | 151 | 156 | 157 | 158 | 163 | 164 | 165 | 170 | 171 | 172 | 177 | 178 | 179 | 184 | 185 | 186 |
TitleAge
145 | 146 | Catalog potential sources of dependency information. 147 | 148 | 0 days
152 | 153 | Create a convertFromWhiteSource.js utility 154 | 155 | 0 days
159 | 160 | Create a generateConfluenceMarkdown.js utility 161 | 162 | 0 days
166 | 167 | Create a simple generateHtml.js utility 168 | 169 | 0 days
173 | 174 | Update gh-pages landing page to show found issues. 175 | 176 | 0 days
180 | 181 | Update the readme to close documentation gaps. 182 | 183 | 23 days
187 | 188 | -------------------------------------------------------------------------------- /Utilities/convertFromWhiteSourceDueDiligence.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | path = require('path') 3 | 4 | const whitesourceInputPath = 5 | process.env.MARINER_WHITESOURCE_INPUT || 6 | path.join(__dirname, '..', 'InputFiles', 'whitesource-due-diligence-sample.json'); 7 | const whitesourceOutputPath = 8 | process.env.MARINER_WHITESOURCE_OUTPUT || 9 | path.join(__dirname, '..', 'OutputFiles', 'whitesource-due-diligence-report-converted.json'); 10 | 11 | var cases = { 12 | "one" : false, 13 | "two" : false, 14 | "three" : false, 15 | "four" : false, 16 | "five" : false, 17 | "six" : false 18 | } 19 | 20 | function stripProtocol(url) { 21 | return url.replace(/^(http|https):\/\//, '') 22 | } 23 | 24 | function explodeUrl(url) { 25 | return url.split('/') 26 | } 27 | 28 | function convert(deps) { 29 | result = {} 30 | sortedOutput = {} 31 | dependencyList = [] 32 | 33 | for (library in deps.licenses) { 34 | var dependency = false 35 | occurrences = 0 36 | homepage = deps.licenses[library].homepage 37 | reference = deps.licenses[library].reference 38 | referenceType = deps.licenses[library].reference_type 39 | 40 | // There are several places that a GitHub URL might show up. 41 | 42 | // Try the homepage first 43 | if(homepage !== undefined && homepage.match(/github/)) { 44 | // Strip out leading http|https 45 | homepage = stripProtocol(homepage) 46 | // Only look at dependencies from GitHub 47 | if (homepage.includes('api.github.com/repos')) { 48 | // Handles cases like apis.github.com/repos/GITHUBUSERORORG/GITHUBREPO/zipball/GITHUBBRANCH 49 | dependency = homepage.replace(/\/zipball\/.+/, '') 50 | dependency = dependency.replace(/api.github.com\/repos\//, '') 51 | // Now it's apis.github.com/repos/GITHUBUSERORORG/GITHUBREPO 52 | if (cases.one === false) { 53 | cases.one = deps.licenses[library] 54 | } 55 | console.log("** REPO CASE 1: " + dependency); 56 | } else if (homepage.match(/^github.com/)) { 57 | // Handles cases like github.com/ORG/REPO/MAYBEMORE 58 | exploded = explodeUrl(homepage) 59 | // Now it's [ 'github.com', 'GITHUBUSERORORG', 'GITHUBREPO', 'MAYBEOTHERSTUFF' ] 60 | if (exploded[2] === undefined || exploded[2] === '') { 61 | // They used a github.com domain but didn't specify a repo 62 | continue 63 | } 64 | dependency = (exploded[1] + '/' + exploded[2]).toLowerCase() 65 | if (cases.two === false) { 66 | cases.two = deps.licenses[library] 67 | } 68 | console.log("** REPO CASE 2: " + dependency); 69 | } else if (homepage.match(/(.*)\.github.io/)) { 70 | // Handles cases like GITHUBUSERORORG.github.io/GITHUBREPO/MAYBEOTHERSTUFF 71 | exploded = explodeUrl(homepage) 72 | if (exploded[1] === undefined) { 73 | // They used a github.io domain but didn't specify a repo 74 | continue 75 | } 76 | // Now it's [ 'GITHUBUSERORORG.github.io', 'GITHUBREPO', 'MAYBEOTHERSTUFF' ] 77 | exploded[0] = exploded[0].replace(/\.github\.io/, '') 78 | // Now it's [ 'GITHUBUSERORORG', 'GITHUBREPO', 'MAYBEOTHERSTUFF' ] 79 | dependency = exploded[0] + '/' + exploded[1] 80 | if (cases.three === false) { 81 | cases.three = deps.licenses[library] 82 | } 83 | console.log("** REPO CASE 3: " + dependency); 84 | } else { 85 | // There is no github in the homepage 86 | // TODO: We should try the other methods/locations for finding GitHub URLs 87 | continue 88 | } 89 | } else if (referenceType !== undefined && reference !== undefined) { 90 | if (referenceType === 'Project home page') { 91 | if (reference.match(/github.com/)) { 92 | // This coule be improved to handle node_modules and github.io pages but it's a good start. 93 | homepage = stripProtocol(reference) 94 | exploded = explodeUrl(homepage) 95 | if (exploded[2] === undefined || exploded[2] === 'LICENSE') { 96 | // They used a github.com domain but didn't specify a repo 97 | continue 98 | } 99 | dependency = exploded[1] + '/' + exploded[2] 100 | if (cases.four === false) { 101 | cases.four = deps.licenses[library] 102 | } 103 | console.log("** REPO CASE 4: " + dependency); 104 | } 105 | } else if (referenceType === 'POM file') { 106 | // To Date, none of the POM file references have had github data in them, but just in case... 107 | if (reference.match(/.+\/com\/github\/.+/i)) { 108 | exploded = explodeUrl(reference) 109 | index = exploded.indexOf('github') 110 | dependency = exploded[index+1] + '/' + exploded[index+2] 111 | if (cases.five === false) { 112 | cases.five = deps.licenses[library] 113 | } 114 | 115 | console.log("** REPO CASE 5: " + dependency) 116 | } 117 | } else if (referenceType === 'License File') { 118 | if (reference.match(/.+github.+/)) { 119 | homepage = stripProtocol(reference.replace(/^.+http/, 'http')) 120 | exploded = explodeUrl(homepage) 121 | if (exploded[2] === undefined || exploded[2] === 'LICENSE') { 122 | // They used a github.com domain but didn't specify a repo 123 | continue 124 | } 125 | dependency = exploded[1] + '/' + exploded[2] 126 | if (cases.six === false) { 127 | cases.six = deps.licenses[library] 128 | } 129 | console.log("** REPO CASE 6: " + dependency); 130 | } 131 | } else if (reference.match(/cdnjs/)){ 132 | // TODO: We can get at these by querying the CDNJS API 133 | // cf https://api.cdnjs.com/libraries?search=bluebird&fields=repository 134 | // cf https://cdnjs.com/api 135 | } else { 136 | // Reference type but no reference, skip this one 137 | continue 138 | } 139 | } else { 140 | // No homepage and no solid reference, skip this one 141 | continue 142 | } 143 | if (dependency !== false) { 144 | dependency = dependency.replace(/\#.+$/, '') 145 | dependency = dependency.replace(/\.git$/, '') 146 | if (result.hasOwnProperty(dependency)) { 147 | occurrences = result[dependency] + 1 148 | } else { 149 | dependencyList.push(dependency) 150 | occurrences = 1 151 | } 152 | result[dependency] = occurrences 153 | } 154 | } 155 | 156 | dependencyList.sort(function (a, b) { 157 | return result[b] - result[a]; 158 | }); 159 | 160 | for (dependency in dependencyList) { 161 | sortedOutput[dependencyList[dependency]] = result[dependencyList[dependency]] 162 | } 163 | 164 | return sortedOutput 165 | } 166 | 167 | //Read file, send it out,and get it back 168 | async function gather() { 169 | const whitesourceInput = fs.readFileSync(whitesourceInputPath, { 170 | encoding: 'utf8' 171 | }) 172 | const whitesourceReport = JSON.parse(whitesourceInput, { 173 | compact: true, spaces: 4 174 | }) 175 | builtJSON = await convert(whitesourceReport) 176 | fs.writeFileSync(whitesourceOutputPath, JSON.stringify(builtJSON, null, 4)) 177 | } 178 | 179 | gather(); 180 | --------------------------------------------------------------------------------