├── .dockerignore ├── .gitattributes ├── .sequelizerc ├── .gitignore ├── start-server.sh ├── webpack.config.js ├── aws-dev └── init.sh ├── Dockerfile ├── client ├── css │ └── style.css ├── views │ ├── alerts.html │ ├── domain.html │ ├── logs.html │ ├── vuln.html │ ├── vulns.html │ ├── home.html │ ├── settings.html │ ├── dashboard.html │ ├── scans.html │ └── launchScan.html ├── package.json ├── js │ ├── services │ │ ├── IntegrationService.js │ │ ├── ScansService.js │ │ └── DomainService.js │ ├── controllers │ │ ├── VulnController.js │ │ ├── DomainController.js │ │ ├── SettingsController.js │ │ ├── Controller.js │ │ ├── ScansController.js │ │ └── DashboardController.js │ ├── app.js │ └── appRoutes.js ├── index.html └── package-lock.json ├── migrations ├── 20190721221652-add_paths.js ├── 20190723161747-add_alerts.js ├── 20190405053904-create-domain.js ├── 20190723030824-add_task_status.js └── 20190405054059-create-vulnerability.js ├── .env.example ├── sequelize-config.js ├── models ├── alert.js ├── domain.js ├── taskstatus.js ├── vulnerability.js └── index.js ├── INTENT.md ├── docker-compose.yml ├── server ├── utils.js ├── bd_api.js ├── h1_api.js ├── routes.js └── scans.js ├── CONTRIBUTORS.md ├── LICENSE.md ├── package.json ├── test └── test.js ├── README.md ├── server.js └── CONTRIBUTING.md /.dockerignore: -------------------------------------------------------------------------------- 1 | tmp/* 2 | .DS_STORE 3 | */.DS_STORE 4 | client/dist/* 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.sequelizerc: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | "config": "./sequelize-config.js" 5 | }; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | client/node_modules 3 | .env 4 | tmp/* 5 | .DS_STORE 6 | */.DS_STORE 7 | config/config.json 8 | client/dist/* 9 | -------------------------------------------------------------------------------- /start-server.sh: -------------------------------------------------------------------------------- 1 | npx sequelize db:migrate 2 | 3 | 4 | if [ $1 = "production" ] 5 | then 6 | forever start server.js 7 | else 8 | npm run develop & nodemon server.js --ignore client 9 | fi 10 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | var webpack = require("webpack"); 2 | module.exports = { 3 | entry: { 4 | app: __dirname + "/client/js/app.js" 5 | }, 6 | output: { 7 | path: __dirname + "/client/dist/", 8 | filename: "app.bundle.js" 9 | } 10 | }; 11 | -------------------------------------------------------------------------------- /aws-dev/init.sh: -------------------------------------------------------------------------------- 1 | # Invoked when initializing localstack 2 | 3 | set -x 4 | 5 | # Create bucket 6 | 7 | awslocal s3 mb s3://crossfeed-output-development 8 | 9 | # Create queue 10 | 11 | awslocal sqs create-queue --queue-name default 12 | 13 | set +x 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:latest 2 | 3 | RUN npm install -g nodemon forever 4 | 5 | RUN apt-get update && apt-get -y install cron 6 | 7 | COPY . app 8 | 9 | WORKDIR /app 10 | 11 | RUN npm install 12 | 13 | RUN npm install --prefix client 14 | 15 | RUN npm run build 16 | 17 | EXPOSE 3000 18 | 19 | CMD [ "node", "server.js" ] 20 | -------------------------------------------------------------------------------- /client/css/style.css: -------------------------------------------------------------------------------- 1 | body { padding-top:30px; } 2 | 3 | td { 4 | padding:10px; 5 | } 6 | 7 | .text-small { 8 | font-size: 18px !important; 9 | } 10 | 11 | .mono { 12 | font-family: monospace; 13 | } 14 | 15 | .chart-med { 16 | height: 330px; 17 | width:330px; 18 | } 19 | 20 | .chart-med h4 { 21 | text-align: center; 22 | } 23 | -------------------------------------------------------------------------------- /migrations/20190721221652-add_paths.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | module.exports = { 4 | up: (queryInterface, Sequelize) => { 5 | return queryInterface.addColumn("Domains", "paths", Sequelize.TEXT); 6 | }, 7 | 8 | down: (queryInterface, Sequelize) => { 9 | return queryInterface.removeColumn("Domains", "paths"); 10 | } 11 | }; 12 | -------------------------------------------------------------------------------- /client/views/alerts.html: -------------------------------------------------------------------------------- 1 |
2 |

Latest alerts

3 | 4 | 5 |
6 | {{alert.text}} 7 |
8 | 9 |
10 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | PG_USERNAME=crossfeed 2 | PG_PASSWORD=password 3 | PG_DATABASE=crossfeed 4 | # Keep as db for docker 5 | PG_HOST=db 6 | 7 | APP_PASSWORD= 8 | SESSION_SECRET=CHANGE_ME 9 | LOG_FILE=/logs 10 | ENVIRONMENT=DEV 11 | NODE_ENV=development 12 | 13 | BD_API_KEY= 14 | 15 | GOOGLE_CLIENT_ID= 16 | GOOGLE_CLIENT_SECRET= 17 | GOOGLE_REDIRECT_URL=http://localhost:3000/auth/google/callback 18 | 19 | AWS_REGION=us-east-1 20 | SQS_URL= 21 | -------------------------------------------------------------------------------- /sequelize-config.js: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); // this is important! 2 | 3 | let config = { 4 | username: process.env.PG_USERNAME, 5 | password: process.env.PG_PASSWORD, 6 | database: process.env.PG_DATABASE, 7 | host: process.env.PG_HOST, 8 | dialect: "postgres", 9 | logging: false 10 | }; 11 | 12 | module.exports = { 13 | development: config, 14 | test: config, 15 | production: config 16 | }; 17 | -------------------------------------------------------------------------------- /client/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "angular": "^1.7.9", 4 | "angular-animate": "^1.7.8", 5 | "angular-chart.js": "^1.1.1", 6 | "angular-route": "^1.7.8", 7 | "angular-ui-bootstrap": "^2.5.6", 8 | "angular-ui-router": "^1.0.22", 9 | "angularjs-toaster": "^3.0.0", 10 | "bootstrap": "^3.3.7", 11 | "jquery": "^3.5.0", 12 | "ng-table": "^3.0.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /models/alert.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = (sequelize, DataTypes) => { 3 | const Alert = sequelize.define( 4 | "Alert", 5 | { 6 | id: { type: DataTypes.INTEGER, primaryKey: true }, 7 | source: DataTypes.STRING, 8 | priority: DataTypes.INTEGER, 9 | text: DataTypes.STRING 10 | }, 11 | {} 12 | ); 13 | Alert.associate = function(models) { 14 | // associations can be defined here 15 | }; 16 | return Alert; 17 | }; 18 | -------------------------------------------------------------------------------- /models/domain.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = (sequelize, DataTypes) => { 3 | const Domain = sequelize.define('Domain', { 4 | name: { type: DataTypes.STRING, unique: true }, 5 | ip: DataTypes.STRING, 6 | ports: DataTypes.STRING, 7 | screenshot: DataTypes.STRING, 8 | services: DataTypes.STRING, 9 | response_data: DataTypes.STRING 10 | }, {}); 11 | Domain.associate = function(models) { 12 | // associations can be defined here 13 | }; 14 | return Domain; 15 | }; -------------------------------------------------------------------------------- /models/taskstatus.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = (sequelize, DataTypes) => { 3 | const TaskStatus = sequelize.define( 4 | "TaskStatus", 5 | { 6 | id: { type: DataTypes.INTEGER, primaryKey: true }, 7 | command: DataTypes.STRING, 8 | status: DataTypes.STRING, 9 | percentage: DataTypes.INTEGER 10 | }, 11 | {} 12 | ); 13 | TaskStatus.associate = function(models) { 14 | // associations can be defined here 15 | }; 16 | return TaskStatus; 17 | }; 18 | -------------------------------------------------------------------------------- /client/js/services/IntegrationService.js: -------------------------------------------------------------------------------- 1 | angular.module("Integration", []).service("Integration", [ 2 | "$http", 3 | function($http) { 4 | this.fetchBDSources = function() { 5 | return $http.get("/api/bd/sources"); 6 | }; 7 | 8 | this.importBDSource = function(id) { 9 | return $http.post("/api/bd/sources/import", { id: id }); 10 | }; 11 | 12 | this.importH1Contents = function(cookie) { 13 | return $http.post("/api/h1/importContents", { cookie: cookie }); 14 | }; 15 | } 16 | ]); 17 | -------------------------------------------------------------------------------- /client/js/controllers/VulnController.js: -------------------------------------------------------------------------------- 1 | angular.module("VulnController", []).controller("VulnController", [ 2 | "$scope", 3 | "$state", 4 | "$stateParams", 5 | "$window", 6 | "Domain", 7 | "NgTableParams", 8 | function($scope, $state, $stateParams, $window, Domain, NgTableParams) { 9 | this.vuln = {}; 10 | 11 | this.fetch = () => { 12 | Domain.fetchOne("vulns", $stateParams.id) 13 | .then(response => { 14 | this.vuln = response.data; 15 | }) 16 | .catch(error => { 17 | console.log(error); 18 | }); 19 | }; 20 | } 21 | ]); 22 | -------------------------------------------------------------------------------- /models/vulnerability.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = (sequelize, DataTypes) => { 3 | const Vulnerability = sequelize.define('Vulnerability', { 4 | hackerone_id: DataTypes.STRING, 5 | title: DataTypes.STRING, 6 | contents: DataTypes.STRING, 7 | state: DataTypes.STRING, 8 | substate: DataTypes.STRING, 9 | weakness: DataTypes.STRING, 10 | severity: DataTypes.STRING, 11 | domains: DataTypes.STRING, 12 | reported_at: DataTypes.STRING, 13 | closed_at: DataTypes.STRING 14 | }, {}); 15 | Vulnerability.associate = function(models) { 16 | // associations can be defined here 17 | }; 18 | return Vulnerability; 19 | }; -------------------------------------------------------------------------------- /client/views/domain.html: -------------------------------------------------------------------------------- 1 |
2 |

{{ctrl.domain.name}}

3 |

{{ctrl.domain.ip}}

4 |

Open domain in new tab

5 | 6 |
7 | 8 |

Open ports: {{ctrl.domain.ports}}

9 | 10 | 11 |

12 | 13 |

services: {{ctrl.domain.services}}

14 |

response_data: {{ctrl.domain.response_data}}

15 |

Created: {{ctrl.domain.createdAt | date:'medium'}}

16 |

Updated: {{ctrl.domain.updatedAt | date:'medium'}}

17 | 18 |
-------------------------------------------------------------------------------- /client/js/controllers/DomainController.js: -------------------------------------------------------------------------------- 1 | angular.module("DomainController", []).controller("DomainController", [ 2 | "$scope", 3 | "$state", 4 | "$stateParams", 5 | "$window", 6 | "Domain", 7 | "NgTableParams", 8 | function($scope, $state, $stateParams, $window, Domain, NgTableParams) { 9 | this.domain = {}; 10 | 11 | this.fetch = () => { 12 | Domain.fetchOne("domains", $stateParams.id) 13 | .then(response => { 14 | this.domain = response.data; 15 | this.domain.link = this.domain.ports.includes("443") 16 | ? "https://" + this.domain.name 17 | : "http://" + this.domain.name; 18 | }) 19 | .catch(error => { 20 | console.log(error); 21 | }); 22 | }; 23 | } 24 | ]); 25 | -------------------------------------------------------------------------------- /client/js/app.js: -------------------------------------------------------------------------------- 1 | require("./controllers/Controller"); 2 | require("./controllers/DashboardController"); 3 | require("./controllers/SettingsController"); 4 | require("./controllers/DomainController"); 5 | require("./controllers/ScansController"); 6 | require("./controllers/VulnController"); 7 | 8 | require("./services/DomainService"); 9 | require("./services/IntegrationService"); 10 | require("./services/ScansService"); 11 | 12 | require("./appRoutes"); 13 | 14 | angular.module("app", [ 15 | "ngRoute", 16 | "ui.router", 17 | "appRoutes", 18 | "Controller", 19 | "DomainController", 20 | "VulnController", 21 | "SettingsController", 22 | "DashboardController", 23 | "ScansController", 24 | "Domain", 25 | "Integration", 26 | "ngTable", 27 | "toaster", 28 | "Scans", 29 | "chart.js", 30 | "ui.bootstrap", 31 | "ngAnimate" 32 | ]); 33 | -------------------------------------------------------------------------------- /INTENT.md: -------------------------------------------------------------------------------- 1 | Licensing Intent 2 | 3 | The intent is that this software and documentation ("Project") should be treated as if it is licensed under the license associated with the Project ("License") in the LICENSE.md file. However, because we are part of the United States (U.S.) Federal Government, it is not that simple. 4 | 5 | The portions of this Project written by United States (U.S.) Federal government employees within the scope of their federal employment are ineligible for copyright protection in the U.S.; this is generally understood to mean that these portions of the Project are placed in the public domain. 6 | 7 | In countries where copyright protection is available (which does not include the U.S.), contributions made by U.S. Federal government employees are released under the License. Merged contributions from private contributors are released under the License. 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | crossfeed-web: 5 | build: . 6 | volumes: 7 | - ./:/app 8 | - /var/spool/cron/crontabs/:/var/spool/cron/crontabs/ 9 | entrypoint: ["/bin/bash", "./start-server.sh", "${NODE_ENV}"] 10 | ports: 11 | - "3000:3000" 12 | 13 | db: 14 | image: postgres:latest 15 | container_name: db 16 | volumes: 17 | - "db-data:/var/lib/postgresql/data" 18 | ports: 19 | - "5432:5432" 20 | environment: 21 | POSTGRES_USER: ${PG_USERNAME} 22 | POSTGRES_PASSWORD: ${PG_PASSWORD} 23 | POSTGRES_DB: ${PG_DATABASE} 24 | 25 | localstack: 26 | image: localstack/localstack 27 | ports: 28 | - "4567-4599:4567-4599" 29 | - "4600:8080" 30 | volumes: 31 | - ./aws-dev:/docker-entrypoint-initaws.d 32 | 33 | volumes: 34 | db-data: 35 | -------------------------------------------------------------------------------- /server/utils.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | // Identifies domains in a given block of text 4 | exports.findDomains = function(text) { 5 | var domains = new Set(); 6 | var exts = ['.mil', '.gov', '.edu', '.com', '.net']; 7 | for (ext of exts) { 8 | var anyFound = false; 9 | var split = text.split(ext); 10 | for (var i=0; i { 4 | return queryInterface.createTable("Alerts", { 5 | source: { 6 | type: Sequelize.STRING 7 | }, 8 | id: { 9 | allowNull: false, 10 | autoIncrement: true, 11 | primaryKey: true, 12 | type: Sequelize.INTEGER 13 | }, 14 | priority: { 15 | type: Sequelize.INTEGER 16 | }, 17 | text: { 18 | type: Sequelize.TEXT 19 | }, 20 | createdAt: { 21 | allowNull: false, 22 | type: Sequelize.DATE, 23 | defaultValue: Sequelize.fn("NOW") 24 | }, 25 | updatedAt: { 26 | allowNull: false, 27 | type: Sequelize.DATE, 28 | defaultValue: Sequelize.fn("NOW") 29 | } 30 | }); 31 | }, 32 | down: (queryInterface, Sequelize) => { 33 | return queryInterface.dropTable("Alerts"); 34 | } 35 | }; 36 | -------------------------------------------------------------------------------- /models/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const path = require("path"); 5 | const Sequelize = require("sequelize"); 6 | const basename = path.basename(__filename); 7 | const db = {}; 8 | 9 | let config = { 10 | username: process.env.PG_USERNAME, 11 | password: process.env.PG_PASSWORD, 12 | database: process.env.PG_DATABASE, 13 | host: process.env.PG_HOST, 14 | dialect: "postgres", 15 | logging: false, 16 | ssl: true 17 | }; 18 | 19 | let sequelize = new Sequelize(config.database, config.username, config.password, config); 20 | 21 | fs.readdirSync(__dirname) 22 | .filter(file => { 23 | return file.indexOf(".") !== 0 && file !== basename && file.slice(-3) === ".js"; 24 | }) 25 | .forEach(file => { 26 | const model = sequelize["import"](path.join(__dirname, file)); 27 | db[model.name] = model; 28 | }); 29 | 30 | Object.keys(db).forEach(modelName => { 31 | if (db[modelName].associate) { 32 | db[modelName].associate(db); 33 | } 34 | }); 35 | 36 | db.sequelize = sequelize; 37 | db.Sequelize = Sequelize; 38 | 39 | module.exports = db; 40 | -------------------------------------------------------------------------------- /client/views/logs.html: -------------------------------------------------------------------------------- 1 |
2 |

Active Scans

3 |
4 |
5 |

No actively running tasks

6 |
7 |
8 |

{{task.command}}

9 | {{task.percentage}}% 12 |
13 |
14 | 15 |

Scan Logs

16 | 17 |
18 | 19 |

Displaying the 1000 most recent lines of logs.

20 |
21 | 
{{ctrl.logs}}
22 | 23 |

Scan History

24 |
25 |
26 |

{{task.status}}: {{task.command}}

27 |

{{task.updatedAt | date}}

28 | {{task.percentage}}% 31 |

32 |
33 |
34 |
35 | -------------------------------------------------------------------------------- /client/js/controllers/SettingsController.js: -------------------------------------------------------------------------------- 1 | angular.module("SettingsController", []).controller("SettingsController", [ 2 | "$scope", 3 | "$state", 4 | "$stateParams", 5 | "$window", 6 | "Integration", 7 | "NgTableParams", 8 | "toaster", 9 | function($scope, $state, $stateParams, $window, Integration, NgTableParams, toaster) { 10 | this.loadBDSources = function() { 11 | Integration.fetchBDSources() 12 | .then(response => { 13 | this.bdsources = response.data.searches; 14 | }) 15 | .catch(error => { 16 | console.log(error); 17 | }); 18 | }; 19 | 20 | this.importBD = function(id) { 21 | Integration.importBDSource(id) 22 | .then(response => { 23 | toaster.pop("success", "Success", response.data.status); 24 | }) 25 | .catch(error => { 26 | console.log(error); 27 | }); 28 | }; 29 | 30 | this.importH1Contents = function() { 31 | Integration.importH1Contents(this.h1cookie) 32 | .then(response => { 33 | console.log(response); 34 | }) 35 | .catch(error => { 36 | console.log(error); 37 | }); 38 | }; 39 | } 40 | ]); 41 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 U.S. Federal Government (in countries where recognized) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /migrations/20190405053904-create-domain.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = { 3 | up: (queryInterface, Sequelize) => { 4 | return queryInterface.createTable("Domains", { 5 | id: { 6 | allowNull: false, 7 | autoIncrement: true, 8 | primaryKey: true, 9 | type: Sequelize.INTEGER 10 | }, 11 | name: { 12 | type: Sequelize.STRING 13 | }, 14 | ip: { 15 | type: Sequelize.STRING 16 | }, 17 | ports: { 18 | type: Sequelize.STRING 19 | }, 20 | services: { 21 | type: Sequelize.STRING 22 | }, 23 | screenshot: { 24 | type: Sequelize.STRING 25 | }, 26 | wappalyzer_data: { 27 | type: Sequelize.STRING 28 | }, 29 | response_data: { 30 | type: Sequelize.TEXT 31 | }, 32 | createdAt: { 33 | allowNull: false, 34 | type: Sequelize.DATE 35 | }, 36 | updatedAt: { 37 | allowNull: false, 38 | type: Sequelize.DATE 39 | } 40 | }); 41 | }, 42 | down: (queryInterface, Sequelize) => { 43 | return queryInterface.dropTable("Domains"); 44 | } 45 | }; 46 | -------------------------------------------------------------------------------- /client/js/services/ScansService.js: -------------------------------------------------------------------------------- 1 | angular.module("Scans", []).service("Scans", [ 2 | "$http", 3 | function($http) { 4 | this.fetchLogs = function() { 5 | return $http.get("/api/scans/logs"); 6 | }; 7 | 8 | this.fetchAlerts = function() { 9 | return $http.get("/api/alerts"); 10 | }; 11 | 12 | this.getConfig = function() { 13 | return $http.get("/api/scans/configure"); 14 | }; 15 | 16 | this.getTasksWithStatus = function(status) { 17 | return $http.get("/api/tasks/" + status); 18 | }; 19 | 20 | this.enqueueJob = function(command, args) { 21 | return $http.post("/api/scans/enqueue", { command: command, args: args }); 22 | }; 23 | 24 | this.scheduleCron = function(args) { 25 | return $http.post("/api/scans/configure", args); 26 | }; 27 | 28 | this.removeCron = function(index) { 29 | return $http.post("/api/scans/remove", { index: index }); 30 | }; 31 | 32 | this.launchScan = function(body) { 33 | return $http.post("/api/scans/launch", body); 34 | }; 35 | 36 | this.previewCount = function(body) { 37 | return $http.post("/api/scans/launch/preview", body); 38 | }; 39 | } 40 | ]); 41 | -------------------------------------------------------------------------------- /migrations/20190723030824-add_task_status.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = { 3 | up: (queryInterface, Sequelize) => { 4 | return queryInterface 5 | .createTable("TaskStatuses", { 6 | command: { 7 | type: Sequelize.STRING 8 | }, 9 | id: { 10 | allowNull: false, 11 | autoIncrement: true, 12 | primaryKey: true, 13 | type: Sequelize.INTEGER 14 | }, 15 | status: { 16 | type: Sequelize.STRING 17 | }, 18 | percentage: { 19 | type: Sequelize.INTEGER 20 | }, 21 | createdAt: { 22 | allowNull: false, 23 | type: Sequelize.DATE, 24 | defaultValue: Sequelize.fn("NOW") 25 | }, 26 | updatedAt: { 27 | allowNull: false, 28 | type: Sequelize.DATE, 29 | defaultValue: Sequelize.fn("NOW") 30 | } 31 | }) 32 | .then(() => 33 | queryInterface.addIndex("TaskStatuses", ["status"], { 34 | name: "status_index", 35 | using: "BTREE" 36 | }) 37 | ); 38 | }, 39 | down: (queryInterface, Sequelize) => { 40 | return queryInterface.dropTable("TaskStatuses"); 41 | } 42 | }; 43 | -------------------------------------------------------------------------------- /client/views/vuln.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 6 | 16 |
17 |

{{ctrl.vuln.title}}

18 |

#{{ctrl.vuln.hackerone_id}}

19 | 20 |

Affected domains: {{ctrl.vuln.domains}}

21 |

State: {{ctrl.vuln.state}} ({{ctrl.vuln.substate}})

22 |

Severity: {{ctrl.vuln.severity}}

23 |

Weakness: {{ctrl.vuln.weakness}}

24 |

Reported at: {{ctrl.vuln.reported_at | date:'medium'}}

25 |

Closed at: {{ctrl.vuln.closed_at | date:'medium'}}

26 | 27 |

Contents:

28 |
{{ctrl.vuln.contents}}
29 |
30 | -------------------------------------------------------------------------------- /migrations/20190405054059-create-vulnerability.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | module.exports = { 3 | up: (queryInterface, Sequelize) => { 4 | return queryInterface.createTable("Vulnerabilities", { 5 | id: { 6 | allowNull: false, 7 | autoIncrement: true, 8 | primaryKey: true, 9 | type: Sequelize.INTEGER 10 | }, 11 | hackerone_id: { 12 | type: Sequelize.INTEGER 13 | }, 14 | title: { 15 | type: Sequelize.TEXT 16 | }, 17 | contents: { 18 | type: Sequelize.TEXT 19 | }, 20 | state: { 21 | type: Sequelize.STRING 22 | }, 23 | substate: { 24 | type: Sequelize.STRING 25 | }, 26 | severity: { 27 | type: Sequelize.STRING 28 | }, 29 | reported_at: { 30 | type: Sequelize.STRING 31 | }, 32 | closed_at: { 33 | type: Sequelize.STRING 34 | }, 35 | weakness: { 36 | type: Sequelize.STRING 37 | }, 38 | domains: { 39 | type: Sequelize.STRING 40 | }, 41 | createdAt: { 42 | allowNull: false, 43 | type: Sequelize.DATE 44 | }, 45 | updatedAt: { 46 | allowNull: false, 47 | type: Sequelize.DATE 48 | } 49 | }); 50 | }, 51 | down: (queryInterface, Sequelize) => { 52 | return queryInterface.dropTable("Vulnerabilities"); 53 | } 54 | }; 55 | -------------------------------------------------------------------------------- /client/views/vulns.html: -------------------------------------------------------------------------------- 1 |
2 |

Vulnerabilities

3 | 4 | 5 | 7 | 9 | 11 | 13 | 15 | 16 | 17 |
6 | {{vuln.title}} 8 | {{vuln.domains | limitTo: 25}}{{vuln.domains.length > 25 ? '...' : ''}} 10 | {{vuln.severity}} 12 | {{vuln.reported_at | formatDate}} 14 | {{vuln.state}}
18 | 19 |

{{count | number}} results

20 |
21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crossfeed-web", 3 | "private": "true", 4 | "main": "server.js", 5 | "dependencies": { 6 | "async": "^2.6.3", 7 | "aws-sdk": "^2.618.0", 8 | "body-parser": "^1.19.0", 9 | "cron": "^1.8.2", 10 | "crontab": "^1.3.0", 11 | "dotenv": "^7.0.0", 12 | "express": "^4.17.1", 13 | "express-basic-auth": "^1.2.0", 14 | "express-session": "^1.17.0", 15 | "fast-csv": "^2.5.0", 16 | "html-to-text": "^4.0.0", 17 | "http-string-parser": "0.0.6", 18 | "method-override": "^2.3.9", 19 | "moment": "^2.24.0", 20 | "morgan": "^1.9.1", 21 | "multer": "^1.4.2", 22 | "ng-table": "^3.0.1", 23 | "npm": "^6.13.7", 24 | "passport": "^0.4.1", 25 | "passport-google-oauth": "^2.0.0", 26 | "pg": "^7.18.1", 27 | "pg-hstore": "^2.3.3", 28 | "read-last-lines": "^1.7.2", 29 | "request": "^2.88.2", 30 | "sequelize": "^4.44.3", 31 | "shell-escape": "^0.2.0" 32 | }, 33 | "devDependencies": { 34 | "chai": "^4.2.0", 35 | "chai-http": "^4.3.0", 36 | "mocha": "^6.2.2", 37 | "sequelize-cli": "^5.5.1", 38 | "webpack": "^4.41.6", 39 | "webpack-cli": "^3.3.11" 40 | }, 41 | "scripts": { 42 | "develop": "webpack --mode development --watch", 43 | "build": "webpack --mode production", 44 | "test": "mocha --exit" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /client/js/services/DomainService.js: -------------------------------------------------------------------------------- 1 | angular.module("Domain", []).service("Domain", [ 2 | "$http", 3 | function($http) { 4 | this.create = function(domain) { 5 | return $http.post("/api/domains", { domain: domain }); 6 | }; 7 | 8 | this.fetch = function(type, queryParams) { 9 | return $http.post("/api/" + type + "/search", queryParams); 10 | }; 11 | 12 | this.fetchOne = function(type, id) { 13 | return $http.get("/api/" + type + "/" + id); 14 | }; 15 | 16 | this.search = function(searchString) { 17 | return $http.get("/api/domains/search?q=" + encodeURIComponent(searchString)); 18 | }; 19 | 20 | this.delete = function(id) { 21 | return $http.delete("/api/domains/" + id); 22 | }; 23 | 24 | // Caches and loads all unique values for a given column 25 | this.loadAll = function(type) { 26 | var savedState = localStorage.getItem(type); 27 | if (savedState) { 28 | var vals = JSON.parse(savedState); 29 | return new Promise(resolve => resolve(vals)); 30 | } else { 31 | return $http.get("/api/values/?type=" + type).then(response => { 32 | if (type == "ports") { 33 | processed = response.data.sort((a, b) => parseInt(a) - parseInt(b)).map(p => ({ title: p, id: p })); 34 | } else { 35 | processed = response.data; 36 | } 37 | localStorage.setItem(type, JSON.stringify(processed)); 38 | return processed; 39 | }); 40 | } 41 | }; 42 | } 43 | ]); 44 | -------------------------------------------------------------------------------- /client/views/home.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | Home 4 | 5 |

6 | 7 |

8 | 9 | 10 | 11 | 14 | 17 | 26 | 34 | 39 | 40 |
12 | {{domain.name}} 13 | 15 | {{domain.ip}} 16 | 24 | {{domain.ports}} 25 | 32 | {{domain.services}} 33 | 35 | 38 |
41 | 42 |

{{count | number}} results

43 |
44 | 45 | 48 | -------------------------------------------------------------------------------- /client/views/settings.html: -------------------------------------------------------------------------------- 1 |
2 |

Settings

3 | 4 |
5 | 6 |

HackerOne Integration

7 |
    8 |
  1. Visit HackerOne and export reports via email.
    2. 9 |
  2. Upload the vulnerability CSV here:

    3. 10 | 11 |
    12 |
    13 | 14 |

    15 |
  3. View uploaded reports at the vulnerabilities tab.
    4. 16 |
  4. Fetch report contents:

    5. 17 |
    18 |
    19 |
    20 | 21 |
    22 |
    23 |
24 | 25 |
26 | 27 |

BitDiscovery Integration

28 | 29 |

30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 |
Source nameResults countDate createdImport subdomains
{{source.keyword}}{{source.results_count}}{{source.created_at | date}}
40 | 41 |
42 | -------------------------------------------------------------------------------- /client/js/appRoutes.js: -------------------------------------------------------------------------------- 1 | angular.module("appRoutes", []).config([ 2 | "$routeProvider", 3 | "$locationProvider", 4 | "$stateProvider", 5 | function($routeProvider, $locationProvider, $stateProvider) { 6 | $stateProvider 7 | .state("home", { 8 | url: "/", 9 | templateUrl: "views/home.html", 10 | controller: "Controller", 11 | controllerAs: "ctrl" 12 | }) 13 | .state("settings", { 14 | url: "/settings", 15 | templateUrl: "views/settings.html", 16 | controller: "SettingsController", 17 | controllerAs: "ctrl" 18 | }) 19 | .state("scans", { 20 | url: "/scans", 21 | templateUrl: "views/scans.html", 22 | controller: "ScansController", 23 | controllerAs: "ctrl" 24 | }) 25 | .state("logs", { 26 | url: "/logs", 27 | templateUrl: "views/logs.html", 28 | controller: "ScansController", 29 | controllerAs: "ctrl" 30 | }) 31 | .state("vulns", { 32 | url: "/vulns", 33 | templateUrl: "views/vulns.html", 34 | controller: "Controller", 35 | controllerAs: "ctrl" 36 | }) 37 | .state("view-domain", { 38 | url: "/domain/:id", 39 | templateUrl: "views/domain.html", 40 | controller: "DomainController", 41 | controllerAs: "ctrl" 42 | }) 43 | .state("view-vuln", { 44 | url: "/vuln/:id", 45 | templateUrl: "views/vuln.html", 46 | controller: "VulnController", 47 | controllerAs: "ctrl" 48 | }) 49 | .state("dashboard", { 50 | url: "/dashboard", 51 | templateUrl: "views/dashboard.html", 52 | controller: "DashboardController", 53 | controllerAs: "ctrl" 54 | }) 55 | .state("alerts", { 56 | url: "/alerts", 57 | templateUrl: "views/alerts.html", 58 | controller: "ScansController", 59 | controllerAs: "ctrl" 60 | }) 61 | .state("launch-scan", { 62 | url: "/scans/launch?vulnId", 63 | templateUrl: "views/launchScan.html", 64 | controller: "ScansController", 65 | controllerAs: "ctrl" 66 | }); 67 | 68 | $locationProvider.html5Mode(true); 69 | } 70 | ]); 71 | -------------------------------------------------------------------------------- /client/views/dashboard.html: -------------------------------------------------------------------------------- 1 |
2 |

Risk Dashboard

3 |

{{ctrl.stats.total}} reports | {{ctrl.stats.numOpen}} open

4 |
5 |
6 |
7 |

Reports over time

8 | 15 | 16 |
17 |
18 |

Severity breakdown

19 | 26 | 27 |
28 |
29 |

Top weaknesses

30 | 36 | 37 |
38 |
39 |


40 |
41 |
42 |

Top reported domains

43 | 49 | 50 |
51 |
52 |

High/critical outstanding reports by domain

53 | 59 | 60 |
61 |
62 |
63 | -------------------------------------------------------------------------------- /server/bd_api.js: -------------------------------------------------------------------------------- 1 | var models = require('../models'); 2 | var Sequelize = require("sequelize"); 3 | var express = require('express'); 4 | var request = require('request'); 5 | var router = express.Router(); 6 | 7 | router.get('/sources', function(req, res) { 8 | request({ 9 | url: 'https://bitdiscovery.com/api/1.0/sources?offset=0&limit=1000', 10 | json:true, 11 | headers: { 12 | 'Authorization': process.env.BD_API_KEY 13 | } 14 | }, (err, res2, body) => { 15 | if (err) { 16 | return res.status(500); 17 | } 18 | res.status(200).json(body); 19 | }) 20 | }); 21 | 22 | router.post('/sources/import', function(req, res) { 23 | request({ 24 | url: 'https://bitdiscovery.com/api/1.0/source/' + req.body.id + '?offset=0&limit=10000&sortorder=true', 25 | method: 'POST', 26 | json: [], 27 | headers: { 28 | 'Authorization': process.env.BD_API_KEY 29 | } 30 | }, async (err, res2, body) => { 31 | if (err) { 32 | return res.status(500); 33 | } 34 | console.log(body) 35 | var numCreated = 0; 36 | var numUpdated = 0; 37 | for (asset of body.assets) { 38 | var domain = { 39 | name: asset['bd.original_hostname'] || asset['bd.hostname'], 40 | ip: asset['bd.ip_address'], 41 | screenshot: asset['screenshot.screenshot'] == 'no' ? null : asset['screenshot.screenshot'], 42 | ports: asset['ports.ports'] ? asset['ports.ports'].join(',') : null, 43 | services: asset['ports.services'] ? asset['ports.services'].join(',') : null, 44 | response_data: asset['ports.banners'] ? asset['ports.banners'].join(',') : null 45 | } 46 | await models.Domain.findOne({ where: {name: domain.name} }) 47 | .then(function(obj) { 48 | if(obj) { 49 | numUpdated++; 50 | return obj.update(domain); 51 | } 52 | else { // insert 53 | numCreated++; 54 | return models.Domain.create(domain); 55 | } 56 | }) 57 | } 58 | console.log(`BD import success. ${numCreated} created and ${numUpdated} updated.`) 59 | res.status(200).json({'status': `Success. ${numCreated} created and ${numUpdated} updated.`}); 60 | }) 61 | }); 62 | 63 | module.exports = router; -------------------------------------------------------------------------------- /client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Crossfeed 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
34 | 35 | 49 | 50 | 51 | 52 |
53 |
54 | 55 | 56 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var chai = require("chai"); 2 | var chaiHttp = require("chai-http"); 3 | 4 | chai.use(chaiHttp); 5 | chai.should(); 6 | 7 | var app = require("../server"); 8 | 9 | describe("Server", () => { 10 | describe("GET /", () => { 11 | it("should respond 302 when unauthenticated", done => { 12 | chai 13 | .request(app) 14 | .get("/") 15 | .redirects(0) 16 | .end((err, res) => { 17 | res.should.have.status(302); 18 | done(); 19 | }); 20 | }); 21 | it("should respond 200 when authenticated", done => { 22 | chai 23 | .request(app) 24 | .get("/") 25 | .auth("admin", process.env.APP_PASSWORD) 26 | .end((err, res) => { 27 | res.should.have.status(200); 28 | done(); 29 | }); 30 | }); 31 | }); 32 | describe("POST /api/domains/search", () => { 33 | it("should return domains", done => { 34 | chai 35 | .request(app) 36 | .post("/api/domains/search") 37 | .auth("admin", process.env.APP_PASSWORD) 38 | .end((err, res) => { 39 | res.should.have.status(200); 40 | res.body.should.be.a("object"); 41 | done(); 42 | }); 43 | }); 44 | }); 45 | describe("GET /api/scans/logs", () => { 46 | it("should return logs", done => { 47 | chai 48 | .request(app) 49 | .get("/api/scans/logs") 50 | .auth("admin", process.env.APP_PASSWORD) 51 | .end((err, res) => { 52 | res.should.have.status(200); 53 | res.body.should.be.a("object"); 54 | done(); 55 | }); 56 | }); 57 | }); 58 | describe("GET /api/scans/configure", () => { 59 | it("should return scheduled cron jobs", done => { 60 | chai 61 | .request(app) 62 | .get("/api/scans/configure") 63 | .auth("admin", process.env.APP_PASSWORD) 64 | .end((err, res) => { 65 | res.should.have.status(200); 66 | res.body.should.be.a("object"); 67 | done(); 68 | }); 69 | }); 70 | }); 71 | describe("POST /api/scans/launch/preview", () => { 72 | it("should allow previewing a scan", done => { 73 | chai 74 | .request(app) 75 | .post("/api/scans/launch/preview") 76 | .send({ filters: { ports: ["80"], services: ["PHP"] } }) 77 | .auth("admin", process.env.APP_PASSWORD) 78 | .end((err, res) => { 79 | res.should.have.status(200); 80 | res.body.should.be.a("object"); 81 | done(); 82 | }); 83 | }); 84 | }); 85 | }); 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This repository has been archived. Please see https://github.com/deptofdefense/Crossfeed for the new, redesigned version of Crossfeed. 2 | 3 | # Crossfeed 4 | 5 | External monitoring for organization assets 6 | 7 | Crossfeed is a tool that blends external asset information with known vulnerabilities from the VDP in order to better secure DoD systems. Crossfeed continually scans for public facing assets using a number of OSINT and minimally invasive techniques. This information is then used in scans to discover indicators of vulnerabilities. 8 | 9 | Current features: 10 | 11 | - Continually tracked database of DoD assets 12 | - Database of vulnerability reports from VDP 13 | - Passive scans for open ports utilizing Rapid7's [Project Sonar](https://www.rapid7.com/research/project-sonar/) 14 | - Host fingerprinting using [Wappalyzer](https://www.wappalyzer.com/) 15 | - Recurring vulnerability scans based on past vulnerabilities 16 | - Slack notifications when new ports and vulnerabilities found 17 | 18 | ## Infrastructure 19 | 20 | Crossfeed Web (this repository) sits as the user-facing end of Crossfeed. This displays all information and allows scheduling scans. 21 | 22 | [Crossfeed Agent](https://github.com/Code-dot-mil/crossfeed-agent) is the backend scanner, which launches and coordinates scans. 23 | 24 | Scans are queued via Amazon SQS and dispatched by crossfeed agent. This is designed for a multi-host environment, where backend scanners process incoming scan requests asynchronously. 25 | 26 | ## Development 27 | 28 | To get started, first copy relevent config files: 29 | 30 | 1. Run `cp .env.example .env` 31 | 2. Run `cp config/config.example.json config/config.json` 32 | 3. In the [agent](https://github.com/Code-dot-mil/crossfeed-agent), run `cp config.example.json config.json` 33 | 34 | ### Install and configure Docker 35 | 36 | - Install [Docker](https://docs.docker.com/install/). 37 | 38 | Configure the Postgres database information in `.env` on web and `config.json` for the agent. Likewise, configure the SQS information in `.env` for web and `config.json` for the agent. 39 | 40 | ### Obtain API Keys 41 | 42 | Crossfeed integrates with several APIs. Configure the following API keys to make full use of the tool: 43 | 44 | - `SONAR_API_KEY` (agent) - The Rapid7 [Project Sonar](https://www.rapid7.com/research/project-sonar/) API key, used to download port scan data 45 | - `SLACK_WEBHOOK_URL` (agent) - A Slack [incoming webhook](https://api.slack.com/incoming-webhooks) url, used to post alerts to Slack 46 | - `BD_API_KEY` (web, optional) - A [BitDiscovery](https://bitdiscovery.com) API key, optionally used for importing data 47 | 48 | ### Quick start 49 | 50 | 1. Run `docker-compose up` 51 | -------------------------------------------------------------------------------- /client/views/scans.html: -------------------------------------------------------------------------------- 1 |
2 |

Configure Scans

3 | 4 |
5 | 6 |

Automatic Scans

7 |

The following scans are configured to run automatically

8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 |
Scan FrequencyCommand
{{job.freq}}{{job.command}}
19 | 20 |

Schedule Scan

21 |
22 |
23 | 30 | 36 |

37 | 38 |
39 |
40 | 41 |
42 | 43 |

Run Manual Scan

44 | 45 |

Runs a passive port scan using Rapid7's Project Sonar data.

46 |
47 |
48 | 49 | 50 |
51 |
52 | 53 |
54 | 55 |

Fetches given path(s) for every live host

56 |
57 |
58 | 59 | 60 |
61 |
62 | 63 |
64 | 65 |

Send custom HTTP request to select hosts

66 | 67 | 68 |
69 | -------------------------------------------------------------------------------- /client/js/controllers/Controller.js: -------------------------------------------------------------------------------- 1 | angular 2 | .module("Controller", []) 3 | .controller("Controller", [ 4 | "$scope", 5 | "$state", 6 | "$stateParams", 7 | "$window", 8 | "Domain", 9 | "NgTableParams", 10 | function($scope, $state, $stateParams, $window, Domain, NgTableParams) { 11 | this.domains = []; 12 | this.domain = ""; 13 | 14 | this.searchString = ""; 15 | 16 | this.type = "domains"; // Default to domains search 17 | 18 | this.all = { 19 | ports: [], 20 | services: [] 21 | }; 22 | 23 | this.fetchAll = () => { 24 | var tableParams = { 25 | page: 1, // show first page 26 | count: 25 // count per page 27 | }; 28 | var savedState = localStorage.getItem("tableParams-" + this.type); 29 | if (savedState) { 30 | tableParams = JSON.parse(savedState); 31 | } 32 | 33 | this.tableParams = new NgTableParams(tableParams, { 34 | filterDelay: 1000, 35 | getData: this.search 36 | }); 37 | 38 | this.loadAll("services"); 39 | }; 40 | 41 | this.search = params => { 42 | var queryParams = { 43 | page: params.url().page, 44 | count: params.url().count, 45 | sorting: params.sorting(), 46 | filter: params.filter() 47 | }; 48 | localStorage.setItem("tableParams-" + this.type, JSON.stringify({ count: params.url().count })); 49 | return Domain.fetch(this.type, queryParams) 50 | .then(response => { 51 | params.total(response.data.count); 52 | var domains = response.data.rows; 53 | $scope.count = response.data.count; 54 | return domains; 55 | }) 56 | .catch(error => { 57 | console.log(error); 58 | }); 59 | }; 60 | 61 | this.add = function() { 62 | Domain.create(this.url) 63 | .then(response => { 64 | for (url of response.data) { 65 | this.urls.unshift(url); 66 | } 67 | }) 68 | .catch(error => { 69 | $scope.formError = error.data.message; 70 | console.log(error); 71 | }); 72 | }; 73 | 74 | this.clear = function() { 75 | this.tableParams.filter({}); 76 | this.tableParams.sorting({}); 77 | this.tableParams.url({}); 78 | }; 79 | 80 | this.delete = function(id) { 81 | Url.delete(id) 82 | .then(response => { 83 | this.urls = this.urls.filter(url => url.id !== id); 84 | }) 85 | .catch(error => { 86 | console.log(error); 87 | }); 88 | }; 89 | 90 | this.loadAll = type => { 91 | return Domain.loadAll(type) 92 | .then(vals => { 93 | this.all[type] = vals; 94 | return vals; 95 | }) 96 | .catch(error => { 97 | console.log(error); 98 | }); 99 | }; 100 | } 101 | ]) 102 | 103 | .filter("formatDate", () => { 104 | return function(value) { 105 | return value.split(" ")[0]; 106 | }; 107 | }); 108 | -------------------------------------------------------------------------------- /server/h1_api.js: -------------------------------------------------------------------------------- 1 | var models = require('../models'); 2 | var Sequelize = require("sequelize"); 3 | var express = require('express'); 4 | var request = require('request'); 5 | var router = express.Router(); 6 | var multer = require('multer'); 7 | var csv = require('fast-csv'); 8 | var fs = require('fs'); 9 | var async = require('async'); 10 | 11 | var upload = multer({dest: 'tmp/csv/'}); 12 | 13 | // Import vulnerability CSV 14 | router.post('/importCsv', upload.single('csv'), function(req, res) { 15 | var fileRows = [], fileHeader; 16 | 17 | // open uploaded file 18 | csv.fromPath(req.file.path) 19 | .on("data", function (data) { 20 | fileRows.push(data); // push each row 21 | }) 22 | .on("end", async () => { 23 | fs.unlinkSync(req.file.path); // remove temp file 24 | 25 | fileRows.shift(); //skip header row 26 | var items = fileRows.map((row) => { 27 | return { 28 | hackerone_id: row[1], 29 | title: row[2], 30 | severity: row[3], 31 | state: row[5], 32 | substate: row[6], 33 | weakness: row[7], 34 | reported_at: row[8], 35 | closed_at: row[11] 36 | } 37 | }) 38 | 39 | console.log(items) 40 | 41 | var blacklistedSubstates = ['informative', 'duplicate', 'not-applicable', 'spam'] 42 | items = items.filter(item => !blacklistedSubstates.includes(item.substate)) 43 | 44 | 45 | var numCreated = 0; 46 | var numUpdated = 0; 47 | for (item of items) { 48 | await models.Vulnerability.findOne({ where: {hackerone_id: item.hackerone_id} }) 49 | .then(function(obj) { 50 | if(obj) { 51 | numUpdated++; 52 | return obj.update(item); 53 | } 54 | else { // insert 55 | numCreated++; 56 | return models.Vulnerability.create(item); 57 | } 58 | }) 59 | } 60 | 61 | console.log(`H1 import success. ${numCreated} created and ${numUpdated} updated.`) 62 | res.redirect('/vulns'); 63 | }); 64 | }); 65 | 66 | router.post('/importContents', function(req, res) { 67 | models.Vulnerability.findAll({ where: { contents: null}}) 68 | .then(function(vulns) { 69 | var q = async.queue(function (vuln, callback) { 70 | console.log('Starting report ' + vuln.hackerone_id); 71 | request({ 72 | url: 'https://hackerone.com/reports/' + vuln.hackerone_id + '.json', 73 | json: true, 74 | headers: { 75 | 'Cookie': '__Host-session=' + req.body.cookie 76 | } 77 | }, (err, res2, body) => { 78 | if (err || !body || !body.vulnerability_information) { 79 | console.log('Could not retrieve report ' + vuln.hackerone_id); 80 | callback(); 81 | return; 82 | } 83 | console.log(body) 84 | vuln.update({ 85 | contents: body.vulnerability_information 86 | }).then(() => { 87 | console.log('updated') 88 | callback(); 89 | }) 90 | }) 91 | }, 2); 92 | 93 | q.drain = function() { 94 | res.status(200).json('success.'); 95 | } 96 | 97 | q.push(vulns); 98 | }) 99 | }); 100 | 101 | module.exports = router; -------------------------------------------------------------------------------- /client/views/launchScan.html: -------------------------------------------------------------------------------- 1 |
2 |

Launch Scan

3 |

Source vulnerability #{{ctrl.vuln.hackerone_id}}

4 | 5 |
6 |
7 | 8 |
10 | 11 | 12 |
13 |
18 | 19 | 22 |
23 | 24 |
25 |
26 | 27 | 28 |
29 |

No port filters (not recommended)

30 |
31 | 34 | 37 |
38 | 39 |
40 |
41 | 42 |
43 |

Scanning regardless of service

44 |
45 | 46 | 49 |
50 | 51 |
52 |
53 | 54 |

55 |
56 |

The following HTTP request will be sent to {{ctrl.countPreview | number}} domains.

57 | 
{{ctrl.newScan.request}}
58 |
59 | 60 | 61 |
62 |
63 | 64 |

Report Contents:

65 | 
{{ctrl.vuln.contents}}
66 |
67 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var express = require("express"); 2 | var https = require("https"); 3 | var fs = require("fs"); 4 | var app = express(); 5 | var bodyParser = require("body-parser"); 6 | var methodOverride = require("method-override"); 7 | var basicAuth = require("express-basic-auth"); 8 | const session = require("express-session"); 9 | var passport = require("passport"); 10 | var GoogleStrategy = require("passport-google-oauth").OAuth2Strategy; 11 | var morgan = require("morgan"); 12 | 13 | require("dotenv").config(); 14 | 15 | app.use(bodyParser.json()); // parse application/json 16 | app.use(bodyParser.json({ type: "application/vnd.api+json" })); // parse application/vnd.api+json as json 17 | app.use(bodyParser.urlencoded({ extended: true })); // parse application/x-www-form-urlencoded 18 | 19 | if (process.env.ENVIRONMENT == "production") { 20 | app.use(morgan("combined")); 21 | } else { 22 | app.use(morgan("tiny")); 23 | } 24 | 25 | app.use( 26 | session({ 27 | secret: process.env.SESSION_SECRET, 28 | resave: false, 29 | saveUninitialized: false, 30 | cookie: { secure: process.env.ENVIRONMENT == "production" } 31 | }) 32 | ); 33 | 34 | app.use(passport.initialize()); 35 | app.use(passport.session()); 36 | passport.serializeUser(function(user, done) { 37 | done(null, user.profile.id); 38 | }); 39 | 40 | passport.deserializeUser(function(id, done) { 41 | done(null, { profile: { id: id } }); 42 | }); 43 | 44 | if (process.env.GOOGLE_CLIENT_ID) { 45 | passport.use( 46 | new GoogleStrategy( 47 | { 48 | clientID: process.env.GOOGLE_CLIENT_ID, 49 | clientSecret: process.env.GOOGLE_CLIENT_SECRET, 50 | callbackURL: process.env.GOOGLE_REDIRECT_URL 51 | }, 52 | function(token, refreshToken, profile, done) { 53 | return done(null, { 54 | profile: profile, 55 | token: token 56 | }); 57 | } 58 | ) 59 | ); 60 | 61 | app.get("/auth/google", passport.authenticate("google", { scope: ["https://www.googleapis.com/auth/plus.login"] })); 62 | 63 | // Support Google OAuth logon 64 | app.get("/auth/google/callback", passport.authenticate("google", { failureRedirect: "/" }), function(req, res) { 65 | console.log("New login: " + req.user.profile.displayName + " " + req.user.profile.id); 66 | 67 | req.session.authorized = true; 68 | res.redirect("/"); 69 | }); 70 | } 71 | 72 | // Support password-based logon if APP_PASSWORD is set 73 | app.get("/login", (req, res, next) => { 74 | const b64auth = (req.headers.authorization || "").split(" ")[1] || ""; 75 | const [login, password] = new Buffer(b64auth, "base64").toString().split(":"); 76 | if (process.env.APP_PASSWORD && password === process.env.APP_PASSWORD) { 77 | req.session.authorized = true; 78 | res.redirect("/"); 79 | return; 80 | } 81 | res.set("WWW-Authenticate", 'Basic realm="example"'); 82 | res.status(401).send(); 83 | }); 84 | 85 | app.all("*", passport.session(), (req, res, next) => { 86 | if (!req.session.authorized) { 87 | res.redirect("/auth/google"); 88 | return; 89 | } 90 | next(); 91 | }); 92 | 93 | app.use(express.static(__dirname + "/client")); 94 | 95 | require("./server/routes")(app); // pass application into routes 96 | 97 | if (process.env.ENVIRONMENT == "production") { 98 | const privateKey = fs.readFileSync("privkey.pem", "utf8"); 99 | const certificate = fs.readFileSync("cert.pem", "utf8"); 100 | const ca = fs.readFileSync("chain.pem", "utf8"); 101 | 102 | const credentials = { 103 | key: privateKey, 104 | cert: certificate, 105 | ca: ca 106 | }; 107 | 108 | https.createServer(credentials, app).listen(443, () => { 109 | console.log("Express server listening on port %d.", 443); 110 | }); 111 | } else { 112 | app.listen(3000, () => { 113 | console.log("Express server listening on port %d.", 3000); 114 | }); 115 | } 116 | 117 | module.exports = app; 118 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Our Projects, Version 1.5 2 | 3 | **NOTE: This CONTRIBUTING.md is for software contributions. You do not need to follow the Developer's Certificate of Origin (DCO) process for commenting on the Crossfeed repository documentation, such as CONTRIBUTING.md, INTENT.md, etc. or for submitting issues.** 4 | 5 | Thanks for thinking about using or contributing to this software ("Project") and its documentation! 6 | 7 | - [Policy & Legal Info](#policy) 8 | - [Getting Started](#getting-started) 9 | - [Submitting an Issue](#submitting-an-issue) 10 | - [Submitting Code](#submitting-code) 11 | 12 | ## Policy 13 | 14 | ### 1. Introduction 15 | 16 | The project maintainer for this Project will only accept contributions using the Developer's Certificate of Origin 1.1 located at [developercertificate.org](https://developercertificate.org) ("DCO"). The DCO is a legally binding statement asserting that you are the creator of your contribution, or that you otherwise have the authority to distribute the contribution, and that you are intentionally making the contribution available under the license associated with the Project ("License"). 17 | 18 | ### 2. Developer Certificate of Origin Process 19 | 20 | Before submitting contributing code to this repository for the first time, you'll need to sign a Developer Certificate of Origin (DCO) (see below). To agree to the DCO, add your name and email address to the [CONTRIBUTORS.md](https://github.com/deptofdefense/crossfeed-web/blob/master/CONTRIBUTORS.md) file. At a high level, adding your information to this file tells us that you have the right to submit the work you're contributing and indicates that you consent to our treating the contribution in a way consistent with the license associated with this software (as described in [LICENSE.md](https://github.com/deptofdefense/crossfeed-web/blob/master/LICENSE.md)) and its documentation ("Project"). 21 | 22 | ### 3. Important Points 23 | 24 | Pseudonymous or anonymous contributions are permissible, but you must be reachable at the email address provided in the Signed-off-by line. 25 | 26 | If your contribution is significant, you are also welcome to add your name and copyright date to the source file header. 27 | 28 | U.S. Federal law prevents the government from accepting gratuitous services unless certain conditions are met. By submitting a pull request, you acknowledge that your services are offered without expectation of payment and that you expressly waive any future pay claims against the U.S. Federal government related to your contribution. 29 | 30 | If you are a U.S. Federal government employee and use a `*.mil` or `*.gov` email address, we interpret your Signed-off-by to mean that the contribution was created in whole or in part by you and that your contribution is not subject to copyright protections. 31 | 32 | ### 4. DCO Text 33 | 34 | The full text of the DCO is included below and is available online at [developercertificate.org](https://developercertificate.org): 35 | 36 | ```txt 37 | Developer Certificate of Origin 38 | Version 1.1 39 | 40 | Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 41 | 1 Letterman Drive 42 | Suite D4700 43 | San Francisco, CA, 94129 44 | 45 | Everyone is permitted to copy and distribute verbatim copies of this 46 | license document, but changing it is not allowed. 47 | 48 | Developer's Certificate of Origin 1.1 49 | 50 | By making a contribution to this project, I certify that: 51 | 52 | (a) The contribution was created in whole or in part by me and I 53 | have the right to submit it under the open source license 54 | indicated in the file; or 55 | 56 | (b) The contribution is based upon previous work that, to the best 57 | of my knowledge, is covered under an appropriate open source 58 | license and I have the right under that license to submit that 59 | work with modifications, whether created in whole or in part 60 | by me, under the same open source license (unless I am 61 | permitted to submit under a different license), as indicated 62 | in the file; or 63 | 64 | (c) The contribution was provided directly to me by some other 65 | person who certified (a), (b) or (c) and I have not modified 66 | it. 67 | 68 | (d) I understand and agree that this project and the contribution 69 | are public and that a record of the contribution (including all 70 | personal information I submit with it, including my sign-off) is 71 | maintained indefinitely and may be redistributed consistent with 72 | this project or the open source license(s) involved. 73 | ``` 74 | 75 | ## Submitting an Issue 76 | 77 | You should feel free to [submit an issue](https://github.com/deptofdefense/crossfeed-web/issues) on our GitHub repository for anything you find that needs attention on the website. That includes content, functionality, design, or anything else! 78 | 79 | ### Submitting a Bug Report 80 | 81 | When submitting a bug report on the website, please be sure to include accurate and thorough information about the problem you're observing. Be sure to include: 82 | 83 | - Steps to reproduce the problem, 84 | - The URL of the page where you observed the problem, 85 | - What you expected to happen, 86 | - What actually happend (or didn't happen), and 87 | - Technical details including your Operating System name and version and Web browser name and version number. 88 | -------------------------------------------------------------------------------- /client/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "@types/angular": { 6 | "version": "1.6.56", 7 | "resolved": "https://registry.npmjs.org/@types/angular/-/angular-1.6.56.tgz", 8 | "integrity": "sha512-HxtqilvklZ7i6XOaiP7uIJIrFXEVEhfbSY45nfv2DeBRngncI58Y4ZOUMiUkcT8sqgLL1ablmbfylChUg7A3GA==" 9 | }, 10 | "@uirouter/core": { 11 | "version": "5.0.23", 12 | "resolved": "https://registry.npmjs.org/@uirouter/core/-/core-5.0.23.tgz", 13 | "integrity": "sha512-rwFOH++z/KY8y+h0IOpQ5uC8Nim6E0EBCQrIjhVCr+XKYXgpK+VdtuOLFdogvbJ3AAi5Z7ei00qdEr7Did5CAg==" 14 | }, 15 | "angular": { 16 | "version": "1.7.9", 17 | "resolved": "https://registry.npmjs.org/angular/-/angular-1.7.9.tgz", 18 | "integrity": "sha512-5se7ZpcOtu0MBFlzGv5dsM1quQDoDeUTwZrWjGtTNA7O88cD8TEk5IEKCTDa3uECV9XnvKREVUr7du1ACiWGFQ==" 19 | }, 20 | "angular-animate": { 21 | "version": "1.7.8", 22 | "resolved": "https://registry.npmjs.org/angular-animate/-/angular-animate-1.7.8.tgz", 23 | "integrity": "sha512-bINtzizq7TbJzfVrDpwLfTxVl0Qd7fRNWFb5jKYI1vaFZobQNX/QONXlLow6ySsDbZ6eLECycB7mvWtfh1YYaw==" 24 | }, 25 | "angular-chart.js": { 26 | "version": "1.1.1", 27 | "resolved": "https://registry.npmjs.org/angular-chart.js/-/angular-chart.js-1.1.1.tgz", 28 | "integrity": "sha1-SfDhjQgXYrbUyXkeSHr/L7sw9a4=", 29 | "requires": { 30 | "angular": "1.x", 31 | "chart.js": "2.3.x" 32 | } 33 | }, 34 | "angular-route": { 35 | "version": "1.7.8", 36 | "resolved": "https://registry.npmjs.org/angular-route/-/angular-route-1.7.8.tgz", 37 | "integrity": "sha512-VVk89PH0fsY5kfbx+N7IVX1IwnaPWYhMGY0uA+rjej2v1sjvrTx1SLkxUK4E0UpW1hXeLJhN7ncBcwoBiPtAtA==" 38 | }, 39 | "angular-ui-bootstrap": { 40 | "version": "2.5.6", 41 | "resolved": "https://registry.npmjs.org/angular-ui-bootstrap/-/angular-ui-bootstrap-2.5.6.tgz", 42 | "integrity": "sha512-yzcHpPMLQl0232nDzm5P4iAFTFQ9dMw0QgFLuKYbDj9M0xJ62z0oudYD/Lvh1pWfRsukiytP4Xj6BHOSrSXP8A==" 43 | }, 44 | "angular-ui-router": { 45 | "version": "1.0.22", 46 | "resolved": "https://registry.npmjs.org/angular-ui-router/-/angular-ui-router-1.0.22.tgz", 47 | "integrity": "sha512-cuq0+Di6spKEIQ1aZCGORPU9uZzJRFzuiMQDB2vLUDlo1DS8tR8/KxlThqmz6iw+8KtmUArI+5IMINSp/jwUKg==", 48 | "requires": { 49 | "@uirouter/core": "5.0.23" 50 | } 51 | }, 52 | "angularjs-toaster": { 53 | "version": "3.0.0", 54 | "resolved": "https://registry.npmjs.org/angularjs-toaster/-/angularjs-toaster-3.0.0.tgz", 55 | "integrity": "sha512-Upv5lkyrWWv9aIK4Jo7X9nCp1tPaWerSEeUrdjibm0GhwuW0/aoaUjn8Ou7IS8/QIGZspXrXHWGTa/CRBOj6Jg==" 56 | }, 57 | "bootstrap": { 58 | "version": "3.4.1", 59 | "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", 60 | "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==" 61 | }, 62 | "chart.js": { 63 | "version": "2.3.0", 64 | "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.3.0.tgz", 65 | "integrity": "sha1-QEYOSOLEF8BfwzJc2E97AA3H19Y=", 66 | "requires": { 67 | "chartjs-color": "^2.0.0", 68 | "moment": "^2.10.6" 69 | } 70 | }, 71 | "chartjs-color": { 72 | "version": "2.3.0", 73 | "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.3.0.tgz", 74 | "integrity": "sha512-hEvVheqczsoHD+fZ+tfPUE+1+RbV6b+eksp2LwAhwRTVXEjCSEavvk+Hg3H6SZfGlPh/UfmWKGIvZbtobOEm3g==", 75 | "requires": { 76 | "chartjs-color-string": "^0.6.0", 77 | "color-convert": "^0.5.3" 78 | } 79 | }, 80 | "chartjs-color-string": { 81 | "version": "0.6.0", 82 | "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz", 83 | "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==", 84 | "requires": { 85 | "color-name": "^1.0.0" 86 | } 87 | }, 88 | "color-convert": { 89 | "version": "0.5.3", 90 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-0.5.3.tgz", 91 | "integrity": "sha1-vbbGnOZg+t/+CwAHzER+G59ygr0=" 92 | }, 93 | "color-name": { 94 | "version": "1.1.4", 95 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", 96 | "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" 97 | }, 98 | "jquery": { 99 | "version": "3.5.0", 100 | "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.5.0.tgz", 101 | "integrity": "sha512-Xb7SVYMvygPxbFMpTFQiHh1J7HClEaThguL15N/Gg37Lri/qKyhRGZYzHRyLH8Stq3Aow0LsHO2O2ci86fCrNQ==" 102 | }, 103 | "moment": { 104 | "version": "2.24.0", 105 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", 106 | "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" 107 | }, 108 | "ng-table": { 109 | "version": "3.0.1", 110 | "resolved": "https://registry.npmjs.org/ng-table/-/ng-table-3.0.1.tgz", 111 | "integrity": "sha1-4O+liIfPQYBct5c4FNpQnd24k2g=", 112 | "requires": { 113 | "@types/angular": "^1.5.13" 114 | } 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /client/js/controllers/ScansController.js: -------------------------------------------------------------------------------- 1 | angular.module("ScansController", []).controller("ScansController", [ 2 | "$scope", 3 | "$state", 4 | "$stateParams", 5 | "$window", 6 | "Scans", 7 | "NgTableParams", 8 | "toaster", 9 | "Domain", 10 | function($scope, $state, $stateParams, $window, Scans, NgTableParams, toaster, Domain) { 11 | this.frequnit = "minutes"; 12 | this.commandArgs = ""; 13 | this.freq = ""; 14 | this.commandType = ""; 15 | this.newScan = { 16 | responseMatches: [{}], 17 | ports: [{}], 18 | services: [{}], 19 | request: `GET / HTTP/1.1 20 | Host: {host} 21 | User-Agent: crossfeed-scanner 22 | Connection: close` 23 | }; 24 | 25 | this.alerts = []; 26 | 27 | $scope.tasks = {}; 28 | 29 | this.allValues = { 30 | ports: [], 31 | services: [] 32 | }; 33 | 34 | this.fetchLogs = function() { 35 | Scans.fetchLogs() 36 | .then(response => { 37 | logs = response.data.logs.split("\n").reverse(); 38 | if (logs[0] == "") { 39 | logs.shift(); 40 | } 41 | this.logs = logs.join("\n"); 42 | }) 43 | .catch(error => { 44 | toaster.pop("error", "Error", error.data.error); 45 | }); 46 | }; 47 | 48 | this.fetchAlerts = function() { 49 | Scans.fetchAlerts() 50 | .then(response => { 51 | this.alerts = response.data; 52 | for (var alert of this.alerts) { 53 | alert.isOpen = true; 54 | } 55 | console.log(this.alerts); 56 | }) 57 | .catch(error => { 58 | toaster.pop("error", "Error", error.data.error); 59 | }); 60 | }; 61 | 62 | this.initPortScan = function() { 63 | Scans.enqueueJob("scan-ports", [this.port]) 64 | .then(response => { 65 | toaster.pop("success", "Success", response.data.status); 66 | }) 67 | .catch(response => { 68 | toaster.pop("error", "Error", response.data.error); 69 | }); 70 | }; 71 | 72 | this.initHostScan = function() { 73 | Scans.enqueueJob("scan-hosts", [this.path]) 74 | .then(response => { 75 | toaster.pop("success", "Success", response.data.status); 76 | }) 77 | .catch(response => { 78 | toaster.pop("error", "Error", response.data.error); 79 | }); 80 | }; 81 | 82 | this.getConfig = function() { 83 | Scans.getConfig() 84 | .then(response => { 85 | this.jobs = response.data.jobs; 86 | }) 87 | .catch(error => { 88 | toaster.pop("error", "Error", error.data.error); 89 | }); 90 | }; 91 | 92 | this.scheduleCron = function() { 93 | Scans.scheduleCron({ 94 | freq: this.freq, 95 | frequnit: this.frequnit, 96 | commandType: this.commandType, 97 | commandArgs: this.commandArgs 98 | }) 99 | .then(response => { 100 | this.jobs = response.data.jobs; 101 | toaster.pop("success", "Success", "Successfully added job"); 102 | }) 103 | .catch(error => { 104 | console.log(error); 105 | toaster.pop("error", "Error", error.data.error); 106 | }); 107 | }; 108 | 109 | this.removeCron = function(index) { 110 | Scans.removeCron(index) 111 | .then(response => { 112 | this.jobs = response.data.jobs; 113 | }) 114 | .catch(error => { 115 | toaster.pop("error", "Error", error.data.error); 116 | }); 117 | }; 118 | 119 | this.pollRunningTasks = function() { 120 | this.getTasks("running"); 121 | this.getTasks("all"); 122 | setInterval(this.getRunningTasks, 5000); 123 | }; 124 | 125 | this.getTasks = function(type) { 126 | Scans.getTasksWithStatus(type) 127 | .then(response => { 128 | for (task of response.data) { 129 | if (task.status == "running") { 130 | task.type = "warning"; 131 | } else if (task.status == "failed") { 132 | task.type = "danger"; 133 | } else if (task.status == "finished") { 134 | task.type = "success"; 135 | } 136 | task.status = task.status[0].toUpperCase() + task.status.substring(1); 137 | } 138 | $scope.tasks[type] = response.data; 139 | console.log($scope.tasks[type]); 140 | }) 141 | .catch(error => { 142 | toaster.pop("error", "Error", error.data.error); 143 | }); 144 | }; 145 | 146 | this.getAllTasks = function() { 147 | Scans.getTasksWithStatus("all") 148 | .then(response => { 149 | $scope.allTasks = response.data; 150 | }) 151 | .catch(error => { 152 | toaster.pop("error", "Error", error.data.error); 153 | }); 154 | }; 155 | 156 | this.fetchVulnerability = () => { 157 | Domain.fetchOne("vulns", $stateParams.vulnId) 158 | .then(response => { 159 | this.vuln = response.data; 160 | }) 161 | .catch(error => { 162 | console.log(error); 163 | }); 164 | }; 165 | 166 | this.loadAll = type => { 167 | Domain.loadAll(type) 168 | .then(vals => { 169 | this.allValues[type] = vals; 170 | }) 171 | .catch(error => { 172 | console.log(error); 173 | }); 174 | }; 175 | 176 | this.beginScan = () => { 177 | Scans.launchScan({ 178 | greps: this.filterVals(this.newScan.responseMatches), 179 | request: this.newScan.request, 180 | filters: { 181 | ports: this.filterVals(this.newScan.ports), 182 | services: this.filterVals(this.newScan.services) 183 | } 184 | }) 185 | .then(response => { 186 | this.jobs = response.data.jobs; 187 | toaster.pop("success", "Success", "Successfully added scan to queue!"); 188 | }) 189 | .catch(error => { 190 | console.log(error); 191 | toaster.pop("error", "Error", error.data.error); 192 | }); 193 | }; 194 | 195 | this.previewCount = () => { 196 | Scans.previewCount({ 197 | filters: { 198 | ports: this.filterVals(this.newScan.ports), 199 | services: this.filterVals(this.newScan.services) 200 | } 201 | }) 202 | .then(response => { 203 | this.countPreview = response.data.count; 204 | }) 205 | .catch(error => { 206 | console.log(error); 207 | }); 208 | }; 209 | 210 | this.filterVals = vals => { 211 | return vals.map(val => val.value).filter(val => val); 212 | }; 213 | } 214 | ]); 215 | -------------------------------------------------------------------------------- /server/routes.js: -------------------------------------------------------------------------------- 1 | var models = require("../models"); 2 | var request = require("request"); 3 | var Sequelize = require("sequelize"); 4 | 5 | var bd_api = require("./bd_api.js"); 6 | var h1_api = require("./h1_api.js"); 7 | var scans = require("./scans.js"); 8 | var utils = require("./utils.js"); 9 | 10 | // Helper function to format query parameters, i.e. limit, page, order 11 | function formatQueryParams(body) { 12 | var limit = body.count || 25; 13 | var page = body.page || 1; 14 | var order = []; 15 | 16 | if (body.sorting) { 17 | var sortingKeys = Object.keys(body.sorting); 18 | if (sortingKeys.length > 0) { 19 | order.push([sortingKeys[0], body.sorting[sortingKeys[0]]]); 20 | } else { 21 | order.push(["id", "ASC"]); 22 | } 23 | } 24 | 25 | var where = {}; 26 | for (filter in body.filter) { 27 | if (filter == "ports" && body.filter[filter] == "80,443") { 28 | where["$or"] = [ 29 | { 30 | ports: { $like: "%80%" } 31 | }, 32 | { 33 | ports: { $like: "%443%" } 34 | } 35 | ]; 36 | continue; 37 | } 38 | if (filter == "ports" && body.filter[filter] == "not_null") { 39 | where["$and"] = [ 40 | { 41 | ports: { [Sequelize.Op.ne]: null } 42 | }, 43 | { 44 | ports: { [Sequelize.Op.ne]: "" } 45 | } 46 | ]; 47 | continue; 48 | } 49 | where[filter] = { $like: "%" + body.filter[filter] + "%" }; 50 | } 51 | 52 | return { 53 | limit: limit, 54 | page: page, 55 | offset: limit * (page - 1), 56 | order: order, 57 | where: where 58 | }; 59 | } 60 | 61 | module.exports = function(app) { 62 | app.use("/api/bd", bd_api); 63 | app.use("/api/h1", h1_api); 64 | 65 | app.use("/api/scans", scans); 66 | 67 | app.get("/api/vulns/categorize", function(req, res) { 68 | models.Vulnerability.findAll().then(function(vulns) { 69 | for (vuln of vulns) { 70 | var domains = utils.findDomains(vuln.contents); 71 | vuln.updateAttributes({ domains: domains.join(",") }); 72 | } 73 | res.status(200).json({}); 74 | }); 75 | }); 76 | 77 | app.get("/api/vulns/associate", function(req, res) { 78 | models.Vulnerability.findAll().then(function(vulns) { 79 | /*for (vuln of vulns) { 80 | console.log(vuln.domains) 81 | var domainsArr = vuln.domains.split(',') 82 | for (domain of domainsArr) { 83 | console.log(domain) 84 | models.Domain.findOne({ where: { name: domain } }).then((result) => { 85 | if (result) { 86 | console.log(result.id) 87 | 88 | } 89 | }) 90 | } 91 | } 92 | res.status(200).json({});*/ 93 | }); 94 | }); 95 | 96 | // Search domains 97 | app.post("/api/domains/search", function(req, res) { 98 | var params = formatQueryParams(req.body); 99 | 100 | models.Domain.findAndCountAll({ 101 | order: params.order, 102 | limit: params.limit, 103 | offset: params.offset, 104 | where: params.where, 105 | attributes: ["name", "ip", "ports", "id", "services"] 106 | }).then(function(domains) { 107 | res.status(200).json(domains); 108 | }); 109 | }); 110 | 111 | // Search vulnerabilities 112 | app.post("/api/vulns/search", function(req, res) { 113 | if (Object.entries(req.body).length === 0) { 114 | return models.Vulnerability.findAll({ 115 | attributes: { exclude: ["contents"] } 116 | }).then(function(vulns) { 117 | res.status(200).json(vulns); 118 | }); 119 | } 120 | var params = formatQueryParams(req.body); 121 | 122 | models.Vulnerability.findAndCountAll({ 123 | order: params.order, 124 | limit: params.limit, 125 | offset: params.offset, 126 | where: params.where, 127 | attributes: { exclude: ["contents"] } 128 | }).then(function(vulns) { 129 | res.status(200).json(vulns); 130 | }); 131 | }); 132 | 133 | // Fetch a single domain 134 | app.get("/api/domains/:id", function(req, res) { 135 | models.Domain.findOne({ 136 | where: { id: req.params.id } 137 | }).then(function(domain) { 138 | res.status(200).json(domain); 139 | }); 140 | }); 141 | 142 | // Fetch a single vuln 143 | app.get("/api/vulns/:id", function(req, res) { 144 | models.Vulnerability.findOne({ 145 | where: { id: req.params.id } 146 | }).then(function(vuln) { 147 | res.status(200).json(vuln); 148 | }); 149 | }); 150 | 151 | // Get the status of all tasks 152 | app.get("/api/tasks/:type", function(req, res) { 153 | let filter = {}; 154 | let order = [["id", "desc"]]; 155 | if (req.params.type === "running") { 156 | filter.status = "running"; 157 | } 158 | models.TaskStatus.findAll({ 159 | where: filter, 160 | order: order 161 | }).then(function(tasks) { 162 | res.status(200).json(tasks); 163 | }); 164 | }); 165 | 166 | // Get all alerts 167 | app.get("/api/alerts", function(req, res) { 168 | let filter = {}; 169 | let order = [["id", "desc"]]; 170 | models.Alert.findAll({ 171 | where: filter, 172 | order: order 173 | }).then(function(alerts) { 174 | res.status(200).json(alerts); 175 | }); 176 | }); 177 | 178 | // [deprecated] Add a single domain 179 | app.post("/api/domains", function(req, res) { 180 | var domain = req.body.domain; 181 | console.log(domain); 182 | models.Domain.create({ 183 | name: domain 184 | }).then(domain => { 185 | res.status(200).json(); 186 | }); 187 | }); 188 | 189 | app.delete("/api/domains/:id", function(req, res) { 190 | models.Domain.destroy({ 191 | where: { 192 | id: req.params.id 193 | } 194 | }).then(() => { 195 | res.status(200).json({}); 196 | }); 197 | }); 198 | 199 | app.get("/api/values", function(req, res) { 200 | if (!["services", "ports"].includes(req.query.type)) { 201 | return res.status(422).json({ error: "Invalid type" }); 202 | } 203 | models.Domain.findAll({ 204 | attributes: [[Sequelize.fn("DISTINCT", Sequelize.col(req.query.type)), req.query.type]] 205 | }).then(function(types) { 206 | var allTypes = types.map(type => { 207 | var val = type.dataValues[req.query.type]; 208 | if (val == null) return []; 209 | return val 210 | .split(",") 211 | .map(t => t.trim()) 212 | .filter(t => t != ""); 213 | }); 214 | var flattened = [].concat.apply([], allTypes); 215 | res.status(200).json(Array.from(new Set(flattened))); 216 | }); 217 | }); 218 | 219 | // frontend routes 220 | app.get("*", function(req, res) { 221 | res.sendfile("./client/index.html"); 222 | }); 223 | }; 224 | -------------------------------------------------------------------------------- /server/scans.js: -------------------------------------------------------------------------------- 1 | var models = require("../models"); 2 | var Sequelize = require("sequelize"); 3 | var express = require("express"); 4 | var router = express.Router(); 5 | var fs = require("fs"); 6 | var readLastLines = require("read-last-lines"); 7 | var moment = require("moment"); 8 | var AWS = require("aws-sdk"); 9 | var crontab = require("crontab"); 10 | var shellescape = require("shell-escape"); 11 | var parser = require("http-string-parser"); 12 | 13 | AWS.config.update({ region: process.env.AWS_REGION }); 14 | var sqs = new AWS.SQS(); 15 | 16 | const validCommands = ["scan-ports", "scan-hosts", "subjack"]; 17 | 18 | function renderJobs(jobs) { 19 | var arr = []; 20 | for (i in jobs) { 21 | arr.push(renderJob(jobs[i])); 22 | } 23 | return arr; 24 | } 25 | 26 | function renderJob(job) { 27 | var str = job.render(); 28 | var json = { 29 | freq: str.split(" cd")[0], 30 | command: str.split("./crossfeed-agent enqueue ")[1], 31 | index: i 32 | }; 33 | return json; 34 | } 35 | 36 | function loadCrontab(callback) { 37 | if (process.env.CRONTAB_USER) { 38 | crontab.load(process.env.CRONTAB_USER, callback); 39 | } else { 40 | crontab.load(callback); 41 | } 42 | } 43 | 44 | function formatDomainQuery(body) { 45 | var andConditions = []; 46 | for (var filter in body.filters) { 47 | if (body.filters[filter].length == 0) continue; 48 | andConditions.push({ 49 | $or: body.filters[filter].map(val => { 50 | var condition = {}; 51 | condition[filter] = { 52 | $like: `%${val}%` 53 | }; 54 | return condition; 55 | }) 56 | }); 57 | } 58 | return { 59 | where: Sequelize.and(andConditions) 60 | }; 61 | } 62 | 63 | router.get("/logs", function(req, res) { 64 | path = process.env.LOG_FILE + moment(new Date()).format("YYYY-MM") + ".txt"; 65 | readLastLines 66 | .read(path, 100) 67 | .catch(function(error) { 68 | res.status(500).json({ logs: "Could not find log file." }); 69 | }) 70 | .then(lines => { 71 | res.status(200).json({ logs: lines }); 72 | }); 73 | }); 74 | 75 | router.post("/launch/preview", function(req, res) { 76 | var params = formatDomainQuery(req.body); 77 | 78 | models.Domain.count({ 79 | where: params.where 80 | }).then(function(count) { 81 | res.status(200).json({ count: count }); 82 | }); 83 | }); 84 | 85 | router.post("/launch", function(req, res) { 86 | var scan = { 87 | filters: req.body.filters, 88 | greps: req.body.greps, 89 | request: parser.parseRequest(req.body.request) 90 | }; 91 | 92 | sqs.sendMessage({ 93 | QueueUrl: process.env.SQS_URL, 94 | MessageBody: JSON.stringify({ command: "scan-hosts jsonInput", payload: JSON.stringify(scan) }) 95 | }) 96 | .promise() 97 | .catch(function(error) { 98 | res.status(500).json({ error: "Could not create job." }); 99 | }) 100 | .then(function(job) { 101 | return res.status(200).json({ status: "Successfully created job with id " + job.MessageId }); 102 | }); 103 | }); 104 | 105 | router.post("/enqueue", function(req, res) { 106 | var command = req.body.command; 107 | if (!validCommands.includes(command)) { 108 | return res.status(422).json({ error: "Invalid command" }); 109 | } 110 | if (req.body.args && req.body.args.length > 0) { 111 | command += " " + req.body.args.join(" "); 112 | } 113 | 114 | sqs.sendMessage({ 115 | QueueUrl: process.env.SQS_URL, 116 | MessageBody: JSON.stringify({ command: command }) 117 | }) 118 | .promise() 119 | .then(function(job) { 120 | return res.status(200).json({ status: "Successfully created job with id " + job.MessageId }); 121 | }) 122 | .catch(function(error) { 123 | res.status(500).json({ error: "Could not create job." }); 124 | }); 125 | }); 126 | 127 | router.get("/configure", function(req, res) { 128 | loadCrontab(function(err, crontab) { 129 | var jobs = crontab.jobs(); 130 | res.status(200).json({ jobs: renderJobs(jobs) }); 131 | }); 132 | }); 133 | 134 | router.post("/configure", function(req, res) { 135 | loadCrontab(function(err, crontab) { 136 | if ( 137 | !req.body.commandType || 138 | !validCommands.includes(req.body.commandType) || 139 | !req.body.commandArgs == null || 140 | !req.body.freq || 141 | !req.body.frequnit 142 | ) { 143 | console.log(req.body); 144 | return res.status(422).json({ error: "Invalid command" }); 145 | } 146 | 147 | var commandStr = req.body.commandType; 148 | if (req.body.commandArgs !== "") commandStr += " " + req.body.commandArgs; 149 | 150 | var body = { command: commandStr }; 151 | var command = `aws sqs send-message --queue-url ${process.env.SQS_URL} --message-body '${JSON.stringify( 152 | body 153 | )}'`; 154 | var job = crontab.create(command); 155 | switch (req.body.frequnit) { 156 | case "minutes": 157 | if (req.body.freq < 1 || req.body.freq >= 60) 158 | return res.status(422).json({ error: "Minutes must be between 1 and 59" }); 159 | job.minute().every(req.body.freq); 160 | break; 161 | case "hours": 162 | job.minute().at(0); 163 | if (req.body.freq < 1 || req.body.freq >= 24) 164 | return res.status(422).json({ error: "Hours must be between 1 and 24" }); 165 | if (req.body.freq > 1) job.hour().every(req.body.freq); 166 | break; 167 | case "days": 168 | job.minute().at(0); 169 | job.hour().at(0); 170 | if (req.body.freq < 1 || req.body.freq >= 30) 171 | return res.status(422).json({ error: "Days must be between 1 and 29" }); 172 | if (req.body.freq > 1) job.dom().every(req.body.freq); 173 | break; 174 | case "months": 175 | job.minute().at(0); 176 | job.hour().at(0); 177 | job.dom().at(1); 178 | if (req.body.freq < 1 || req.body.freq >= 12) 179 | return res.status(422).json({ error: "Months must be between 1 and 11" }); 180 | if (req.body.freq > 1) job.month().every(req.body.freq); 181 | break; 182 | case "default": 183 | return res.status(422).json({ error: "Invalid frequency unit" }); 184 | } 185 | crontab.save(function(err, crontab) { 186 | if (err) return res.status(500).json({ error: "Error creating cron job" }); 187 | var jobs = crontab.jobs(); 188 | res.status(200).json({ jobs: renderJobs(jobs) }); 189 | }); 190 | }); 191 | }); 192 | 193 | router.post("/remove", function(req, res) { 194 | loadCrontab(function(err, crontab) { 195 | var jobs = crontab.jobs(); 196 | if (req.body.index < 0 || req.body.index > jobs.length) { 197 | return res.status(422).json({ error: "Invalid index." }); 198 | } 199 | crontab.remove(jobs[req.body.index]); 200 | crontab.save(function(err, crontab) { 201 | var jobs = crontab.jobs(); 202 | res.status(200).json({ jobs: renderJobs(jobs) }); 203 | }); 204 | }); 205 | }); 206 | 207 | module.exports = router; 208 | -------------------------------------------------------------------------------- /client/js/controllers/DashboardController.js: -------------------------------------------------------------------------------- 1 | angular 2 | .module("DashboardController", []) 3 | .controller("DashboardController", [ 4 | "$scope", 5 | "$state", 6 | "$stateParams", 7 | "$window", 8 | "Domain", 9 | "NgTableParams", 10 | function($scope, $state, $stateParams, $window, Domain, NgTableParams) { 11 | this.vulns = []; 12 | this.stats = {}; 13 | 14 | this.getMonthLabels = function(format) { 15 | var months = ["Jan", "Feb", "Mar", "Apr", "May", "June", "July", "Aug", "Sept", "Oct", "Nov", "Dec"]; 16 | var today = new Date(); 17 | var month = today.getMonth(); 18 | var year = today.getYear() + 1900; 19 | var list = []; 20 | for (var i = 0; i < 12; i++) { 21 | list.push(format(month + 1, months[month], year)); 22 | month--; 23 | if (month < 0) { 24 | month = 11; 25 | year -= 1; 26 | } 27 | } 28 | return list.reverse(); 29 | }; 30 | 31 | this.type = "vulns"; 32 | this.charts = { 33 | severities: { 34 | data: [], 35 | labels: ["Critical", "High", "Medium", "Low", "None"], 36 | colors: ["#BB281A", "#CE4A23", "#EDA33B", "#F7CB48", "#6AA746"] 37 | }, 38 | monthly: { 39 | data: [], 40 | labels: this.getMonthLabels((_, month, year) => month + " " + year), 41 | options: { 42 | responsive: true, 43 | maintainAspectRatio: false, 44 | elements: { line: { fill: false } }, 45 | scaleGridLineColor: "#CE4A23" 46 | }, 47 | series: ["Series 1"] 48 | }, 49 | weaknesses: { 50 | data: [], 51 | labels: [], 52 | options: { 53 | responsive: true, 54 | maintainAspectRatio: false 55 | } 56 | }, 57 | topDomains: { 58 | data: [], 59 | labels: [], 60 | options: { 61 | responsive: true, 62 | maintainAspectRatio: false 63 | } 64 | }, 65 | topDomainsHigh: { 66 | data: [], 67 | labels: [], 68 | options: { 69 | responsive: true, 70 | maintainAspectRatio: false 71 | } 72 | } 73 | }; 74 | 75 | this.globalChartOptionsLegend = { 76 | responsive: true, 77 | maintainAspectRatio: false, 78 | legend: { 79 | display: true 80 | } 81 | }; 82 | 83 | this.fetchAll = () => { 84 | var savedState = localStorage.getItem("vulns"); 85 | if (savedState) { 86 | this.vulns = JSON.parse(savedState); 87 | this.computeStats(); 88 | } else { 89 | Domain.fetch(this.type, {}) 90 | .then(response => { 91 | this.vulns = response.data; 92 | localStorage.setItem("vulns", JSON.stringify(this.vulns)); 93 | this.computeStats(); 94 | }) 95 | .catch(error => { 96 | console.log(error); 97 | }); 98 | } 99 | }; 100 | 101 | this.computeStats = () => { 102 | var stats = { 103 | total: this.vulns.length, 104 | numOpen: 0, 105 | severities: { 106 | critical: 0, 107 | high: 0, 108 | medium: 0, 109 | low: 0, 110 | none: 0 111 | }, 112 | topWeaknesses: {}, 113 | commonDomains: {}, 114 | rootDomains: {}, 115 | rootDomainsHigh: {} 116 | }; 117 | for (var vuln of this.vulns) { 118 | if (vuln.state == "open") { 119 | stats.numOpen++; 120 | } 121 | stats.severities[vuln.severity] += 1; 122 | 123 | if (vuln.weakness in stats.topWeaknesses) { 124 | stats.topWeaknesses[vuln.weakness] += 1; 125 | } else { 126 | stats.topWeaknesses[vuln.weakness] = 1; 127 | } 128 | 129 | if (!vuln.domains) continue; 130 | for (var domain of vuln.domains.split(",")) { 131 | if (domain in stats.commonDomains) { 132 | stats.commonDomains[domain] += 1; 133 | } else { 134 | stats.commonDomains[domain] = 1; 135 | } 136 | var split = domain.split("."); 137 | 138 | var rootDomain = split[split.length - 2] + "." + split[split.length - 1]; 139 | if (rootDomain in stats.rootDomains) { 140 | stats.rootDomains[rootDomain] += 1; 141 | } else { 142 | stats.rootDomains[rootDomain] = 1; 143 | } 144 | 145 | if (vuln.state == "open" && (vuln.severity == "high" || vuln.severity == "critical")) { 146 | if (rootDomain in stats.rootDomainsHigh) { 147 | stats.rootDomainsHigh[rootDomain] += 1; 148 | } else { 149 | stats.rootDomainsHigh[rootDomain] = 1; 150 | } 151 | } 152 | } 153 | } 154 | 155 | stats.commonDomains = this.toSortedArray(stats.commonDomains); 156 | stats.rootDomains = this.toSortedArray(stats.rootDomains); 157 | stats.rootDomainsHigh = this.toSortedArray(stats.rootDomainsHigh); 158 | stats.topWeaknesses = this.toSortedArray(stats.topWeaknesses); 159 | 160 | console.log(stats); 161 | 162 | this.stats = stats; 163 | this.charts.severities.data = [ 164 | this.stats.severities.critical, 165 | this.stats.severities.high, 166 | this.stats.severities.medium, 167 | this.stats.severities.low, 168 | this.stats.severities.none 169 | ]; 170 | var months = this.getMonthLabels((ind, _, year) => year + "-" + (ind > 9 ? ind : "0" + ind)); 171 | var data = new Array(12).fill(0); 172 | for (var vuln of this.vulns) { 173 | for (var i in months) { 174 | if (vuln.reported_at.startsWith(months[i])) { 175 | data[i]++; 176 | } 177 | } 178 | } 179 | this.charts.monthly.data = data; 180 | 181 | this.charts.topDomains.labels = this.stats.rootDomains.slice(0, 10).map(a => a[0]); 182 | this.charts.topDomains.data = this.stats.rootDomains.slice(0, 10).map(a => a[1]); 183 | 184 | this.charts.topDomainsHigh.labels = this.stats.rootDomainsHigh.slice(0, 10).map(a => a[0]); 185 | this.charts.topDomainsHigh.data = this.stats.rootDomainsHigh.slice(0, 10).map(a => a[1]); 186 | 187 | this.charts.weaknesses.labels = this.stats.topWeaknesses.slice(0, 10).map(a => a[0]); 188 | console.log(this.charts.weaknesses.labels); 189 | this.charts.weaknesses.data = this.stats.topWeaknesses.slice(0, 10).map(a => a[1]); 190 | console.log(this.charts.weaknesses.data); 191 | }; 192 | 193 | this.toSortedArray = function(obj) { 194 | var sorted = Object.keys(obj).map(key => [key, obj[key]]); 195 | return sorted.sort((first, second) => second[1] - first[1]); 196 | }; 197 | 198 | this.add = function() { 199 | Domain.create(this.url) 200 | .then(response => { 201 | for (url of response.data) { 202 | this.urls.unshift(url); 203 | } 204 | }) 205 | .catch(error => { 206 | $scope.formError = error.data.message; 207 | console.log(error); 208 | }); 209 | }; 210 | 211 | this.clear = function() { 212 | this.tableParams.filter({}); 213 | this.tableParams.sorting({}); 214 | this.tableParams.url({}); 215 | }; 216 | 217 | this.delete = function(id) { 218 | Url.delete(id) 219 | .then(response => { 220 | this.urls = this.urls.filter(url => url.id !== id); 221 | }) 222 | .catch(error => { 223 | console.log(error); 224 | }); 225 | }; 226 | } 227 | ]) 228 | 229 | .filter("formatDate", () => { 230 | return function(value) { 231 | return value.split(" ")[0]; 232 | }; 233 | }); 234 | --------------------------------------------------------------------------------