├── .gitignore ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── PULL_REQUEST_TEMPLATE.md └── CONTRIBUTING.md ├── LICENSE ├── cx_health_mon_config.json ├── Functionality.md ├── README.md └── CxHealthMonitor.ps1 /.gitignore: -------------------------------------------------------------------------------- 1 | json/ 2 | *.log -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest new functionality for this project. 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the problem 11 | 12 | > A clear description of what the problem is. 13 | 14 | ### Proposed solution 15 | 16 | > A clear description of what you want to happen. 17 | 18 | ### Additional details 19 | 20 | > Add any other details / contexts / screenshots about the feature request. 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2020 Checkmarx 2 | 3 | This software is licensed for customer's internal use only. 4 | 5 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 6 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 7 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 8 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 9 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 10 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 11 | THE SOFTWARE. -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Report an issue 3 | about: Create a bug report to fix an existing issue. 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Description 11 | 12 | > Provide a description of the issue 13 | 14 | ### Expected Behavior 15 | 16 | ### Actual Behavior 17 | 18 | ### Reproduction 19 | 20 | > Detail the steps taken to reproduce the issue 21 | > 22 | > Where applicable, please include (exclude sensitive information): 23 | > 24 | > - Code of Files to reproduce the issue 25 | > - Log files 26 | > - Application settings 27 | > - Screenshots 28 | 29 | ### Environment Details 30 | 31 | > Provide any information relating to the environment the issue was identified in - include applicable version and additional runtime information (include OS or other underlying infrastructure) 32 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | By submitting a PR to this repository, you agree to the terms within the [Checkmarx Code of Conduct](https://github.com/checkmarx-ts/open-source-template/blob/master/CODE-OF-CONDUCT.md). Please see the [contributing guidelines](https://github.com/checkmarx-ts/open-source-template/blob/master/CONTRIBUTING.md) for how to create and submit a high-quality PR for this repo. 2 | 3 | ### Description 4 | 5 | > Describe the purpose of this PR along with any background information and the impacts of the proposed change. 6 | 7 | ### References 8 | 9 | > Include supporting link to GitHub Issue/PR number 10 | 11 | ### Testing 12 | 13 | > Describe how this change was tested. Be specific about anything not tested and reasons why. If this solution has unit and/or integration testing, tests should be added for new functionality and existing tests should complete without errors. 14 | > 15 | > Please include any manual steps for testing end-to-end or functionality not covered by unit/integration tests. 16 | 17 | ### Checklist 18 | 19 | - [ ] I have added documentation for new/changed functionality in this PR (if applicable). *If documentaiton is a Wiki Update, please indicate desired changes within PR MD Comment* 20 | - [ ] All active GitHub checks for tests, formatting, and security are passing 21 | - [ ] The correct base branch is being used 22 | -------------------------------------------------------------------------------- /cx_health_mon_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "cx": { 3 | "host": "http://localhost", 4 | "db": { 5 | "instance": "localhost\\SQLExpress" 6 | } 7 | }, 8 | "monitor": { 9 | "useUTCTimeOnClient": "false", 10 | "apiResponseTimeoutSeconds": 120, 11 | "pollIntervalSeconds": 30, 12 | "thresholds": { 13 | "queuedScansThreshold": 5, 14 | "queuedTimeThresholdMinutes": 20, 15 | "scanDurationThresholdMarginPercent": 25.0, 16 | "scanRateAsLOCPerHour": 150000, 17 | "engineResponseThresholdSeconds": 60.0, 18 | "restResponseThresholdSeconds": 60.0 19 | }, 20 | "retries": 5 21 | }, 22 | "alerts": { 23 | "waitingPeriodBetweenAlertsMinutes": 15, 24 | "suppressionRegex": "" 25 | }, 26 | "alertingSystems": { 27 | "smtp": [ 28 | { 29 | "systemType": "smtp", 30 | "name": "Email", 31 | "host": "smtp.mailserver.com", 32 | "port": 587, 33 | "sender": "admin@mailserver.com", 34 | "recipients": "list@of.com, email@addresses.com", 35 | "subject": "Checkmarx Health Monitor Alert", 36 | "useSsl": true 37 | } 38 | ], 39 | "syslog": [ 40 | { 41 | "systemType": "syslog", 42 | "name": "Kiwi", 43 | "host": "localhost", 44 | "port": 514 45 | }, 46 | { 47 | "systemType": "syslog", 48 | "name": "Splunk", 49 | "host": "localhost", 50 | "port": 515 51 | } 52 | ], 53 | "webhooks" : [ 54 | { 55 | "systemType": "slack", 56 | "name": "Slack", 57 | "hook" : "https://hooks.slack.com/services/xxxxxxxxxxxxxxxxxxx" 58 | }, 59 | { 60 | "systemType": "msteams", 61 | "name": "Teams", 62 | "hook" : "https://.webhook.office.com/webhookb2/xxxxxxxxxxxxxxxxxxx/IncomingWebhook/xxxxxxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxxxxx" 63 | } 64 | ] 65 | }, 66 | "log": { 67 | "jsonDirectory": "json" 68 | } 69 | } -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Checkmarx projects 2 | 3 | Welcome and thank you for considering contributing to a Checkmarx open source project. 4 | 5 | Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. It also communicates that you agree to respect the time of the developers managing and developing these open source projects. In return, we will reciprocate that respect by addressing your issue, assessing changes, and helping you finalize your pull requests. 6 | 7 | 8 | ## Quicklinks 9 | 10 | - [Contributing to Checkmarx projects](#contributing-to-checkmarx-projects) 11 | - [Quicklinks](#quicklinks) 12 | - [Code of Conduct](#code-of-conduct) 13 | - [Getting Started](#getting-started) 14 | - [Issues](#issues) 15 | - [Templates](#templates) 16 | - [Pull Requests](#pull-requests) 17 | - [Templates](#templates-1) 18 | - [Resources](#resources) 19 | 20 | ## Code of Conduct 21 | 22 | By participating and contributing any Checkmarx projects, you agree to uphold our [Code of Conduct](https://github.com/checkmarx-ts/open-source-template/blob/master/CODE-OF-CONDUCT.md). 23 | 24 | ## Getting Started 25 | 26 | *The following information applies to repositories within Checkmarx TS github categorized as **Solution** or **SDK*** (topic) 27 | 28 | If you have suggestions for how this project could be improved, or want to report a bug, open an issue. We appreciate all contributions. If you have questions, we'd love to hear them. 29 | 30 | We also appreciate PRs. If you're thinking of a large PR, we advise opening up an issue first to spark a discussion around it. 31 | 32 | Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both: 33 | 34 | - Search for existing Issues and PRs before creating your own to avoid duplicates. 35 | - PRs will only be accepted if associated with an issue (enhancement or bug) that has been submitted and reviewed/labeld as *accepted* by a Checkmarx team member. 36 | - We will work hard to makes sure issues that are raised are handled in a timely manner. 37 | 38 | ## Issues 39 | 40 | Issues should be used to report problems with the solution / source code, request a new feature, or to discuss potential changes before a PR is created. When you create a new Issue, a template will be loaded that will guide you through collecting and providing the information we need to investigate. 41 | 42 | If you find an Issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help be indicating to our maintainers that a particular problem is affecting more than just the reporter. 43 | 44 | ### Templates 45 | 46 | The following templates will be used within Checkmarx TS github repositories categorized as **Solution** or **SDK** (topic) 47 | - [Enhancement/Feature Request Template](.github/ISSUE_TEMPLATE/feature_request.md) 48 | - [Bug Report Template](.github/ISSUE_TEMPLATE/bug_report.md) 49 | 50 | ## Pull Requests 51 | 52 | PRs to our source is always welcome and can be a quick way to get your fix or improvement slated for the next release. In general, PRs should: 53 | 54 | - Only fix/add the functionality in question **or** address code style issues, not both. 55 | - Ensure all necessary details are provided and adhered to 56 | - Add unit or integration tests for fixed or changed functionality (if a test suite already exists) or specify steps taken to ensure changes were tested and functionality works as expected. 57 | - Address a single concern in the least number of changed lines as possible. 58 | - Include documentation in the repo or Provide additional comments in Markdown comments that should be pulled/reflected in GitHub Wiki for the given project. 59 | - Be accompanied by a complete Pull Request template (loaded automatically when a PR is created). 60 | 61 | For changes that address core functionality or would require breaking changes (e.g. a major release), it's best to open an Issue to discuss your proposal first. 62 | 63 | In general, we follow the _fork-and-pull_ Git workflow 64 | 65 | 1. Fork the repository to your own Github account 66 | 2. Clone the project to your machine 67 | 3. Create a branch locally with a succinct but descriptive name (prefix with feature/-descriptive-name> or hotfix/-descriptive-name) 68 | 4. Commit changes to the branch 69 | 5. Push changes to your fork 70 | 6. Open a PR in our repository and follow the PR template so that we can efficiently review and asses the changes. *Ensure an associated Issue has been accepted by the Checkmarx team.* 71 | 7. Assign **Checkmarx TS FieldDev** team as Reviewers of the PR 72 | 73 | ### Templates 74 | The following template will be used within Checkmarx TS github repositories categorized as **Solution** or **SDK** (topic) 75 | 76 | [Pull Request Template](.github/PULL_REQUEST_TEMPLATE.md) 77 | 78 | ## Resources 79 | 80 | - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) 81 | - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) 82 | - [GitHub Help](https://help.github.com) 83 | -------------------------------------------------------------------------------- /Functionality.md: -------------------------------------------------------------------------------- 1 | # CxHealthMonitor - Functionality 2 | The CxHealthMonitor is written in PowerShell and is designed to be modular. Additional functionality can be written and added as needed. 3 | 4 | > **Note : Supported Alerting Systems** The CxHealthMonitor supports the following systems: 5 | > - SMTP (Email) 6 | > - Syslog (Ex. Splunk, Kiwi, AlienVault, any SIEM system that accepts Syslog sources) 7 | > - Webhooks (Ex. Slack, Teams) 8 | > - Event Logging (the monitor writes to the Console, a general log file and structured event data to an events file - to feed into 3rd 9 | > party log aggregator products) Additional alerting systems (such as 10 | > SNMP) can be written and plugged in if required. 11 | 12 | 13 | 14 | ## Queue Monitoring 15 | The monitor evaluates the states of scans currently in the queue. There are two primary conditions it looks for: 16 | - Queue flooding - Excessive number of scans in the queue 17 | - Scans that remains in the 'Queued' state for prolonged periods 18 | 19 | ### Too many scans in queue 20 | Alerts are generated when the number of scans in the queue exceed a configurable threshold. 21 | 22 | Example alert/event: 23 | >`9/9/2019 11:22:58 PM: TOO_MANY_SCANS_IN_QUEUE : Queued: [6]. Threshold: [5]` 24 | 25 | ### Scans stuck in queue for a long time 26 | Alerts are generated when scans remain in the queue beyond a configurable number of minutes. 27 | 28 | Example alert/event: 29 | >`9/10/2019 1:55:10 PM: SCAN_TOO_LONG_IN_QUEUE : Scan [1000198, Spectrum (ReactJS), Full, SourcePullingAndDeployment]. Queued: [00:21:56.1022174]. Threshold: [00:20:00]` 30 | 31 | ## Scan Monitoring 32 | The monitor evaluates the states of running scans. It publishes alerts for the following: 33 | - Slow running scans 34 | - Scan failures 35 | 36 | ### Slow running scans 37 | The monitor generates alerts for Scans that exceed an estimated duration. 38 | 39 | The algorithm that calculates the duration estimate is swappable - and can be swapped out when more complex algorithms are developed. 40 | 41 | The current algorithm estimates the duration of a given scan by one of two methods: 42 | 43 | - LOC (lines of code) based 44 | - Previous run duration + configurable buffer time 45 | 46 | Example alert/event: 47 | >`8/5/2019 10:21:02 AM: SLOW_SCAN : Scan [1000193, Calypso, Full, Scanning] Elapsed: [06:18:21.9170000]. Threshold: duration [05:45:13.2770000]` 48 | 49 | ### Scan failures 50 | The monitor publishes an alert when a scan fails. Scan failure alerts will generally include a reason for the scan failure. 51 | 52 | Example alerts/events: 53 | >`9/6/2019 1:04:41 PM: SCAN_FAILED : Scan [1000237, cxmon-develop, Full, Failed] (Reason: Git clone failed: repository 'https://76eac6906c989f5eabbc0f331ebcef9c9c4129a4@github.com/gemgit7/cxmon.git/' not found` 54 | 55 | >`8/5/2019 6:03:34 PM: SCAN_FAILED : Scan [1000198, Spectrum (ReactJS), Full, Failed] (Reason: Scan failed due to insufficient memory. Engine server has a total 24414 MB out of which only 0 MB are free. To scan project of 96186 lines of code engine requires 384 MB of free memory. Please consider adding more RAM, reducing code size or closing running processes.)` 56 | 57 | ## Engine Monitoring 58 | The monitor evaluates the health of all registered engines and generates alerts for the following: 59 | 60 | - Slow response 61 | - Offline engine 62 | 63 | ### Sluggish engine 64 | There can be several reasons why an engine responds slowly. Generally an overworked and busy engine will respond slower to the monitor's health-check API call. When the monitor detects a sluggish response from a given engine, in relation to a configurable threshold, it publishes an alert. 65 | 66 | Example alert/event: 67 | >`9/9/2019 6:00:32 PM: SLOW_ENGINE : Engine [1, Localhost, Scanning] Response Time: [3.014502] seconds. Threshold: [0.50] seconds.` 68 | 69 | ### Offline engine 70 | The monitor publishes an alert when it detects that an engine is offline. 71 | 72 | Example alert/event: 73 | >`9/10/2019 4:25:44 PM: ENGINE_OFFLINE : Engine [1, Localhost, Offline]` 74 | 75 | ## Portal Monitoring 76 | The monitor evaluates the responsiveness of the portal by measuring how long it takes for the portal to respond to the monitor's page request. If the duration exceeds a configurable threshold, an alert is published. 77 | 78 | The monitor uses an anonymous request to the login page as a proxy for portal responsiveness. 79 | 80 | Example alert/event: 81 | >`9/6/2019 10:26:18 PM: SLOW_PORTAL : Slow Portal response. Response Time: [1.0396037] seconds. Threshold: [0.50] seconds.` 82 | 83 | ## Audit Monitoring 84 | The monitor generates alerts for audit conditions that are frequently requested by customers. The following audit alerts are supported: 85 | - Results Severity, State, Assignment changes 86 | - Project changes 87 | - Query changes 88 | - Preset changes 89 | 90 | ### Enable Audit Monitoring 91 | To enable audit monitoring, add the argument -audit when running CxOverwatch 92 | >`.\CxHealthMonitor.ps1 -audit` 93 | 94 | 95 | ### Scan Results : Severity / State changes 96 | Alerts are generated when scan results are updated - when the Severity and/or the State are changed. 97 | 98 | Example alert/event: 99 | >`9/6/2019 10:47:36 PM: AUDIT : [Results] : Action = [Changed status to Not Exploitable] , Query = [Reflected_XSS_All_Clients] , Project = [dvja-master] , File = [\src\main\webapp\WEB-INF\dvja\ProductList.jsp] , Line = [23] , Column = [446] , User = [admin@cx] , Timestamp = [09/06/2019 22:47:34] ` 100 | 101 | >`9/6/2019 10:51:07 PM: AUDIT : [Results] : Action = [Changed severity to High] , Query = [Insecure_Credential_Storage_Mechanism] , Project = [FreeNote] , File = [\server\rest_test.go] , Line = [46] , Column = [74] , User = [admin@cx] , Timestamp = [09/06/2019 22:50:58] 102 | ` 103 | ### Project changes 104 | Alerts are published when Projects are created/updated. 105 | 106 | Example alert/event: 107 | >`9/6/2019 1:00:58 PM: [Project] : Action : [Update_project] , Name : [DVJA] , User : [admin@cx] , Timestamp : [09/06/2019 13:00:56] 108 | ` 109 | 110 | ### Query changes 111 | Alerts are published when someone creates or updates a query. 112 | 113 | Example alert/event: 114 | >`9/6/2019 11:02:13 PM: AUDIT : [Query] : Action = [Create_Query] , Name = [Find_Interactive_Inputs] , User = [service@cx] , Timestamp = [09/06/2019 23:01:57] 115 | ` 116 | 117 | ### Preset changes 118 | Alerts are published when there are changes to a Preset. 119 | 120 | Example alert/event: 121 | >`9/6/2019 11:07:46 PM: AUDIT : [Preset] : Action = [Update] , Name = [High and Medium] , User = [admin@cx] , Timestamp = [09/06/2019 23:07:43] ` 122 | 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Checkmarx Health Monitor 2 | 3 | The Checkmarx Health Monitor is a tool that monitors the following: 4 | * Queue 5 | * Number of scans in queue 6 | * How long a scan remains in the 'Queued' state 7 | * Scans 8 | * Slow running scan 9 | * Scan failure 10 | * Engine 11 | * Responsiveness 12 | * Offline 13 | * Portal 14 | * Responsiveness 15 | * Audits 16 | * Project changes 17 | * Query changes 18 | * Preset changes 19 | * Results Severity, State, Assignment changes 20 | 21 | ## Getting Started 22 | 23 | The tool consists of two files - a Powershell script (the monitor) and a JSON file (the configuration) 24 | * CxHealthMonitor.ps1 25 | * cx_health_mon_config.json 26 | 27 | 28 | ### Prerequisites 29 | 30 | * Powershell V5 (Windows 10 has powershell 5.1 installed) 31 | * https://docs.microsoft.com/en-us/powershell/scripting/install/installing-windows-powershell?view=powershell-6 32 | * Access to the Checkmarx Server and Database 33 | * Adding the following credentials to the Windows Credential Manager (One time process) 34 | 1) **Checkmarx SAST Credentials** : 35 | 1. Open the Windows Credential Manager ( Control Panel -> Credential Manager) 36 | 2. Click on 'Windows Credentials' and scroll down till you see 'Generic Credentials' section. 37 | 3. Click on 'Add a generic credential' and enter the following values: 38 | Internet or network address : **CxOverwatch.SAST** (Kindly make sure you copy the same name as mentioned here) 39 | User name : cxadmin 40 | Password : xxxxx 41 | 42 | 4. Click 'Ok' to save your credentials. 43 | 44 | 2) **Checkmarx SAST DB Credentials** : 45 | Note : Storing SAST database credentials is Optional. If you want to use Windows Authentication for the SQL Server, DO NOT add the credentials for the database. 46 | 47 | 1. Open the Windows Credential Manager ( Control Panel -> Credential Manager) 48 | 2. Click on 'Windows Credentials' and scroll down till you see 'Generic Credentials' section. 49 | 3. Click on 'Add a generic credential' and enter the following values: 50 | Internet or network address : **CxOverwatch.SAST.DB** (Kindly make sure you copy the same name as mentioned here) 51 | User name : dbuser 52 | Password : xxxxx 53 | 54 | 4. Click 'Ok' to save your credentials. 55 | 56 | 3) **Email Alert Credentials** : 57 | Note : Storing Email credentials is Optional. If you are not using Email as your alert system or if you want to use anonymous SMTP, you can ignore this section. 58 | 59 | 1. Open the Windows Credential Manager ( Control Panel -> Credential Manager) 60 | 2. Click on 'Windows Credentials' and scroll down till you see 'Generic Credentials' section. 61 | 3. Click on 'Add a generic credential' and enter the following values: 62 | Internet or network address : **CxOverwatch.EmailAlert** (Kindly make sure you copy the same name as mentioned here) 63 | User name : username@mailserver.com 64 | Password : xxxxx 65 | 66 | 4. Click 'Ok' to save your credentials. 67 | 68 | Note : If the script is not able to find the Checkmarx SAST credentials in the Credential Manager, it will show a prompt to user asking for the SAST username and password. 69 | 70 | ## Usage 71 | 72 | ``` 73 | .\CxHealthMonitor.ps1 [-cxUser username] [-cxPass password] [-dbUser username] [-dbPass password] [-audit] 74 | ``` 75 | 76 | The optional arguments will override the corresponding values stored in the Credential Manager. 77 | 78 | Add the argument -audit to enable monitoring of audits. This is not enabled by default and should only be used if the user has access to the db. 79 | 80 | Note: If the optional db parameters are skipped and the corresponding entries in the Credential Manager are empty, the monitor will use SQLServer Windows authentication. 81 | 82 | ## Configuration 83 | 84 | The configuration file (cx_health_mon_config.json) consists of the following sections: 85 | * cx 86 | * monitor 87 | * alerts 88 | * alertingSystems 89 | 90 | The **"cx"** section drives connectivity to the Checkmarx server and database. The Checkmarx Server URL and database instance are configured here. 91 | ```json 92 | "cx": { 93 | "host": "http://checkmarx.domain.com", 94 | "db": { 95 | "instance": "localhost\\SQLExpress" 96 | } 97 | } 98 | ``` 99 | 100 | The **"monitor"** section is used to configure thresholds and monitoring parameters. 101 | ```json 102 | "monitor": { 103 | "useUTCTimeOnClient": "true", 104 | "pollIntervalSeconds": 30, 105 | "thresholds": { 106 | "queuedScansThreshold": 5, 107 | "queuedTimeThresholdMinutes": 20, 108 | "scanDurationThresholdMarginPercent": 25.0, 109 | "scanRateAsLOCPerHour": 150000, 110 | "engineResponseThresholdSeconds": 60.0, 111 | "restResponseThresholdSeconds": 60.0 112 | }, 113 | "apiResponseTimeoutSeconds": 120, 114 | "retries": 5 115 | } 116 | ``` 117 | 118 | * _useUTCTimeOnClient_: "true" or "false" - if true forces UTC for calculation of time on the client. Useful when script runs on a machine in a local time zone but server runs in UTC. 119 | * _pollIntervalSeconds_: Polling cadence - how often the monitor will connect to the Checkmarx server for monitoring purposes. 120 | * _queuedScansThreshold_: Threshold for the maximum number of scans in the CxSAST Queue, beyond which alerts will be sent. 121 | * _queuedTimeThresholdMinutes_: Threshold for the number of minutes a scan can remain in the CxSAST Queue, beyond which alerts will be sent. 122 | * _scanDurationThresholdMarginPercent_: Additional duration (added as a percentage) to a scan's estimated duration, beyond which the scan will be marked 'slow'. 123 | * _scanRateAsLOCPerHour_: The scan rate, in Lines Of Code per hour - to be used in estimating a scan's expected duration. 124 | * _engineResponseThresholdSeconds_: Threshold (in seconds) for an engine to respond to the monitor's API call. 125 | * _restResponseThresholdSeconds_: Threshold (in seconds) for the CxManager to respond to the monitor's API call. This is a proxy for Portal responsiveness. 126 | * _apiResponseTimeoutSeconds_: Specified in seconds, this is how long the monitor will wait for a response from the monitored system, before timing out and trying again (see 'retries'). 127 | * _retries_: Number of times the monitor will attempt to connect to the monitored system before giving up. 128 | 129 | 130 | The **"alerts"** section is used to configure values specific to Alerts. 131 | 132 | ```json 133 | "alerts": { 134 | "waitingPeriodBetweenAlertsMinutes": 15, 135 | "suppressionRegex": "" 136 | } 137 | ``` 138 | * _waitingPeriodBetweenAlertsMinutes_: Period in minutes, to wait before sending out subsequent alerts arising from the same monitored subject and the same conditions. This configuration controls/prevents alert flooding. 139 | *_suppressionRegex_: Alert messages that match this regular expression will be suppressed. Supports multiple patterns like "(pattern1|pattern2)". 140 | 141 | The **"alertingSystems"** section is used to configure available Alerting Systems to be used by the monitor. 142 | The monitor ships with multiple Alerting System implementations, such as Email(smtp), Syslog and Event Logs. When new implementations (such as SNMP etc.) are available, this is where they should be configured. 143 | 144 | Leave smtp user and password blank for anonymous smtp. 145 | 146 | Follow instructions on creating an incoming webhook for: 147 | - Slack notifications: https://api.slack.com/messaging/webhooks 148 | - Teams notifications: https://docs.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook 149 | 150 | 151 | ```json 152 | "alertingSystems": { 153 | "smtp": [ 154 | { 155 | "systemType": "smtp", 156 | "name": "Email", 157 | "host": "007-myemailserver.com", 158 | "port": 587, 159 | "sender": "admin@myemailserver.com", 160 | "recipients": "list@of.com, email@addresses.com", 161 | "subject": "Checkmarx Health Monitor Alert", 162 | "useSsl": true 163 | } 164 | ], 165 | "syslog": [ 166 | { 167 | "systemType": "syslog", 168 | "name": "Kiwi", 169 | "host": "localhost", 170 | "port": 514 171 | }, 172 | { 173 | "systemType": "syslog", 174 | "name": "Splunk", 175 | "host": "localhost", 176 | "port": 515 177 | } 178 | ], 179 | "webhooks" : [ 180 | { 181 | "systemType": "slack", 182 | "name": "Slack", 183 | "hook" : "https://hooks.slack.com/services/xxxxxxxxxxxxxxxxxxx" 184 | }, 185 | { 186 | "systemType": "msteams", 187 | "name": "Teams", 188 | "hook" : "https://.webhook.office.com/webhookb2/xxxxxxxxxxxxxxxxxxx/IncomingWebhook/xxxxxxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxxxxx" 189 | } 190 | ] 191 | } 192 | ``` 193 | 194 | The **"log"** section is used to configure the JSON output directory to be used by the monitor. 195 | The _jsonDirectory_ element specifies where the JSON files output by the monitor should be written. 196 | 197 | ```json 198 | "log": { 199 | "jsonDirectory": "json" 200 | } 201 | ``` 202 | 203 | **Log Rotation** : The script will zip the log files every night and move them to the folder named 'zipped_logs'. 204 | 205 | ## Authors 206 | 207 | * Gem Immanuel, Checkmarx Professional Services - *Initial work* 208 | * Benjamin Stokes, Checkmarx Professional Services - patches 209 | 210 | 211 | ## License 212 | 213 | This project is licensed under **TBD** 214 | -------------------------------------------------------------------------------- /CxHealthMonitor.ps1: -------------------------------------------------------------------------------- 1 | <# 2 | Checkmarx CxSAST Health Monitoring System 3 | Version 1.0 4 | Gem Immauel (gem.immanuel@checkmarx.com) 5 | Checkmarx Professional Services 6 | 7 | Usage: .\CxHealthMonitor.ps1 [-cxUser cxaccount] [-cxPass cxpassword] [-audit] [-dbUser dbaccount] [-dbPass dbpassword] 8 | 9 | The command line parameters will override the values read from the 10 | configuration file (cx_health_mon.config.json) 11 | #> 12 | [CmdletBinding()] 13 | Param( 14 | [Parameter(Mandatory = $False)] 15 | [ValidateNotNullOrEmpty()] 16 | [String] 17 | $cxUser = "", 18 | 19 | [Parameter(Mandatory = $False)] 20 | [ValidateNotNullOrEmpty()] 21 | [String] 22 | $cxPass = "", 23 | 24 | [switch] 25 | $audit, 26 | 27 | [Parameter(Mandatory = $False)] 28 | [ValidateNotNullOrEmpty()] 29 | [String] 30 | $dbUser = "", 31 | 32 | [Parameter(Mandatory = $False)] 33 | [ValidateNotNullOrEmpty()] 34 | [String] 35 | $dbPass = "" 36 | ) 37 | 38 | # ----------------------- Module imports ------------------------ # 39 | 40 | if ($audit) { 41 | 42 | if(-not(Get-InstalledModule Invoke-SqlCmd2 -ErrorAction silentlycontinue)) 43 | { 44 | Set-PSRepository PSGallery -InstallationPolicy Trusted 45 | Install-Module Invoke-SqlCmd2 -Confirm:$False -Force 46 | } 47 | Import-Module "Invoke-SqlCmd2" -DisableNameChecking 48 | } 49 | 50 | if(-not(Get-InstalledModule CredentialManager -ErrorAction silentlycontinue)) 51 | { #Module is imported automatically, if any cmdlets are used. 52 | Set-PSRepository PSGallery -InstallationPolicy Trusted 53 | Install-Module CredentialManager -Confirm:$False -Force 54 | } 55 | 56 | # CxSAST REST API auth values 57 | [String] $CX_REST_GRANT_TYPE = "password" 58 | [String] $CX_REST_SCOPE = "sast_rest_api" 59 | [String] $CX_REST_CLIENT_ID = "resource_owner_client" 60 | [String] $CX_REST_CLIENT_SECRET = "014DF517-39D1-4453-B7B3-9930C563627C" 61 | 62 | # ----------------------------------------------------------------- 63 | # Input/Output Utility 64 | # ----------------------------------------------------------------- 65 | Class IO { 66 | 67 | # General logging 68 | static [String] $LOG_FILE = "cx_health_mon.log" 69 | # Event logging 70 | static [String] $EVENT_FILE = "cx_health_mon_events.log" 71 | hidden [DateTimeUtil] $dateUtil = [DateTimeUtil]::new() 72 | 73 | 74 | # Files for JSON output 75 | static [String] $FILE_SUFFIX_TIMESTAMP_FORMAT = "yyyyMMdd_hhmmssfff" 76 | 77 | # Logs given message to configured log file 78 | Log ([String] $message) { 79 | # Write to log file 80 | $this.WriteToFile($message, [IO]::LOG_FILE) 81 | # Also write to console 82 | $this.Console($message) 83 | } 84 | 85 | # Writes given message to configured events file 86 | LogEvent ([String] $message) { 87 | # Write to event file 88 | $this.WriteToFile($message, [IO]::EVENT_FILE) 89 | # Also write to log, console 90 | $this.Log($message) 91 | } 92 | 93 | # Write given string to host console 94 | Console ([String] $message) { 95 | Write-Host $this.AddTimestamp($message) 96 | } 97 | 98 | # Write to JSON file 99 | WriteJSON([AlertType] $jsonFile, [PSCustomObject] $object) { 100 | 101 | # Ensure folder exists 102 | [String] $jsonOutDir = $script:config.log.jsonDirectory 103 | If (!(Test-Path $jsonOutDir)) { 104 | New-Item -ItemType Directory -Force -Path $jsonOutDir 105 | } 106 | $jsonOutDir = (Get-Item -Path $jsonOutDir).FullName 107 | 108 | # Create timestamp 109 | [DateTime] $timestamp = $this.dateUtil.NowUTC() 110 | [String] $fileSuffix = $timestamp.ToString([IO]::FILE_SUFFIX_TIMESTAMP_FORMAT) 111 | 112 | if(((Get-Item -Path $jsonOutDir).LastWriteTime.DayOfYear) -lt ((Get-Date).DayOfYear)) 113 | { #Zip the current json folder 114 | [String] $ZIPPED_LOGS = "zipped_logs" 115 | if(!(Get-Item $ZIPPED_LOGS -ErrorAction SilentlyContinue)) 116 | { 117 | New-Item -ItemType Directory -Name $ZIPPED_LOGS -Force 118 | } 119 | 120 | [String] $jsonFolderName = (Get-Item -Path $jsonOutDir).Name 121 | [String] $newFolderName = $jsonFolderName + "_$fileSuffix.zip" 122 | Compress-Archive -Path $jsonFolderName -DestinationPath "$ZIPPED_LOGS\$newFolderName" -CompressionLevel Fastest 123 | Remove-Item -Path "$jsonFolderName\*" -Force -Recurse 124 | } 125 | 126 | # Update JSON blob with timestamp 127 | $object.EventDate = $this.dateUtil.Format($timestamp) 128 | 129 | # Create file name 130 | [String] $fileName = $jsonFile.ToString().ToLower() + "_$fileSuffix.json" 131 | [String] $jsonFilePath = Join-Path -Path "$jsonOutDir" -ChildPath $fileName 132 | 133 | # Write to file 134 | Add-content $jsonFilePath -Value ($object | ConvertTo-Json) 135 | } 136 | 137 | # Write a pretty header output 138 | WriteHeader() { 139 | Write-Host "-----------------------------------------" -ForegroundColor Green 140 | Write-Host "Checkmarx Health Monitor" -ForegroundColor Green 141 | Write-Host "Checkmarx CxSAST: $($script:config.cx.host)" 142 | if ($script:audit) { 143 | Write-Host "Checkmarx Database: $($script:config.cx.db.instance)" 144 | } 145 | Write-Host "Poll interval (seconds): $($script:config.monitor.pollIntervalSeconds)" 146 | Write-Host "Default scan rate (LOC / Hour): $($script:config.monitor.thresholds.scanRateAsLOCPerHour)" 147 | Write-Host "Threshold for number of scans in the queued state: $($script:config.monitor.thresholds.queuedScansThreshold)" 148 | Write-Host "Threshold for time spent in the queued state: $($script:config.monitor.thresholds.queuedTimeThresholdMinutes) minute(s)" 149 | Write-Host "Scan duration threshold margin: $($script:config.monitor.thresholds.scanDurationThresholdMarginPercent)%" 150 | Write-Host "Alerting Systems: [$($script:alertService.alertSystems.name)]" 151 | Write-Host "-----------------------------------------" -ForegroundColor Green 152 | } 153 | 154 | # Utility that writes to given file 155 | hidden WriteToFile([String] $message, [String] $file) { 156 | 157 | $existingFile = Get-Item $file -ErrorAction SilentlyContinue 158 | if ($existingFile) { 159 | 160 | if(($existingFile.LastWriteTime.DayOfYear) -lt ((Get-Date).DayOfYear)) 161 | { 162 | #Zip the current file 163 | [DateTime] $timestamp = $this.dateUtil.NowUTC() 164 | [String] $fileSuffix = $timestamp.ToString([IO]::FILE_SUFFIX_TIMESTAMP_FORMAT) 165 | [String] $newFileName = $file + "_$fileSuffix" 166 | [String] $newZipFileName = $file + "_$fileSuffix.zip" 167 | 168 | [String] $ZIPPED_LOGS = "zipped_logs" 169 | if(!(Get-Item $ZIPPED_LOGS -ErrorAction SilentlyContinue)) 170 | { 171 | New-Item -ItemType Directory -Name $ZIPPED_LOGS -Force 172 | } 173 | Rename-Item $existingFile.Name -NewName $newFileName 174 | Compress-Archive -Path $newFileName -DestinationPath "$ZIPPED_LOGS\$newZipFileName" 175 | Remove-Item $newFileName -Force 176 | } 177 | } 178 | Add-content $file -Value $this.AddTimestamp($message) 179 | } 180 | 181 | hidden [String] AddTimestamp ([String] $message) { 182 | return $this.dateUtil.NowUTCFormatted() + ": " + $message 183 | } 184 | } 185 | 186 | # ----------------------------------------------------------------- 187 | # Abstract Alert System 188 | # ----------------------------------------------------------------- 189 | Class AlertSystem { 190 | 191 | [String] $name = "Unknown Name. Alert System Name not explicitly set." 192 | [String] $systemType = "Unknown system type. Alert System Type not explicitly set." 193 | [IO] $io = [IO]::new() 194 | [DateTimeUtil] $dateUtil = [DateTimeUtil]::new() 195 | 196 | # Abstract constructor 197 | AlertSystem () { 198 | $type = $this.GetType() 199 | if ($type -eq [AlertSystem]) { 200 | throw("Class $type must be overridden by an alerting system implementation") 201 | } 202 | } 203 | 204 | # Sends an alert for given scan 205 | Send([String] $message) { 206 | # Force implementation by a concrete algo 207 | throw("Method is abstract. Needs to be overriden by an alerting system implementation.") 208 | } 209 | 210 | [String] GetSystemType() { 211 | return $this.systemType 212 | } 213 | 214 | # By default, alert systems do not batch(combine) alert messages. 215 | # Some systems, by design, can (ex. email systems) 216 | # If a system can batch messages, override this to return true 217 | [Bool] IsBatchMessages() { 218 | return $False 219 | } 220 | } 221 | 222 | # ----------------------------------------------------------------- 223 | # Standard Syslog Severities 224 | # ----------------------------------------------------------------- 225 | Enum SyslogSeverity { 226 | EMERGENCY = 0 227 | ALERT = 1 228 | CRITICAL = 2 229 | ERROR = 3 230 | WARNING = 4 231 | NOTICE = 5 232 | INFO = 6 233 | DEBUG = 7 234 | } 235 | 236 | # ----------------------------------------------------------------- 237 | # Support for alerting over the Syslog protocol 238 | # ----------------------------------------------------------------- 239 | Class SyslogAlertSystem : AlertSystem { 240 | 241 | hidden [String] $syslogServer 242 | hidden [int] $syslogPort 243 | hidden [SyslogSeverity] $severity = [SyslogSeverity]::ALERT 244 | 245 | # Constructs a syslog alerting system object 246 | SyslogAlertSystem([String] $systemType, [String] $name, [String] $syslogServer, [int] $syslogPort) { 247 | $this.systemType = $systemType 248 | $this.name = $name 249 | $this.syslogServer = $syslogServer 250 | $this.syslogPort = $syslogPort 251 | } 252 | 253 | # Sends given message over UDP to configured syslog server/port 254 | Send([String] $message) { 255 | 256 | # If there is no message, not much to do 257 | if (!$message) { return } 258 | 259 | # Prepend 'Checkmarx' as marker 260 | $message = "Checkmarx: $message" 261 | 262 | # Syslog Facility 1 : User-level message 263 | [int] $facility = 1 264 | [String] $hostname = $env:computername 265 | # Calculate the priority 266 | [int] $priority = ([int] $facility * 8) + [int] $this.severity.value__ 267 | # "MMM dd HH:mm:ss" or "yyyy:MM:dd:-HH:mm:ss zzz" 268 | [String] $timestamp = ($this.dateUtil.NowUTC()).ToString("MMM dd HH:mm:ss") 269 | 270 | # Syslog packet format 271 | [String] $syslogMessage = "<{0}>{1} {2} {3}" -f $priority, $timestamp, $hostname, $message 272 | 273 | # Create encoded syslog packet 274 | $encoder = [System.Text.Encoding]::ASCII 275 | $encodedPacket = $encoder.GetBytes($syslogMessage) 276 | 277 | # Connect to the syslog server and send packet over UDP 278 | $udpClient = New-Object System.Net.Sockets.UdpClient 279 | $udpClient.Connect($this.syslogServer, $this.syslogPort) 280 | $udpClient.Send($encodedPacket, $encodedPacket.Length) 281 | 282 | $this.io.Log("Sent syslog message to [$($this.name) : $($this.syslogServer)]") 283 | } 284 | } 285 | 286 | # ----------------------------------------------------------------- 287 | # Email Alert System 288 | # ----------------------------------------------------------------- 289 | Class EmailAlertSystem : AlertSystem { 290 | 291 | hidden [IO] $io 292 | hidden [String] $smtpHost 293 | hidden [int] $smtpPort 294 | hidden [pscredential] $smtpCredentials 295 | hidden [String] $subject 296 | hidden [String] $smtpSender 297 | hidden [String[]] $recipients 298 | hidden [Boolean] $useSsl 299 | 300 | # Constructs the email alert system object 301 | EmailAlertSystem ([String] $systemType, [String] $name, [String] $smtpHost, [int] $smtpPort, [String] $smtpUsername, [String] $smtpPassword, [String] $smtpSender, [String[]] $recipients, [String] $subject, [Boolean] $useSsl) { 302 | $this.io = [IO]::new() 303 | $this.systemType = $systemType 304 | $this.name = $name 305 | $this.smtpHost = $smtpHost 306 | $this.smtpPort = $smtpPort 307 | $this.smtpSender = $smtpSender 308 | $this.recipients = $recipients 309 | $this.subject = $subject 310 | $this.useSsl = $useSsl 311 | 312 | # Support anonymous authenticated smtp scenario 313 | if ($smtpUsername.Length -gt 0 -and $smtpPassword.Length -gt 0) { 314 | [CredentialsUtil] $credUtil = [CredentialsUtil]::new() 315 | $this.smtpCredentials = $credUtil.GetPSCredential($smtpUsername, $smtpPassword) 316 | } 317 | } 318 | 319 | # Override default behavior 320 | # Indicate that this system will batch messages. 321 | [Bool] IsBatchMessages() { 322 | return $True 323 | } 324 | 325 | # Sends an email with message 326 | Send([String] $message) { 327 | 328 | # No-frills implementation 329 | try { 330 | $this.io.Log("Sending email alert to [$($this.name) : $($this.recipients)]") 331 | 332 | $mailargs = @{ 333 | From = $this.smtpSender 334 | Body = $message 335 | Subject = $this.subject 336 | To = $this.recipients 337 | Priority = "High" 338 | SmtpServer = $this.smtpHost 339 | Port = $this.smtpPort 340 | } 341 | 342 | # If credentials are not provided then we will use anonymous smtp 343 | if ($this.smtpCredentials) { 344 | $mailargs.Add("Credential", $this.smtpCredentials) 345 | } 346 | 347 | if ($this.useSsl) { 348 | $mailargs.Add("UseSsl", $True) 349 | } 350 | 351 | Send-MailMessage @mailargs 352 | } 353 | catch { 354 | $this.io.Log("ERROR: [$($_.Exception.Message)] Could not send email alert. Verify email configuration.") 355 | } 356 | } 357 | } 358 | 359 | # ----------------------------------------------------------------- 360 | # Webhooks Alert System 361 | # ----------------------------------------------------------------- 362 | Class WebhooksAlertSystem : AlertSystem { 363 | 364 | hidden [IO] $io 365 | hidden [String] $systemType 366 | hidden [String] $name 367 | hidden [String] $hook 368 | 369 | # Constructs the email alert system object 370 | WebhooksAlertSystem ([String] $systemType, [String] $name, [String] $hook) { 371 | $this.io = [IO]::new() 372 | $this.systemType = $systemType 373 | $this.name = $name 374 | $this.hook = $hook 375 | } 376 | 377 | # Sends a Webhooks with message 378 | Send([String] $message) { 379 | 380 | # No-frills implementation 381 | try { 382 | $this.io.Log("Sending alert to [$($this.name)]") 383 | $body = @{ 384 | text = $message 385 | } 386 | $body = $body | ConvertTo-Json 387 | $response = Invoke-RestMethod -Uri $this.hook -Method Post -Body $body -ContentType 'application/json' 388 | } 389 | catch { 390 | $this.io.Log("ERROR: [$($_.Exception.Message)] Could not send Webhooks [$($this.systemType)] alert. Verify Webhooks [$($this.systemType)] configuration.") 391 | } 392 | } 393 | } 394 | 395 | # Enumerates types of alerts that we send 396 | # Helps keep track of which type of alert 397 | # was sent when and for which scan/project. 398 | Enum AlertType { 399 | SCAN_FAILED 400 | SCAN_SLOW 401 | QUEUE_SCAN_TIME_EXCEEDED 402 | QUEUE_SCAN_EXCESS 403 | ENGINE_OFFLINE 404 | ENGINE_RESPONSE_SLOW 405 | ENGINE_IDLE 406 | ENGINE_ERROR 407 | PORTAL_SLOW 408 | AUDIT 409 | } 410 | 411 | # ----------------------------------------------------------------- 412 | # Alert Service 413 | # ----------------------------------------------------------------- 414 | Class AlertService { 415 | 416 | hidden [IO] $io 417 | hidden [System.Collections.ArrayList] $alerts = @() 418 | hidden [System.Collections.ArrayList] $alertSystems = @() 419 | # Key = scanId_projectName_scanType_AlertType 420 | # Value = TimeSent 421 | hidden [Hashtable] $sent = @{ } 422 | 423 | # Number of minutes to wait before sending an 424 | # alert for the same scan and/or condition 425 | hidden [TimeSpan] $waitBetweenAlerts 426 | 427 | AlertService () { 428 | $this.io = [IO]::new() 429 | $this.waitBetweenAlerts = [TimeSpan]::FromMinutes($script:config.alerts.waitingPeriodBetweenAlertsMinutes) 430 | $this.RegisterAlertSystems() 431 | } 432 | 433 | # Register configured alert systems 434 | RegisterAlertSystems() { 435 | # Register alerting systems specified in configuration file 436 | 437 | foreach ($alertingSystem in $script:config.alertingSystems) { 438 | 439 | # For now, we register each type of alerting systems separately. 440 | # Enhancement would be to have configuration self-declare system type 441 | # and have a factory create the system for you :) 442 | 443 | # Register SMTP systems, if configured 444 | if ($alertingSystem.smtp) { 445 | foreach ($smtpSystem in $alertingSystem.smtp) { 446 | 447 | $EMAIL_CREDENTIALS = "CxOverwatch.EmailAlert" 448 | $emailCredentials = Get-StoredCredential -Target $EMAIL_CREDENTIALS #Get from Windows Credential Manager 449 | $emailUsername = "" 450 | $emailPassword = "" 451 | 452 | if ($emailCredentials) { 453 | Write-Host "Found Email credentials." 454 | $emailUsername = $emailCredentials.UserName 455 | $emailPassword = $emailCredentials.GetNetworkCredential().Password 456 | } 457 | 458 | [AlertSystem] $emailAlertSystem = [EmailAlertSystem]::new($smtpSystem.systemType, $smtpSystem.name, $smtpSystem.host, $smtpSystem.port, $emailUsername, $emailPassword, $smtpSystem.sender, $smtpSystem.recipients, $smtpSystem.subject, $smtpSystem.useSsl) 459 | $this.AddAlertSystem($emailAlertSystem); 460 | } 461 | } 462 | 463 | # Register Syslog systems, if configured 464 | if ($alertingSystem.syslog) { 465 | foreach ($syslogSystem in $alertingSystem.syslog) { 466 | [AlertSystem] $syslogAlertSystem = [SyslogAlertSystem]::new($syslogSystem.systemType, $syslogSystem.name, $syslogSystem.host, $syslogSystem.port) 467 | $this.AddAlertSystem($syslogAlertSystem); 468 | } 469 | } 470 | # Register webhooks, if configured 471 | if ($alertingSystem.webhooks) { 472 | foreach ($webhookSystem in $alertingSystem.webhooks) { 473 | [AlertSystem] $webhookAlertSystem = [WebhooksAlertSystem]::new($webhookSystem.systemType, $webhookSystem.name, $webhookSystem.hook) 474 | $this.AddAlertSystem($webhookAlertSystem); 475 | } 476 | } 477 | } 478 | } 479 | 480 | # Add an AlertSystem to the Alert Service 481 | # This enables multiple AlertSystem implementations 482 | # Ex. Email, Syslog, SMNP etc. 483 | AddAlertSystem ([AlertSystem] $alertSystem) { 484 | $this.alertSystems.Add($alertSystem) 485 | $this.io.Log("Config: Added Alert System [$($alertSystem.name)]") 486 | } 487 | 488 | # Track an alert : Type and current timestamp 489 | Track ([String] $scanKey, [AlertType] $alertType) { 490 | $timestamp = Get-Date 491 | [String] $alertKey = $scanKey + "_" + $alertType 492 | if ($this.sent.ContainsKey($alertKey)) { 493 | # Update with current timestamp 494 | $this.sent[$alertKey] = $timestamp 495 | } 496 | else { 497 | $this.sent.Add($alertKey, $timestamp) 498 | } 499 | } 500 | 501 | # Should the alert be sent? 502 | # We determine if an alert should be sent again by: 503 | # checking if an alert had been previously sent for (scanId + projectName + scanType + alertType) 504 | # and we're past the waiting period between alerts 505 | [Bool] ShouldSend([String] $scanKey, [AlertType] $alertType) { 506 | [Bool] $goForIt = $True 507 | [String] $alertKey = $scanKey + "_" + $alertType 508 | if ($this.sent.containsKey($alertKey)) { 509 | [datetime] $now = Get-Date 510 | [datetime] $lastSent = $this.sent[$alertKey] 511 | 512 | # If we're still within (lastSent + waitingPeriod) don't send alert just yet 513 | if ($now -lt $lastSent.Add($this.waitBetweenAlerts)) { 514 | # $this.io.Console("Alert [$alertKey] still within waiting period between alerts.") 515 | $goForIt = $False 516 | } 517 | } 518 | return $goForIt 519 | } 520 | 521 | # Add an alert message to a list that will be sent as a batch on Send() 522 | AddAlert ([AlertType] $alertType, [String] $message, [String] $scanKey) { 523 | $this.io.LogEvent("$alertType : $message") 524 | # Add only if given message should be sent 525 | if ($this.ShouldSend($scanKey, $alertType)) { 526 | $this.alerts.Add("$alertType : $message") 527 | $this.Track($scanKey, $alertType) 528 | } 529 | } 530 | 531 | # Sends out alert message 532 | # via all registered alerting systems 533 | Send () { 534 | 535 | # If we don't have any alerts to send, return 536 | if ($this.alerts.Count -eq 0) { return } 537 | 538 | # Otherwise, send them out to every registered alerting system 539 | foreach ($alertSystem in $this.alertSystems) { 540 | 541 | # Batch(combine) messages is required: 542 | # Email systems, for instance. 543 | if ($alertSystem.IsBatchMessages()) { 544 | [String] $batchMessage = "" 545 | foreach ($message in $this.alerts) { 546 | if ($message -notmatch $script:config.alerts.suppressionRegex -Or [String]::IsNullOrWhiteSpace($script:config.alerts.suppressionRegex)) { 547 | $batchMessage += "$message`n" 548 | } else { 549 | Write-Host Alert [$message] suppressed due to matching suppressionRegex -ForegroundColor DarkRed 550 | } 551 | } 552 | if (![string]::IsNullOrEmpty($batchMessage)) { 553 | $alertSystem.Send($batchMessage) 554 | } 555 | } 556 | else { 557 | foreach ($message in $this.alerts) { 558 | if ($message -notmatch $script:config.alerts.suppressionRegex -Or [String]::IsNullOrWhiteSpace($script:config.alerts.suppressionRegex)) { 559 | $alertSystem.Send($message) 560 | } else { 561 | Write-Host Alert [$message] suppressed due to matching suppressionRegex -ForegroundColor DarkRed 562 | } 563 | } 564 | } 565 | } 566 | $this.alerts.Clear() 567 | } 568 | } 569 | 570 | # ----------------------------------------------------------------- 571 | # Credentials Utility 572 | # ----------------------------------------------------------------- 573 | Class CredentialsUtil { 574 | 575 | # Returns a PSCredential object from given plaintext username/password 576 | [PSCredential] GetPSCredential ([String] $username, [String] $plainTextPassword) { 577 | [SecureString] $secPassword = ConvertTo-SecureString $plainTextPassword -AsPlainText -Force 578 | return New-Object System.Management.Automation.PSCredential ($username, $secPassword) 579 | } 580 | } 581 | 582 | # ----------------------------------------------------------------- 583 | # DateTime Utility 584 | # ----------------------------------------------------------------- 585 | Class DateTimeUtil { 586 | 587 | # Gets timestamp in UTC in configured format 588 | [String] NowUTCFormatted() { 589 | return $this.Format($this.NowUTC()) 590 | } 591 | 592 | # Gets timestamp in UTC 593 | [DateTime] NowUTC() { 594 | return (Get-Date).ToUniversalTime() 595 | } 596 | 597 | # Converts to UTC and formats 598 | [String] ToUTCAndFormat([DateTime] $dateTime) { 599 | return $this.Format($dateTime.ToUniversalTime()) 600 | } 601 | 602 | # Formats time based on configured format 603 | [String] Format([DateTime] $dateTime) { 604 | return $dateTime.ToString($script:config.monitor.timeFormat) 605 | } 606 | 607 | } 608 | 609 | # ----------------------------------------------------------------- 610 | # Simple fixed-size list 611 | # ----------------------------------------------------------------- 612 | Class FixedSizeList { 613 | 614 | hidden $data 615 | hidden [int] $size 616 | 617 | # Constructs a fixed size list 618 | # This is a simple implementation based on a LinkedList :) 619 | # Until CX requirements dictate a more complex impl, this'll do nicely. 620 | FixedSizeList ([int] $size) { 621 | $this.size = $size 622 | $this.data = New-Object Collections.Generic.LinkedList[Object] 623 | } 624 | 625 | # Add data item to the list 626 | Add([Object] $item) { 627 | # Maintain a max of {size} items 628 | if ($this.data.Count -eq $this.size) { 629 | $this.data.RemoveLast() 630 | } 631 | $this.data.AddFirst($item) 632 | } 633 | 634 | # Get internal data 635 | # Tsk,tsk.. 636 | [Array] GetData() { 637 | return $this.data 638 | } 639 | } 640 | 641 | # ----------------------------------------------------------------- 642 | # Abstract Scan Time Estimation algo 643 | # ----------------------------------------------------------------- 644 | Class ScanTimeAlgo { 645 | 646 | # Margin (%) to add to scan duration threshold 647 | [double] $thresholdMargin 648 | 649 | # Abstract constructor 650 | ScanTimeAlgo () { 651 | $type = $this.GetType() 652 | if ($type -eq [ScanTimeAlgo]) { 653 | throw("Class $type must be implemented") 654 | } 655 | } 656 | 657 | # Calculates expected scan duration 658 | [double] Estimate ([Object] $scan) { 659 | # Force implementation by a concrete algo 660 | throw("Method is abstract. Needs to be overriden by implementation.") 661 | } 662 | 663 | # Calculates elapsed time for a scan (in minutes) 664 | [double] GetScanDuration ([Object] $scan) { 665 | 666 | [double] $elapsedTime = 0.0 667 | [String] $scanStart = $scan.engineStartedOn 668 | if ($scanStart) { 669 | [String] $scanEnd = $scan.completedOn 670 | 671 | # Calculate scan duration 672 | $startTime = [Xml.XmlConvert]::ToDateTime($scanStart) 673 | if (!$scanEnd) { 674 | $scanEnd = Get-Date 675 | if ($script:config.monitor.useUTCTimeOnClient -eq "true") { 676 | $scanEnd = (Get-Date).ToUniversalTime() 677 | } 678 | } 679 | $diff = New-TimeSpan -Start $startTime -End $scanEnd 680 | $elapsedTime = $diff.TotalMinutes 681 | } 682 | 683 | return $elapsedTime 684 | } 685 | } 686 | 687 | # ----------------------------------------------------------------- 688 | # Default Scan Time Estimation algo 689 | # ----------------------------------------------------------------- 690 | Class DefaultScanTimeAlgo : ScanTimeAlgo { 691 | 692 | hidden [IO] $io 693 | hidden [Hashtable] $scanHistory 694 | # Default scan rate: LOC / hour 695 | hidden [int] $scanRateLOCPerHour 696 | 697 | DefaultScanTimeAlgo () { 698 | $this.io = [IO]::new() 699 | $this.scanHistory = [Hashtable]::new() 700 | # Sets margin for threshold 701 | # Threshold is calculated as (scan time + margin %) 702 | $this.thresholdMargin = $script:config.monitor.thresholds.scanDurationThresholdMarginPercent 703 | $this.scanRateLOCPerHour = $script:config.monitor.thresholds.scanRateAsLOCPerHour 704 | } 705 | 706 | # Default scan time estimation algo implementation 707 | # Simply maintain last scan duration and use (that+%margin) as benchmark 708 | [double] Estimate ([Object] $scan) { 709 | 710 | [double] $scanDuration = 0.0 711 | 712 | # If we have prior scans from this project 713 | [String] $key = $this.GetKey($scan) 714 | 715 | if ($this.scanHistory.Count -gt 0 -and $this.scanHistory.containsKey($key)) { 716 | 717 | # Fetch previously completed scan 718 | [Object] $priorScan = $this.scanHistory[$key] 719 | 720 | $scanDuration = $this.GetScanDuration($priorScan) 721 | } 722 | else { 723 | # Simple formula : LOC / scan rate and converted to minutes 724 | $scanDuration = ($scan.loc / $this.scanRateLOCPerHour) * 60.0 725 | # $this.io.Console("Based on simple calculation: $($scan.loc) / $($this.scanRateLOCPerHour) * 60 = $scanDuration") 726 | } 727 | 728 | # Margin is a percentage added on top of expected scan duration 729 | $margin = ($scanDuration * ($this.thresholdMargin / 100.0)) 730 | return $scanDuration + $margin 731 | } 732 | 733 | # Returns a machine readable Scan key 734 | [String] GetKey([Object] $scan) { 735 | [String] $scanType = if ($scan.isIncremental -eq $True) { "I" } else { "F" } 736 | return "$($scan.id)_$($scan.project.Name)_$scanType" 737 | } 738 | 739 | # Saves a finished scan's duration to scan history 740 | StoreScanDuration ($scan) { 741 | # Guard for case when engineStartedOn is not available which 742 | # happens when no source code changes were detected. 743 | if ([string]::IsNullOrEmpty($scan.engineStartedOn)) { 744 | return 745 | } 746 | 747 | [String] $key = $this.GetKey($scan) 748 | 749 | # Add scan if no prior scans exist for given key 750 | if (!$this.scanHistory.containsKey($key)) { 751 | # TODO: This table will need to be flushed either on a timely basis, or some other criteria 752 | # Store data only if the scan actually was underway 753 | if ($scan.engineStartedOn) { 754 | $this.scanHistory.Add($key, $scan) 755 | } 756 | } 757 | else { 758 | $priorScan = $this.scanHistory[$key] 759 | 760 | [DateTime] $priorScanStart = [Xml.XmlConvert]::ToDateTime($priorScan.engineStartedOn) 761 | [DateTime] $currentScanStart = [Xml.XmlConvert]::ToDateTime($scan.engineStartedOn) 762 | 763 | # Replace old scan with new scan if scan is newer 764 | if ($priorScan.id -ne $scan.id -and $currentScanStart -gt $priorScanStart) { 765 | # $this.io.Console("Replacing prior scan. Old scanId $($priorScan.id) Current ScanId $($scan.id)") 766 | $this.scanHistory[$key] = $scan 767 | } 768 | } 769 | } 770 | } 771 | 772 | # ----------------------------------------------------------------- 773 | # Reads Configuration from JSON file 774 | # ----------------------------------------------------------------- 775 | Class Config { 776 | 777 | hidden [IO] $io 778 | hidden $config 779 | static [String] $CONFIG_FILE = ".\cx_health_mon_config.json" 780 | 781 | # Constructs and loads configuration from given path 782 | Config () { 783 | $this.io = [IO]::new() 784 | $this.LoadConfig() 785 | } 786 | 787 | # Loads configuration from configured path 788 | LoadConfig () { 789 | try { 790 | $cp = [Config]::CONFIG_FILE 791 | $configFilePath = (Get-Item -Path $cp).FullName 792 | $this.io.Log("Loading configuration from $configFilePath") 793 | $this.config = Get-Content -Path $configFilePath -Raw | ConvertFrom-Json 794 | } 795 | catch { 796 | $this.io.Log("Provided configuration file at [" + [Config]::CONFIG_FILE + "] is missing / corrupt.") 797 | } 798 | } 799 | 800 | [PsCustomObject] GetConfig() { 801 | return $this.config 802 | } 803 | } 804 | 805 | # ----------------------------------------------------------------- 806 | # REST request methods 807 | # ----------------------------------------------------------------- 808 | Enum RESTMethod { 809 | GET 810 | POST 811 | } 812 | 813 | # ----------------------------------------------------------------- 814 | # REST request body 815 | # ----------------------------------------------------------------- 816 | Class RESTBody { 817 | 818 | [String] $grantType 819 | [String] $scope 820 | [String] $clientId 821 | [String] $clientSecret 822 | 823 | RESTBody( 824 | [String] $grantType, 825 | [String] $scope, 826 | [String] $clientId, 827 | [String] $clientSecret 828 | ) { 829 | $this.grantType = $grantType 830 | $this.scope = $scope 831 | $this.clientId = $clientId 832 | $this.clientSecret = $clientSecret 833 | } 834 | } 835 | 836 | # ----------------------------------------------------------------- 837 | # REST Client 838 | # ----------------------------------------------------------------- 839 | Class RESTClient { 840 | 841 | [String] $baseUrl 842 | [RESTBody] $restBody 843 | 844 | hidden [String] $token 845 | hidden [IO] $io = [IO]::new() 846 | 847 | # Constructs a RESTClient based on given base URL and body 848 | RESTClient ([String] $cxHost, [RESTBody] $restBody) { 849 | $this.baseUrl = $cxHost + "/cxrestapi" 850 | $this.restBody = $restBody 851 | } 852 | <# 853 | # returns CxSAST version 854 | #> 855 | [String] version () { 856 | try { 857 | $versionUrl = $this.baseUrl + "/system/version" 858 | $response = Invoke-RestMethod -uri $versionUrl -method GET -TimeoutSec $script:config.monitor.apiResponseTimeoutSeconds 859 | $cxVersion = $response.version + " HF" + $response.hotFix 860 | } 861 | catch { 862 | $this.io.Log("Could not retrieve version CxSAST. Most probably using a version below 9.0. Reason: HTTP [$($_.Exception.Response.StatusCode.value__)] - $($_.Exception.Response.StatusDescription).") 863 | $cxVersion = "below 9.0 version" 864 | } 865 | return $cxVersion 866 | } 867 | 868 | <# 869 | # Logins to the CxSAST REST API 870 | # and returns an API token 871 | #> 872 | [bool] login ([String] $username, [String] $password) { 873 | [bool] $isLoginSuccessful = $False 874 | $body = @{ 875 | username = $username 876 | password = $password 877 | grant_type = $this.restBody.grantType 878 | scope = $this.restBody.scope 879 | client_id = $this.restBody.clientId 880 | client_secret = $this.restBody.clientSecret 881 | } 882 | 883 | [psobject] $response = $null 884 | try { 885 | $loginUrl = $this.baseUrl + "/auth/identity/connect/token" 886 | $response = Invoke-RestMethod -uri $loginUrl -method POST -body $body -contenttype 'application/x-www-form-urlencoded' -TimeoutSec $script:config.monitor.apiResponseTimeoutSeconds 887 | } catch { 888 | $this.io.Log("Could not authenticate against Checkmarx REST API. Reason: HTTP [$($_.Exception.Response.StatusCode.value__)] - $($_.Exception.Response.StatusDescription).") 889 | } 890 | 891 | if ($response -and $response.access_token) { 892 | $isLoginSuccessful = $True 893 | # Track token internally 894 | $this.token = $response.token_type + " " + $response.access_token 895 | } 896 | 897 | 898 | return $isLoginSuccessful 899 | } 900 | 901 | <# 902 | # Invokes a given REST API 903 | #> 904 | [Object] invokeAPI ([String] $requestUri, [RESTMethod] $method, [Object] $body, [int] $apiResponseTimeoutSeconds) { 905 | 906 | # Sanity : If not logged in, do not proceed 907 | if ( ! $this.token) { 908 | throw "Must execute login() first, prior to other API calls." 909 | } 910 | 911 | $headers = @{ 912 | "Authorization" = $this.token 913 | "Accept" = "application/json" 914 | } 915 | 916 | $response = $null 917 | 918 | try { 919 | $uri = $this.baseUrl + $requestUri 920 | if ($method -ieq "GET") { 921 | $response = Invoke-RestMethod -Uri $uri -Method $method -Headers $headers -TimeoutSec $apiResponseTimeoutSeconds 922 | } 923 | else { 924 | $response = Invoke-RestMethod -Uri $uri -Method $method.ToString() -Headers $headers -Body $body -TimeoutSec $apiResponseTimeoutSeconds 925 | } 926 | 927 | Write-Debug "ID: $($response.id)" 928 | Write-Debug "Key: $($response.key)" 929 | Write-Debug "Self: $($response.self)" 930 | } 931 | catch { 932 | $exception = $_.Exception 933 | 934 | $this.io.Log("REST API call failed : [$($exception.Message)]") 935 | $this.io.Log("Status Code: $($exception.Response.StatusCode)") 936 | 937 | if ($exception.Response.StatusCode -eq "BadRequest") { 938 | $respstream = $exception.Response.GetResponseStream() 939 | $sr = new-object System.IO.StreamReader $respstream 940 | $ErrorResult = $sr.ReadToEnd() 941 | $this.io.Log($ErrorResult) 942 | } 943 | } 944 | 945 | return $response 946 | } 947 | } 948 | 949 | 950 | # ----------------------------------------------------------------- 951 | # Database Client 952 | # ----------------------------------------------------------------- 953 | Class DBClient { 954 | 955 | hidden [IO] $io = [IO]::new() 956 | hidden [PSCredential] $sqlAuthCreds 957 | hidden [String] $serverInstance 958 | 959 | # Constructs a DBClient based on given server and creds 960 | DBClient ([String] $serverInstance, [String]$dbUser, [String] $dbPass) { 961 | $this.serverInstance = $serverInstance 962 | if ($dbUser -and $dbPass) { 963 | $this.sqlAuthCreds = [CredentialsUtil]::new().GetPSCredential($dbUser, $dbPass) 964 | } 965 | } 966 | 967 | # Executes given SQL using either SQLServer authentication or Windows, depending on given PSCredential object 968 | [PSObject] ExecSQL ([String] $sql, [Hashtable] $parameters) { 969 | # $this.io.Console("Executing $sql") 970 | try { 971 | if ($this.sqlAuthCreds.UserName) { 972 | $cred = $this.sqlAuthCreds 973 | return Invoke-Sqlcmd2 -ServerInstance $this.serverInstance -Credential @cred -Query $sql -SqlParameters $parameters 974 | } 975 | else { 976 | return Invoke-Sqlcmd2 -ServerInstance $this.serverInstance -Query $sql -SqlParameters $parameters 977 | } 978 | } 979 | catch { 980 | $this.io.Log("Database execution error. $($_.Exception.GetType().FullName), $($_.Exception.Message)") 981 | # Force exit during dev run - runtime savior 982 | Exit 983 | } 984 | } 985 | 986 | } 987 | 988 | 989 | 990 | # ----------------------------------------------------------------- 991 | # Engine(s) Monitor 992 | # ----------------------------------------------------------------- 993 | Class EngineMonitor { 994 | 995 | hidden [IO] $io 996 | hidden [AlertService] $alertService 997 | hidden [RESTClient] $cxSastRestClient 998 | hidden [DateTimeUtil] $dateUtil 999 | hidden [String] $cxVersion 1000 | 1001 | # Constructs a EngineMonitor 1002 | EngineMonitor ([AlertService] $alertService) { 1003 | $this.io = [IO]::new() 1004 | $this.dateUtil = [DateTimeUtil]::new() 1005 | $this.alertService = $alertService 1006 | $this.cxVersion = "Unknown" 1007 | } 1008 | 1009 | Monitor() { 1010 | # Create a RESTBody specific to CxSAST REST API calls 1011 | $cxSastRestBody = [RESTBody]::new($script:CX_REST_GRANT_TYPE, $script:CX_REST_SCOPE, $script:CX_REST_CLIENT_ID, $script:CX_REST_CLIENT_SECRET) 1012 | # Create a REST Client for CxSAST REST API 1013 | $this.cxSastRestClient = [RESTClient]::new($script:config.cx.host, $cxSastRestBody) 1014 | # Get Version of CxSAST server 1015 | $this.cxVersion = $this.cxSastRestClient.version() 1016 | # Login to the CxSAST server 1017 | [bool] $isLoginOk = $this.cxSastRestClient.login($script:cxUsername, $script:cxPassword) 1018 | 1019 | if ($isLoginOk -eq $True) { 1020 | # Fetch Queue Status 1021 | $resp = $this.GetEngineStatus() 1022 | 1023 | # Process the response 1024 | $this.ProcessResponse($resp) 1025 | } 1026 | } 1027 | 1028 | # Fetches the status of the engines 1029 | [Object] GetEngineStatus () { 1030 | [String] $apiUrl = "/sast/engineServers" 1031 | $resp = $this.cxSastRestClient.invokeAPI($apiUrl, [RESTMethod]::GET, $null, $script:config.monitor.apiResponseTimeoutSeconds) 1032 | return $resp 1033 | } 1034 | 1035 | # Try to reach the specific engine's WSDL 1036 | [Object] GetEngineWSDL ([String] $name, [String]$apiUri) { 1037 | [Object] $resp = $null 1038 | for ($i = 0; $i -lt $script:config.monitor.retries; $i++) { 1039 | try { 1040 | if($this.cxVersion.StartsWith("9.3")){ 1041 | $resp = Invoke-WebRequest -UseBasicParsing -Uri "${apiUri}/swagger/index.html" -TimeoutSec $script:config.monitor.apiResponseTimeoutSeconds 1042 | break 1043 | } else{ 1044 | $resp = Invoke-WebRequest -UseBasicParsing -Uri $apiUri -TimeoutSec $script:config.monitor.apiResponseTimeoutSeconds 1045 | break 1046 | } 1047 | } catch { 1048 | $resp = $_.Exception.Response 1049 | $this.io.Log("ERROR: Checking engine $name - $apiUri : [$($_.Exception.Message)]") 1050 | if ($i -lt $script:config.mnitor.retries) { 1051 | $this.io.Log("Attempting again...") 1052 | } 1053 | } 1054 | } 1055 | return $resp 1056 | } 1057 | 1058 | # Processes response from CxSAST REST API call 1059 | ProcessResponse ([Object] $apiResp) { 1060 | 1061 | # If there are registered engines 1062 | if ($apiResp.Count -gt 0) { 1063 | 1064 | foreach ($engine in $apiResp) { 1065 | 1066 | [String] $engineInfo = "Engine [$($engine.id), $($engine.name), $($engine.status.value)]" 1067 | 1068 | [PSCustomObject] $engineDetails = $this.GetEngineDetails($engine) 1069 | 1070 | # Try connecting if not offline 1071 | if ($engine.status.value -eq "Offline") { 1072 | $this.alertService.AddAlert([AlertType]::ENGINE_OFFLINE, $engineInfo, $engineInfo) 1073 | $this.io.WriteJSON([AlertType]::ENGINE_OFFLINE, $engineDetails) 1074 | } 1075 | else { 1076 | # Attempt to access the Engine's WSDL and report response time 1077 | $stopwatch = [system.diagnostics.stopwatch]::StartNew() 1078 | $wsdlResp = $this.GetEngineWSDL($engine.name, $engine.uri) 1079 | $stopwatch.Stop() 1080 | 1081 | $this.io.Log($engineInfo + " Responded in [$($stopwatch.elapsed.TotalSeconds)] seconds.") 1082 | 1083 | # Log any response other than HTTP 200 OK 1084 | if ($wsdlResp.StatusCode -ne [system.net.httpstatuscode]::OK) { 1085 | $message = $engineInfo + " Responded [$($wsdlResp.StatusCode)] instead of [OK]" 1086 | $this.io.LogEvent($message) 1087 | $this.alertService.AddAlert([AlertType]::ENGINE_ERROR, $message, $engineInfo) 1088 | } 1089 | 1090 | # Check if engine is idle 1091 | if ($engine.status.value -eq "Idle") { 1092 | 1093 | # Write idle engines JSON 1094 | $this.io.WriteJSON([AlertType]::ENGINE_IDLE, $engineDetails) 1095 | 1096 | # Check if this idle engine could have taken on existing queued scans 1097 | $this.CheckQueuedScansForMatch($engine) 1098 | } 1099 | 1100 | # Check API call elapsed time 1101 | [TimeSpan] $threshold = [TimeSpan]::FromSeconds($script:config.monitor.thresholds.engineResponseThresholdSeconds) 1102 | if ($stopwatch.elapsed.TotalSeconds -gt $script:config.monitor.thresholds.engineResponseThresholdSeconds) { 1103 | $message = $engineInfo + " Response Time: [$($stopWatch.elapsed)]. Threshold: [$threshold]." 1104 | $this.io.LogEvent($message) 1105 | $this.alertService.AddAlert([AlertType]::ENGINE_RESPONSE_SLOW, $message, $engineInfo) 1106 | 1107 | # Create a JSON structure for the slow engine 1108 | $engineDetails.Add("ResponseTime", "$($stopwatch.elapsed.Milliseconds)") 1109 | $engineDetails.Add("Threshold", "$($script:config.monitor.thresholds.engineResponseThresholdSeconds * 1000)") 1110 | $this.io.WriteJSON([AlertType]::ENGINE_RESPONSE_SLOW, $engineDetails) 1111 | } 1112 | } 1113 | } 1114 | 1115 | # Send out alerts 1116 | $this.alertService.Send() 1117 | } 1118 | 1119 | } 1120 | 1121 | # Checks queued scans against given idle engine to see if it 'fits' 1122 | CheckQueuedScansForMatch ([PSObject] $engine) { 1123 | 1124 | # Check if there are idle engines that could have executed one of the queued scans 1125 | if ($script:queuedEntriesDto) { 1126 | 1127 | [System.Collections.ArrayList] $matchedScans = @() 1128 | 1129 | # attempt to find scans that could 'fit' 1130 | foreach ($queuedScan in $script:queuedEntriesDto) { 1131 | 1132 | # Applies only to 'Queued' scans - not to ones that are in SourcePulling etc. 1133 | if ($queuedScan.stage.value -eq "Queued") { 1134 | 1135 | # IdleEngineMinLOC <= QueuedScanLOC <= IdleEngineMaxLOC 1136 | if ($engine.ScanMinLoc -le $queuedScan.loc -and $queuedScan.loc -le $engine.ScanMaxLoc) { 1137 | # This algo is very eager, 1138 | # in that the engine could have just published 'idle' and we pick it up :) 1139 | $matchedScans.Add($queuedScan) 1140 | } 1141 | } 1142 | } 1143 | 1144 | # Publish alert only if we found scans within an idle engine's parameters 1145 | # NOTE: TODO: we're not looking at the concurrent capacity of the engine 1146 | if ($matchedScans.Count -gt 0) { 1147 | [String] $engineInfo = "Engine [$($engine.id), $($engine.name), $($engine.status.value)]" 1148 | [String] $message = "Idle engine [" + $engine.name + "]. Potential scans: [" + $matchedScans.id + "]"; 1149 | $this.alertService.AddAlert([AlertType]::ENGINE_IDLE, $message, $engineInfo) 1150 | } 1151 | } 1152 | } 1153 | 1154 | # Maps engine details into a Hashtable 1155 | [PSCustomObject] GetEngineDetails ($engine) { 1156 | return [ordered] @{ 1157 | EventDate = "" 1158 | EngineId = "$($engine.id)" 1159 | EngineServerName = "$($engine.name)" 1160 | ScanMinLoc = "$($engine.minLoc)" 1161 | ScanMaxLoc = "$($engine.maxLoc)" 1162 | MaxConcurrentScans = "$($engine.maxScans)" 1163 | ProductVersion = "$($engine.cxVersion)" 1164 | StatusId = "$($engine.status.id)" 1165 | StatusName = "$($engine.status.value)" 1166 | } 1167 | } 1168 | } 1169 | 1170 | 1171 | # ----------------------------------------------------------------- 1172 | # Database table metadata 1173 | # 1174 | # The AuditMonitor executes SQL based on metadata of 1175 | # the database item that needs to be audited/monitored, 1176 | # to generate the alert message. 1177 | # 1178 | # The DBQueryMetadata contains the SQL query to execute, and 1179 | # a map of [reportable Labels to DB Columns]. 1180 | # Example: ["User that changed the value" : "updating_user"] 1181 | # ----------------------------------------------------------------- 1182 | Class DBQueryMetadata { 1183 | [String] $name 1184 | [String] $sql 1185 | # Notice the hashtable is ordered. 1186 | # This enables control over the order of the labels 1187 | # when we auto-generate the alert message. 1188 | # See AuditMonitor's PopulateAuditMetadata() method 1189 | # for the order of the labels. 1190 | $meta = [ordered] @{ } 1191 | } 1192 | 1193 | 1194 | 1195 | # ----------------------------------------------------------------- 1196 | # Audit(s) Monitor 1197 | # ----------------------------------------------------------------- 1198 | Class AuditMonitor { 1199 | 1200 | hidden [IO] $io 1201 | hidden [AlertService] $alertService 1202 | hidden [DateTime] $lastRun 1203 | hidden [DBClient] $dbClient 1204 | hidden [System.Collections.ArrayList] $dbQueryMetadata = @() 1205 | hidden [DateTimeUtil] $dateUtil 1206 | 1207 | # Constructs an AuditMonitor 1208 | AuditMonitor ([AlertService] $alertService) { 1209 | $this.io = [IO]::new() 1210 | $this.dateUtil = [DateTimeUtil]::new() 1211 | $this.alertService = $alertService 1212 | $this.lastRun = Get-Date 1213 | $this.PopulateAuditMetadata() 1214 | } 1215 | 1216 | # Inject suitable DB client 1217 | SetDbClient ([DBClient] $dbClient) { 1218 | $this.dbClient = $dbClient 1219 | } 1220 | 1221 | # Add metadata for database queries 1222 | AddDBQueryMetadata ([DBQueryMetadata] $metadata) { 1223 | $this.dbQueryMetadata.Add($metadata) 1224 | } 1225 | 1226 | # Monitor for changes in audit tables 1227 | Monitor() { 1228 | 1229 | # Big picture: 1230 | # For each database item that needs to be monitored, 1231 | # we run the specified SQL query 1232 | # and extract the columns specified in the meta map 1233 | # and generate an alert message string that is published to the alerting service 1234 | foreach ($metadata in $this.dbQueryMetadata) { 1235 | 1236 | # All the queries depend on a timestamp column. 1237 | # We only process new database entries since the last time the monitor was run. 1238 | # Ex. We only process/send alerts for new preset changes (since the last run). 1239 | [Hashtable] $parameters = @{ lastRun = $this.lastRun } 1240 | 1241 | [PSObject] $results = $this.dbClient.ExecSQL($metadata.sql, $parameters) 1242 | 1243 | if ($results) { 1244 | 1245 | foreach ($result in $results) { 1246 | 1247 | # Start constructing the alert message 1248 | [String] $message = "[$($metadata.name)] : " 1249 | 1250 | # Extract the columns that are specified in the meta map 1251 | [int] $i = 0 1252 | foreach ($key in $metadata.meta.Keys) { 1253 | 1254 | $columnName = $metadata.meta[$key] 1255 | $value = $result[$columnName] 1256 | $message += "$key = [$value] " 1257 | 1258 | # Append a comma if it's not the last item 1259 | if ($i + 1 -lt $metadata.meta.Keys.Count) { 1260 | $message += ", " 1261 | $i++ 1262 | } 1263 | } 1264 | # Provide new guid to uniquely identify each audit alert 1265 | # Otherwise the alerting service will assume it is the same alert and may not send it 1266 | $this.alertService.AddAlert([AlertType]::AUDIT, $message, [Guid]::NewGuid()) 1267 | } 1268 | # Send out the alert messages 1269 | $this.alertService.Send() 1270 | } 1271 | } 1272 | # Update the last run marker 1273 | $this.lastRun = Get-Date 1274 | } 1275 | 1276 | # Create database query metadata for the items that need to be monitored 1277 | # and add them to the monitor 1278 | PopulateAuditMetadata() { 1279 | 1280 | # Too bad we don't have a REST API to get these audit entries :( 1281 | 1282 | # Audit_Projects 1283 | [DBQueryMetadata] $auditProject = [DBQueryMetadata]::new() 1284 | $auditProject.name = "Project" 1285 | $auditProject.sql = "select * from CxActivity.dbo.Audit_Projects where [TimeStamp] >= @lastRun" 1286 | $auditProject.meta["Action"] = "Event" 1287 | $auditProject.meta["Name"] = "ProjectName" 1288 | $auditProject.meta["User"] = "OwnerName" 1289 | $auditProject.meta["Timestamp"] = "TimeStamp" 1290 | $this.AddDBQueryMetadata($auditProject) 1291 | 1292 | # Audit_Presets 1293 | [DBQueryMetadata] $auditPresets = [DBQueryMetadata]::new() 1294 | $auditPresets.name = "Preset" 1295 | $auditPresets.sql = "select * from CxActivity.dbo.Audit_Presets where [TimeStamp] >= @lastRun" 1296 | $auditPresets.meta["Action"] = "Event" 1297 | $auditPresets.meta["Name"] = "PresetName" 1298 | $auditPresets.meta["User"] = "OwnerName" 1299 | $auditPresets.meta["Timestamp"] = "TimeStamp" 1300 | $this.AddDBQueryMetadata($auditPresets) 1301 | 1302 | # Audit_Queries 1303 | [DBQueryMetadata] $auditQueries = [DBQueryMetadata]::new() 1304 | $auditQueries.name = "Query" 1305 | $auditQueries.sql = "select * from CxActivity.dbo.Audit_Queries where [TimeStamp] >= @lastRun" 1306 | $auditQueries.meta["Action"] = "Event" 1307 | $auditQueries.meta["Name"] = "Name" 1308 | $auditQueries.meta["User"] = "OwnerName" 1309 | $auditQueries.meta["Timestamp"] = "TimeStamp" 1310 | $this.AddDBQueryMetadata($auditQueries) 1311 | 1312 | # Audit Results 1313 | [DBQueryMetadata] $auditResults = [DBQueryMetadata]::new() 1314 | $auditResults.name = "Results" 1315 | # Modifed version of the CxDB.dbo.[GetAllLabelsForScanByProject] stored procedure 1316 | $auditResults.sql = 1317 | "SELECT DISTINCT 1318 | labels.[StringData] AS Action, 1319 | labels.[UpdateDate] AS [Timestamp], 1320 | labels.[UpdatingUser] AS Username, 1321 | projects.[Name] AS ProjectName, 1322 | queryVersion.[Name] As QueryName, 1323 | nodeResults.File_Name AS [File], 1324 | nodeResults.Line, 1325 | nodeResults.Col AS [Column] 1326 | FROM 1327 | CxDB.dbo.ResultsLabels labels 1328 | INNER JOIN CxDB.dbo.Projects projects ON labels.[ProjectId] = projects.[Id] 1329 | INNER JOIN CxDB.dbo.PathResults scanPaths ON scanPaths.[Similarity_Hash] = labels.[SimilarityId] 1330 | INNER JOIN CxDB.dbo.QueryVersion queryVersion ON scanPaths.QueryVersionCode = queryVersion.QueryVersionCode 1331 | INNER JOIN CxDB.dbo.NodeResults nodeResults ON nodeResults.[ResultId] = labels.[ResultId] AND nodeResults.Path_Id = labels.PathID AND nodeResults.Node_Id = 1 1332 | LEFT JOIN ( 1333 | SELECT QueryVersion.QueryVersionCode FROM CxDB.dbo.QueryVersion INNER JOIN CxDB.dbo.QueryGroup ON QueryVersion.PackageId = QueryGroup.PackageId) QueryIDs2 1334 | ON QueryIDs2.QueryVersionCode = scanPaths.QueryVersionCode 1335 | WHERE labels.LabelType=1 and labels.UpdateDate >= @lastRun" 1336 | 1337 | $auditResults.meta.Add("Action", "Action") 1338 | $auditResults.meta.Add("Query", "QueryName") 1339 | $auditResults.meta.Add("Project", "ProjectName") 1340 | $auditResults.meta.Add("File", "File") 1341 | $auditResults.meta.Add("Line", "Line") 1342 | $auditResults.meta.Add("Column", "Column") 1343 | $auditResults.meta.Add("User", "Username") 1344 | $auditResults.meta.Add("Timestamp", "Timestamp") 1345 | $this.AddDBQueryMetadata($auditResults) 1346 | } 1347 | } 1348 | 1349 | 1350 | # ----------------------------------------------------------------- 1351 | # Queue Monitor 1352 | # ----------------------------------------------------------------- 1353 | Class QueueMonitor { 1354 | 1355 | hidden [IO] $io 1356 | # Threshold for number of scans in queued state 1357 | hidden [int] $queuedScansThreshold 1358 | # Threshold for time spent in queued state (minutes) 1359 | hidden [TimeSpan] $queuedTimeThreshold 1360 | # Algo that estimates scan duration 1361 | hidden [ScanTimeAlgo] $scanTimeAlgo 1362 | hidden [AlertService] $alertService 1363 | hidden [RESTClient] $cxSastRestClient 1364 | hidden [DateTimeUtil] $dateUtil 1365 | 1366 | # CxSAST 8.9 Scan Stages 1367 | # New, PreScan, Queued, Scanning, PostScan, Finished, Canceled, Failed, SourcePullingAndDeployment, None 1368 | 1369 | # Prior-to-Running states 1370 | static [String[]] $queuedStates = @("New", "Queued", "SourcePullingAndDeployment", "PreScan") 1371 | 1372 | # Running States 1373 | static [String[]] $runningStates = @("Scanning", "PostScan") 1374 | 1375 | # Failed States 1376 | static [String[]] $failedStates = @("Failed") 1377 | 1378 | # Finished States 1379 | static [String[]] $finishedStates = @("Canceled", "Deleted", "Finished") 1380 | 1381 | # Constructs a QueueMonitor 1382 | QueueMonitor ([ScanTimeAlgo] $scanTimeAlgo, [AlertService] $alertService) { 1383 | $this.io = [IO]::new() 1384 | $this.dateUtil = [DateTimeUtil]::new() 1385 | $this.queuedScansThreshold = $script:config.monitor.thresholds.queuedScansThreshold 1386 | $this.queuedTimeThreshold = [TimeSpan]::FromMinutes($script:config.monitor.thresholds.queuedTimeThresholdMinutes) 1387 | $this.scanTimeAlgo = $scanTimeAlgo 1388 | $this.alertService = $alertService 1389 | } 1390 | 1391 | # Check on the CxSAST queue 1392 | Monitor() { 1393 | # Create a RESTBody specific to CxSAST REST API calls 1394 | $cxSastRestBody = [RESTBody]::new($script:CX_REST_GRANT_TYPE, $script:CX_REST_SCOPE, $script:CX_REST_CLIENT_ID, $script:CX_REST_CLIENT_SECRET) 1395 | # Create a REST Client for CxSAST REST API 1396 | $this.cxSastRestClient = [RESTClient]::new($script:config.cx.host, $cxSastRestBody) 1397 | 1398 | # Login to the CxSAST server 1399 | [bool] $isLoginOk = $this.cxSastRestClient.login($script:cxUsername, $script:cxPassword) 1400 | 1401 | if ($isLoginOk -eq $True) { 1402 | # Fetch Queue Status 1403 | $qStatusResp = $this.GetQueueStatus() 1404 | 1405 | # Process the response 1406 | $this.ProcessResponse($qStatusResp) 1407 | 1408 | # Check Portal responsiveness 1409 | $this.CheckPortalResponsiveness() 1410 | } 1411 | 1412 | } 1413 | 1414 | # Check for Portal responsiveness 1415 | CheckPortalResponsiveness() { 1416 | # Check portal responsiveness 1417 | $stopwatch = [system.diagnostics.stopwatch]::StartNew() 1418 | $response = $this.GetLoginPage() 1419 | $stopwatch.Stop() 1420 | 1421 | # NOTE: Watch out for the condition where the Invoke-WebRequest -TimeoutSec is smaller than the restResponseThresholdSeconds threshold 1422 | if ($response) { 1423 | if ($response.StatusCode -eq 200) { 1424 | $this.io.Log("Portal is responsive. Portal responded in [$($stopwatch.elapsed.TotalSeconds)] seconds.") 1425 | } 1426 | else { 1427 | $this.io.LogEvent("Portal responded with HTTP [$($response.StatusCode)]") 1428 | } 1429 | } 1430 | 1431 | # Check if the portal response exceeded threshold 1432 | if ($stopwatch.elapsed.TotalSeconds -gt $script:config.monitor.thresholds.restResponseThresholdSeconds) { 1433 | $message = "Slow Portal response. Response Time: [$($stopWatch.elapsed.TotalSeconds)] seconds. Threshold: [$($script:config.monitor.thresholds.restResponseThresholdSeconds)] seconds." 1434 | $this.io.LogEvent($message) 1435 | $this.alertService.AddAlert([AlertType]::PORTAL_SLOW, $message, "") 1436 | } 1437 | } 1438 | 1439 | # Fetches the status of jobs in the CxSAST scan queue 1440 | [Object] GetQueueStatus () { 1441 | [String] $apiUrl = "/sast/scansQueue" 1442 | return $this.cxSastRestClient.invokeAPI($apiUrl, [RESTMethod]::GET, $null, $script:config.monitor.apiResponseTimeoutSeconds) 1443 | } 1444 | 1445 | # Try to fetch the portal login /CxWebClient/Login.aspx page 1446 | # proxy for portal performance 1447 | [Object] GetLoginPage () { 1448 | [Object] $resp = $null 1449 | [String] $pageUrl = $script:config.cx.host + "/CxWebClient/Login.aspx" 1450 | try { 1451 | $resp = Invoke-WebRequest -UseBasicParsing -Uri $pageUrl -TimeoutSec $script:config.monitor.apiResponseTimeoutSeconds 1452 | } 1453 | catch { 1454 | $resp = $_.Exception.Response 1455 | $this.io.Log("ERROR: [$($_.Exception.Message)]") 1456 | } 1457 | return $resp 1458 | } 1459 | 1460 | # Processes response from CxSAST Queue Status REST API call 1461 | ProcessResponse ([Object] $apiResp) { 1462 | 1463 | # If there are entries in the queue 1464 | if ($apiResp.Count -gt 0) { 1465 | 1466 | # Split entries for processing 1467 | [Object[]] $queuedEntries = $apiResp | Where-Object { [QueueMonitor]::queuedStates -contains $_.stage.value } 1468 | [Object[]] $runningEntries = $apiResp | Where-Object { [QueueMonitor]::runningStates -contains $_.stage.value } 1469 | [Object[]] $failedEntries = $apiResp | Where-Object { [QueueMonitor]::failedStates -contains $_.stage.value } 1470 | [Object[]] $finishedEntries = $apiResp | Where-Object { [QueueMonitor]::finishedStates -contains $_.stage.value } 1471 | 1472 | $this.io.LogEvent("Queued: $($queuedEntries.Count), Running: $($runningEntries.Count), Failed: $($failedEntries.Count), Finished: $($finishedEntries.Count)") 1473 | 1474 | # Save the queuedEntries to cross-check against idle engines later 1475 | $script:queuedEntriesDto = $queuedEntries 1476 | 1477 | # Process finished scans first, so that we can derive thresholds 1478 | $this.ProcessFinishedScans($finishedEntries) 1479 | $this.ProcessQueuedScans($queuedEntries) 1480 | $this.ProcessRunningScans($runningEntries) 1481 | $this.ProcessFailedScans($failedEntries) 1482 | } 1483 | } 1484 | 1485 | # Processes Finished scans. 1486 | # Finished scans are interesting because we can 1487 | # derive a good estimate of the next similar 1488 | # scan's exec time. 1489 | # The ScanTimeAlgo injected into the QueueMonitor 1490 | # will determine how exactly the estimate is 1491 | # calculated. 1492 | ProcessFinishedScans ([Object[]] $finishedScans) { 1493 | 1494 | # If we don't have anything to process, return 1495 | if (!$finishedScans -or $finishedScans.Count -eq 0) { 1496 | return 1497 | } 1498 | 1499 | # Store scan info for next scan duration estimate calculations 1500 | foreach ($scan in $finishedScans) { 1501 | $this.scanTimeAlgo.StoreScanDuration($scan) 1502 | } 1503 | } 1504 | 1505 | # Processes Queued scans. 1506 | # There are two primary areas of interest here: 1507 | # 1. Number of scans in the queue 1508 | # 2. How long scans stay in the queued state 1509 | ProcessQueuedScans ([Object[]] $queuedScans) { 1510 | 1511 | # If we don't have anything to process, return 1512 | if (!$queuedScans -or $queuedScans.Count -eq 0) { 1513 | return 1514 | } 1515 | 1516 | # JSON structures for queue monitor 1517 | [PSCustomObject] $queueScanExcess = $null 1518 | 1519 | # If the number of scans in the queue exceeds 1520 | # a threshold, send out an alert. 1521 | # If an alert has been sent, 1522 | # wait for a configurable number of minutes before checking 1523 | # if the number of queued items has increased since the last time the alert was sent 1524 | # before sending out the next alert 1525 | 1526 | if ($queuedScans -and $queuedScans.Count -gt $this.queuedScansThreshold) { 1527 | [String] $alertMsg = "Queued: [$($queuedScans.Count)]. Threshold: [$($this.queuedScansThreshold)]" 1528 | $this.alertService.AddAlert([AlertType]::QUEUE_SCAN_EXCESS, $alertMsg, "") 1529 | # Create a JSON structure for the excess scans 1530 | [PSCustomObject] $queueScanExcess = [ordered] @{ 1531 | EventDate = "" 1532 | ScansQueued = "$($queuedScans.Count)" 1533 | Threshold = "$($this.queuedScansThreshold)" 1534 | } 1535 | } 1536 | 1537 | # For every queued scan 1538 | [System.Collections.ArrayList] $scanIds = @() 1539 | foreach ($scan in $queuedScans) { 1540 | 1541 | $scanIds.Add($scan.Id) 1542 | 1543 | # Calculate queued time 1544 | [DateTime] $dateCreated = [Xml.XmlConvert]::ToDateTime($scan.dateCreated) 1545 | [String] $queuedDate = "" 1546 | if ($scan.queuedOn) { 1547 | [DateTime] $queuedOn = [Xml.XmlConvert]::ToDateTime($scan.queuedOn) 1548 | $queuedDate = $this.dateUtil.ToUTCAndFormat($queuedOn) 1549 | } 1550 | [DateTime] $now = Get-Date 1551 | [TimeSpan] $queuedTIme = New-TimeSpan -Start $dateCreated -End $now 1552 | 1553 | # If the queued duration exceeds a threshold, send an alert. 1554 | # The threshold is provided as a configurable parameter. 1555 | if ($queuedTIme -gt $this.queuedTimeThreshold) { 1556 | 1557 | [String] $scanInfo = $this.GetScanIdentifierForHumans($scan) 1558 | [String] $scanKey = $this.GetKey($scan) 1559 | [String] $alertMsg = "$scanInfo. Queued: [$queuedTIme]. Threshold: [$($this.queuedTimeThreshold)]" 1560 | $this.alertService.AddAlert([AlertType]::QUEUE_SCAN_TIME_EXCEEDED, $alertMsg, $scanKey) 1561 | 1562 | # Create a JSON structure for scans that exceed threshold for queued state 1563 | [PSCustomObject] $scanDetail = [ordered] @{ 1564 | EventDate = "" 1565 | Threshold = "$($this.queuedTimeThreshold)" 1566 | ScanId = "$($scan.Id)" 1567 | ProjectId = "$($scan.project.id)" 1568 | ProjectName = "$($scan.project.name)" 1569 | Origin = "$($scan.origin)" 1570 | IsPublic = "$($scan.isPublic)" 1571 | IsIncremental = "$($scan.isIncremental)" 1572 | ScanRequestDate = "$($this.dateUtil.ToUTCAndFormat($dateCreated))" 1573 | QueuedDate = "$queuedDate" 1574 | } 1575 | $this.io.WriteJSON([AlertType]::QUEUE_SCAN_TIME_EXCEEDED, $scanDetail) 1576 | } 1577 | } 1578 | 1579 | # If there are excess scans in the queue 1580 | # write out the corresponding JSON file 1581 | if ($queueScanExcess) { 1582 | $queueScanExcess.Add("ScanIDs", $scanIds) 1583 | $this.io.WriteJSON([AlertType]::QUEUE_SCAN_EXCESS, $queueScanExcess) 1584 | } 1585 | 1586 | # Sends out alerts if there are any 1587 | $this.alertService.Send() 1588 | } 1589 | 1590 | # Processes Running scans 1591 | ProcessRunningScans ([Object[]] $runningScans) { 1592 | 1593 | foreach ($scan in $runningScans) { 1594 | 1595 | [double] $estimatedMinutes = $this.scanTimeAlgo.Estimate($scan) 1596 | $estimated = [TimeSpan]::FromMinutes($estimatedMinutes) 1597 | 1598 | # Calculate elapsed time 1599 | [double] $elapsedMinutes = $this.scanTimeAlgo.GetScanDuration($scan) 1600 | $elapsed = [TimeSpan]::FromMinutes($elapsedMinutes) 1601 | 1602 | [String] $scanInfo = $this.GetScanIdentifierForHumans($scan) + " Elapsed: [$elapsed]. Threshold: duration [$estimated]" 1603 | 1604 | # If the scan duration exceeds estimate 1605 | # mark the scan as 'slow'. 1606 | if ($elapsedMinutes -gt $estimatedMinutes) { 1607 | $scanKey = $this.GetKey($scan) 1608 | $this.alertService.AddAlert([AlertType]::SCAN_SLOW, $scanInfo, $scanKey) 1609 | 1610 | # Create a JSON structure for slow scans that exceed estimated threshold 1611 | [DateTime] $dateCreated = [Xml.XmlConvert]::ToDateTime($scan.dateCreated) 1612 | [PSCustomObject] $scanDetail = [ordered] @{ 1613 | EventDate = "" 1614 | ScanId = "$($scan.id)" 1615 | ProjectId = "$($scan.project.id)" 1616 | ProjectName = "$($scan.project.name)" 1617 | Origin = "$($scan.origin)" 1618 | IsIncremental = "$($scan.isIncremental)" 1619 | ScanRequestDate = "$($this.dateUtil.ToUTCAndFormat($dateCreated))" 1620 | Loc = "$($scan.loc)" 1621 | ScanStatus = "$($scan.stage.value)" 1622 | ScanDurationMilliseconds = "$($elapsedMinutes * 60 * 1000)" 1623 | EstimatedScanDurationMilliseconds = "$($estimatedMinutes * 60 * 1000)" 1624 | ScannedLanguages = $scan.languages 1625 | } 1626 | $this.io.WriteJSON([AlertType]::SCAN_SLOW, $scanDetail) 1627 | } 1628 | else { 1629 | # Log scan data 1630 | $this.io.LogEvent($scanInfo) 1631 | } 1632 | } 1633 | 1634 | # Sends out alerts if there are any 1635 | $this.alertService.Send() 1636 | } 1637 | 1638 | # Processes Failed scans 1639 | # Sends out alerts for failed scans 1640 | ProcessFailedScans ([Object[]] $failedScans) { 1641 | 1642 | [System.Collections.ArrayList] $failedScans = @() 1643 | 1644 | foreach ($scan in $failedScans) { 1645 | $scanInfo = $this.GetScanIdentifierForHumans($scan) 1646 | $scanKey = $this.GetKey($scan) 1647 | $reason = $scan.stageDetails 1648 | if ($reason) { $reason = "(Reason: $reason)" } 1649 | $this.alertService.AddAlert([AlertType]::SCAN_FAILED, "$scanInfo $reason", $scanKey) 1650 | 1651 | [DateTime] $dateCreated = [Xml.XmlConvert]::ToDateTime($scan.dateCreated) 1652 | 1653 | # Create JSON structure for failed scans 1654 | [PSCustomObject] $failed = [ordered] @{ 1655 | EventDate = "" 1656 | ScanId = "$($scan.id)" 1657 | ProjectId = "$($scan.project.id)" 1658 | ProjectName = "$($scan.project.name)" 1659 | Origin = "$($scan.origin)" 1660 | IsPublic = "$($scan.isPublic)" 1661 | IsIncremental = "$($scan.isIncremental)" 1662 | ScanRequestDate = "$($this.dateUtil.ToUTCAndFormat($dateCreated))" 1663 | Loc = "$($scan.loc)" 1664 | FailReason = "$reason" 1665 | ScannedLanguages = $scan.languages 1666 | } 1667 | $this.io.WriteJSON([AlertType]::SCAN_FAILED, $failed) 1668 | } 1669 | 1670 | # Sends out alerts only if there are any 1671 | $this.alertService.Send() 1672 | } 1673 | 1674 | # Returns a human readable Scan Identifier 1675 | [String] GetScanIdentifierForHumans($scan) { 1676 | [String] $scanType = if ($scan.isIncremental -eq $True) { "Incremental" } else { "Full" } 1677 | return "Scan [id: $($scan.id), project: $($scan.project.Name), type: $scanType, stage: $($scan.stage.value), loc: $($scan.loc), stage/total %: $($scan.stagePercent)/$($scan.totalPercent), engine: $($scan.engine.id)]" 1678 | } 1679 | 1680 | # Returns a machine readable Scan key 1681 | [String] GetKey([Object] $scan) { 1682 | [String] $scanType = if ($scan.isIncremental -eq $True) { "I" } else { "F" } 1683 | return "$($scan.id)_$($scan.project.Name)_$scanType" 1684 | } 1685 | } 1686 | 1687 | # ----------------------------------------------------------------- 1688 | # ----------------------------------------------------------------- 1689 | # 1690 | # Execution entry 1691 | # 1692 | # ----------------------------------------------------------------- 1693 | # ----------------------------------------------------------------- 1694 | 1695 | # Check if PS v5+ 1696 | $psv = $PSVersionTable.PSVersion.Major 1697 | if ($psv -and $psv -lt 5) { 1698 | Write-Host "Requires PSv5 and greater." 1699 | Exit 1700 | } 1701 | 1702 | # Load configuration 1703 | [PSCustomObject] $config = [Config]::new().GetConfig() 1704 | # Override if values were explicitly overridden via the commandline 1705 | 1706 | #Target name for storing the SAST credentials in Windows Credential Manager 1707 | [String] $SAST_CREDENTIALS = "CxOverwatch.SAST" 1708 | [String] $SAST_DB_CREDENTIALS = "CxOverwatch.SAST.DB" 1709 | 1710 | 1711 | if ($cxUser -and $cxPass) #SAST Credentials provided through commandline argument. 1712 | { 1713 | $cxUsername = $cxUser 1714 | $cxPassword = $cxPass 1715 | #Store them for future use. 1716 | New-StoredCredential -Target $SAST_CREDENTIALS -Persist 'LocalMachine' -Comment 'Used for CxOverwatch.' -UserName $cxUser -Password $cxPass | Out-Null 1717 | 1718 | } 1719 | else { # Use Credential Manager 1720 | #Check if the SAST credentials are present in Windows Credential Manager. 1721 | $sastCredentials = Get-StoredCredential -Target $SAST_CREDENTIALS 1722 | 1723 | if (!$sastCredentials) { 1724 | Write-Host "Required credentials not found in Windows Credential Manager." 1725 | Write-Host "Kindly enter the CxSAST credentials." 1726 | 1727 | #Create new credentials - store against LocalMachine (so can be used by other sessions of the current user) 1728 | New-StoredCredential -Target $SAST_CREDENTIALS -Persist 'LocalMachine' -Comment 'Used for CxOverwatch.' -Credentials $(Get-Credential) | Out-Null 1729 | $sastCredentials = Get-StoredCredential -Target $SAST_CREDENTIALS 1730 | } 1731 | 1732 | $cxUsername = $sastCredentials.UserName 1733 | $cxPassword = $sastCredentials.GetNetworkCredential().Password 1734 | 1735 | } 1736 | 1737 | if ($dbUser -and $dbPass) # SAST-DB Credentials provided through commandline argument. 1738 | { 1739 | Write-Host "Using the user provided credentials for SAST-DB" 1740 | $cxDbUser = $dbUser 1741 | $cxDbPassword = $dbPass 1742 | #Store them for future use. 1743 | New-StoredCredential -Target $SAST_DB_CREDENTIALS -Persist 'LocalMachine' -Comment 'Used for CxOverwatch SAST-DB.' -UserName $dbUser -Password $dbPass | Out-Null 1744 | 1745 | } 1746 | else { # Use Credential Manager 1747 | #Check if the SAST-DB credentials are present in Windows Credential Manager. 1748 | $sastDatabaseCredentials = Get-StoredCredential -Target $SAST_DB_CREDENTIALS 1749 | 1750 | if ($sastDatabaseCredentials) { 1751 | Write-Host "Found SAST-DB credentials." 1752 | $cxDbUser = $sastDatabaseCredentials.UserName 1753 | $cxDbPassword = $sastDatabaseCredentials.GetNetworkCredential().Password 1754 | } 1755 | else { 1756 | Write-Host "Using Windows Authentication for SQL Server" 1757 | $cxDbUser = "" 1758 | $cxDbPassword = "" 1759 | } 1760 | } 1761 | 1762 | 1763 | # Create an IO utility object 1764 | [IO] $io = [IO]::new() 1765 | 1766 | # Create the Alert Service 1767 | [AlertService] $alertService = [AlertService]::new() 1768 | 1769 | # Load a scan time estimation algo 1770 | [ScanTimeAlgo] $scanTimeAlgo = [DefaultScanTimeAlgo]::new() 1771 | 1772 | # Create Queue Monitor and inject dependencies - scan duration calculator and alerting service 1773 | [QueueMonitor] $qMonitor = [QueueMonitor]::new($scanTimeAlgo, $alertService) 1774 | 1775 | # Create Engine(s) monitor 1776 | [EngineMonitor] $engineMonitor = [EngineMonitor]::new($alertService) 1777 | 1778 | if ($audit) { 1779 | # Create a DB Client 1780 | [DBClient] $dbClient = [DBClient]::new($config.cx.db.instance, $cxDbUser, $cxDbPassword) 1781 | 1782 | # Create Audit(s) monitor 1783 | [AuditMonitor] $auditMonitor = [AuditMonitor]::new($alertService) 1784 | # Inject a DB Client 1785 | $auditMonitor.SetDbClient($dbClient) 1786 | } 1787 | 1788 | # Spit out pretty headers 1789 | $io.WriteHeader() 1790 | 1791 | # Force TLS1.2 1792 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 1793 | 1794 | [Object[]] $queuedEntriesDto = $null 1795 | 1796 | # Continuous monitoring 1797 | $io.Log("Monitoring CxSAST Health") 1798 | while ($True) { 1799 | 1800 | # Process Queue Status response 1801 | $qMonitor.Monitor() 1802 | 1803 | # Check the engine(s) 1804 | $engineMonitor.Monitor() 1805 | 1806 | # Poll Audit DBs 1807 | if ($audit) { 1808 | $auditMonitor.Monitor() 1809 | } 1810 | 1811 | # Wait a bit before polling again 1812 | Start-Sleep -Seconds $script:config.monitor.pollIntervalSeconds 1813 | } 1814 | --------------------------------------------------------------------------------