├── .gitignore ├── CODEOWNERS ├── Empire ├── README.md ├── __init__.py ├── bugsystems │ ├── __init__.py │ ├── base.py │ └── jira │ │ ├── JiraAPI.py │ │ └── __init__.py ├── cloudservices │ ├── __init__.py │ └── github │ │ ├── GithubAPI.py │ │ └── __init__.py ├── creds │ ├── __init__.py │ ├── credentials.py │ └── encrypt-cred.py └── helpers │ └── __init__.py ├── License.txt ├── README.md ├── alerts ├── __init__.py ├── email_alert.py └── templates │ ├── test.html │ └── test_email.txt ├── config.json.example ├── config.py ├── credentials.json.example ├── db.py ├── detection ├── __init__.py ├── behavior │ ├── __init__.py │ └── membertracker.py └── pattern │ ├── __init__.py │ ├── regex_rule_engine │ ├── __init__.py │ └── templates │ │ ├── ruleengine_github.html │ │ └── ruleengine_github.txt │ ├── regexruleengine.py │ └── templates │ ├── ruleengine_github.html │ └── ruleengine_github.txt ├── lib ├── __init__.py └── helpers.py ├── models └── __init__.py ├── plugins ├── README.md ├── __init__.py ├── base.py ├── basic_plugin_register │ └── main.py ├── basic_plugin_watcher │ ├── java.json │ ├── js.json │ └── main.py ├── forcedotcom_apex │ ├── main.py │ └── regex.json ├── forcedotcom_aura_cmp_ui │ ├── main.py │ └── regex.json ├── forcedotcom_aura_js │ ├── main.py │ └── regex.json ├── forcedotcom_base_watcher │ └── __init__.py ├── forcedotcom_pmd │ ├── README.md │ └── main.py └── forcedotcom_vf │ ├── main.py │ └── regex.json ├── providence.py ├── remove_pyc.sh ├── repos ├── __init__.py ├── base.py ├── diffparser.py ├── github.py ├── perforce.py └── repotracker.py ├── requirements.txt ├── settings.py └── tests ├── __init__.py ├── detection ├── __init__.py └── regex_rule_engine │ ├── __init__.py │ ├── config_one.json │ └── test_regexruleengine.py ├── plugins ├── __init__.py ├── test_pluginloader.config.json ├── test_pluginloader.py └── test_pluginloader_testplugins │ └── debug │ └── main.py ├── repos ├── __init__.py ├── test_diffparser.py └── test_repotracker.py └── test_providence.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | .*.sw[a-z] 5 | venv/ 6 | providence.log 7 | credentials.json 8 | config.json 9 | key 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Comment line immediately above ownership line is reserved for related gus information. Please be careful while editing. 2 | #ECCN:Open Source 3 | -------------------------------------------------------------------------------- /Empire/README.md: -------------------------------------------------------------------------------- 1 | Empire 2 | ========== 3 | Common API Libraries and libraries related to acquisition systems 4 | 5 | 6 | ## Requirements 7 | python2.7 8 | 9 | ## Setup 10 | The following is only necessary if using Empire separate from Providence. The Providence setup will also set up Empire. 11 | 12 | ### 1. OS X Prerequisites 13 | * Homebrew (http://brew.sh) 14 | * XCode (once installed, open up and accept license) 15 | * Xcode Command Line Tools (from Terminal.app) 16 | xcode-select --install 17 | 18 | ### 2. Install the VirtualEnv in your Empire Directory & Install Dependencies 19 | #### Linux 20 | sudo apt-get install python-pip python-virtualenv 21 | 22 | #### OS X 23 | sudo easy_install virtualenv 24 | 25 | #### Confiuration On All Systems 26 | You may need to follow the instructions here for the cryptography module in the next steps: https://cryptography.io/en/latest/installation/ 27 | 28 | On OSX the install will fail without the proper environmental variables being set. 29 | 30 | #### All Systems 31 | virtualenv venv 32 | source venv/bin/activate 33 | pip install -r requirements.txt 34 | 35 | If you need to deactivate the virtualenv just type `deactivate` 36 | 37 | ### 3. Generate a Credentials Key 38 | ``` 39 | dd if=/dev/urandom bs=32 count=1 2>/dev/null | openssl base64 40 | ``` 41 | This key can be stored in the environmental variable $CREDENTIAL_KEY or entered when Providence is first run. It's highly 42 | recommended you don't keep the key on the same server as the credentials.json file, and use something like LastPass for 43 | keeping it safe. 44 | 45 | ### 4. Entering Credentials 46 | When you start up Providence it will ask you for credentials that aren't found, or you can edit the credentials.json file yourself (useful if one github account works for several repositories). 47 | 48 | You can encrypt a passwords using the command: 49 | ``` 50 | python Empire/creds/encrypt-cred.py 51 | ``` 52 | 53 | copy example_credentials.json to credentials.json and update it as needed: 54 | ```json 55 | { 56 | "github": { 57 | "type":"password", 58 | "username":"myusername", 59 | "password":"plaintext-password or fernet-encrypted password" 60 | } 61 | } 62 | ``` 63 | -------------------------------------------------------------------------------- /Empire/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | __all__ = ["creds", "Github"] 17 | -------------------------------------------------------------------------------- /Empire/bugsystems/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Connectors to bug tracking systems 18 | """ 19 | 20 | __copyright__ = "2015 Salesforce.com, Inc" 21 | __status__ = "Prototype" 22 | -------------------------------------------------------------------------------- /Empire/bugsystems/base.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Base classes defining default bug & system behavior to be inherited by specific system 18 | classes 19 | """ 20 | 21 | from datetime import datetime 22 | import pytz 23 | import hashlib 24 | 25 | __copyright__ = "2015 Salesforce.com, Inc" 26 | __status__ = "Prototype" 27 | 28 | class BugSystem(object): 29 | def __init__(self, creds): 30 | pass 31 | 32 | def bugs_for(self, email=None, team=None): 33 | """ Bugs for a certain user """ 34 | pass 35 | 36 | def bugs_touched_recently(self, days_ago=7): 37 | pass 38 | 39 | def bugs_open_in_period(self, from_date=None, to_date=None): 40 | # default from 2 years ago to today 41 | pass 42 | 43 | def bugs_closed_in_period(self, from_date=None, to_date=None): 44 | raise NotImplementedError 45 | 46 | def open_bugs(self,p0=True,p1=True,p2=True): 47 | """ Returns open p0/p1/p2 bugs """ 48 | pass 49 | 50 | class Bug(object): 51 | OPS_GROUP = "ops" 52 | PRODUCT_GROUP = "product" 53 | def __init__(self, id=None, title=None, rating=None, area=None, cloud=None, team=None, email=None, 54 | open_date=None, closed_date=None, url=None): 55 | self.cloud = cloud 56 | self.area = area 57 | self.id = id 58 | self.status = None 59 | self.title = title 60 | self.email = email 61 | self.team = team 62 | self.rating = rating 63 | self.open_date = open_date 64 | self.updated_date = None 65 | self.closed_date = closed_date 66 | self.url = url 67 | self.group = None 68 | 69 | def __getattribute__(self, name): 70 | if name=="priority": 71 | return self.rating 72 | else: 73 | return object.__getattribute__(self, name) 74 | 75 | def time_open(self): 76 | if self.closed_date is None: 77 | return None 78 | return self.closed_date - self.open_date 79 | 80 | def age(self, at_date=None): 81 | if at_date is None: 82 | at_date = self.closed_date 83 | else: 84 | if self.closed_date is not None and self.closed_date < at_date: 85 | at_date = self.closed_date 86 | if at_date is None: 87 | now_aware = pytz.utc.localize(datetime.utcnow()) 88 | return now_aware - self.open_date 89 | return at_date - self.open_date 90 | 91 | def titles(self): 92 | return ["cloud","area","team","group", 93 | "id","title","rating","status","age","email", 94 | "open date","closed date","url"] 95 | 96 | def list(self, json=False): 97 | open_date = self.open_date 98 | closed_date = self.closed_date 99 | age = self.age().days 100 | if (json is True): 101 | open_date = str(open_date) 102 | closed_date = str(closed_date) 103 | age = str(age) 104 | return [self.cloud, self.area, self.team, self.group, 105 | self.id, self.title, self.rating, self.status, age, self.email, 106 | open_date, closed_date, self.url] 107 | 108 | def dictionary(self): 109 | dictionary = dict(zip(self.titles(), self.list())) 110 | return dictionary 111 | 112 | def json(self): 113 | dictionary = dict(zip(self.titles(), self.list(json=True))) 114 | return dictionary 115 | 116 | def tracking_record(self): 117 | def hash(data): 118 | return hashlib.sha224(data).hexdigest() 119 | return {"Title":hash(self.title), 120 | "Priority":self.priority, 121 | "Owner":self.email, 122 | "Group":self.group, 123 | "Team":self.team, 124 | "Status":self.status, 125 | "id":self.id 126 | } 127 | 128 | -------------------------------------------------------------------------------- /Empire/bugsystems/jira/JiraAPI.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | JiraAPI - Sends JQL (Jira Query Language) requests to a Jira instance and returns the results 18 | """ 19 | import ssl 20 | 21 | import requests 22 | import json 23 | import urllib 24 | import os 25 | 26 | __copyright__ = "2015 Salesforce.com, Inc" 27 | __status__ = "Prototype" 28 | 29 | 30 | class JiraAPI(object): 31 | def __init__(self, server, credentials): 32 | super(JiraAPI, self).__init__() 33 | self.server = server 34 | self.credentials = credentials 35 | self.verify = None 36 | __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) 37 | 38 | def fetchCommitDetails(self, url): 39 | r = requests.get(url, auth=self.auth(), verify=self.verify); 40 | if r.headers['x-ratelimit-remaining']: 41 | remaining_requests = int(r.headers['x-ratelimit-remaining']) 42 | if (remaining_requests == 0): 43 | self._no_more_requests_until = datetime.datetime.fromtimestamp(float(r.headers['x-ratelimit-reset'])); 44 | #reset_time = long(r.headers['x-ratelimit-reset']) 45 | return None 46 | if(r.ok): 47 | return r.json() 48 | return None 49 | 50 | def jql(self, query, offset=None): 51 | verbose = False 52 | resource_name = "search" 53 | url = "https://%s/rest/api/latest/%s" % (self.server, urllib.quote(resource_name)) 54 | params = {"jql":query} 55 | if offset is not None: 56 | params["startAt"] = offset 57 | r = requests.get(url, params=params, headers={ "Authorization":self.credentials.authorizationHeaderValue() }, verify=self.verify) 58 | if(r.ok): 59 | results = r.json() 60 | return results 61 | else: 62 | print "Request failed: ", r.status_code 63 | print r.text 64 | return None 65 | 66 | if __name__ == '__main__': 67 | pass -------------------------------------------------------------------------------- /Empire/bugsystems/jira/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Jira Bug Tracking System Interface 18 | """ 19 | 20 | __copyright__ = "2015 Salesforce.com, Inc" 21 | __status__ = "Prototype" 22 | 23 | import sys 24 | import os.path 25 | sys.path.append( 26 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 27 | import requests 28 | import json 29 | 30 | from bugsystems.jira.JiraAPI import JiraAPI 31 | from bugsystems.base import Bug, BugSystem 32 | 33 | from dateutil import parser 34 | from dateutil import relativedelta 35 | from datetime import datetime 36 | from datetime import timedelta 37 | 38 | __copyright__ = "2015 Salesforce.com, Inc" 39 | __status__ = "Prototype" 40 | 41 | class JiraBugSystem(BugSystem): 42 | def __init__(self, creds, server): 43 | super(JiraBugSystem, self).__init__(creds) 44 | self.creds = creds 45 | self.jira = JiraAPI(server, self.creds) 46 | 47 | def from_search_json(self, issue): 48 | pass 49 | 50 | def query_bugs(self, query): 51 | bugs = [] 52 | def __api_call(offset=None): 53 | query_results = self.jira.jql(query, offset) 54 | if query_results: 55 | issues = query_results["issues"] 56 | for issue in issues: 57 | bug = self.from_search_json(issue) 58 | bugs.append(bug) 59 | try: 60 | starts_at = query_results.get("startAt") 61 | max_results = query_results.get("maxResults") 62 | total = query_results.get("total") 63 | ends_at = int(starts_at) + int(max_results) 64 | if ends_at < int(total): 65 | __api_call(offset=ends_at) 66 | except TypeError: 67 | pass 68 | 69 | __api_call(); 70 | return bugs 71 | -------------------------------------------------------------------------------- /Empire/cloudservices/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Interfaces to Cloud services 18 | """ 19 | 20 | __copyright__ = "2015 Salesforce.com, Inc" 21 | __status__ = "Prototype" 22 | -------------------------------------------------------------------------------- /Empire/cloudservices/github/GithubAPI.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | GitHubCommit - Convert JSON message from GH into an object representative of the commit 18 | GitHubRepo - Represent the basic information needed to interact with a GH repo 19 | GitHubAPI - Send and receive data from the REST API 20 | """ 21 | 22 | # TODO: 23 | # - Pagination the github way 24 | # - Groups / Users / Org 25 | # - Security 26 | # - Stale branches 27 | 28 | import sys 29 | import os.path 30 | sys.path.append( 31 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 32 | from creds.credentials import Credentials 33 | import requests 34 | import urllib 35 | import datetime 36 | import pytz 37 | import json 38 | import logging 39 | logger = logging.getLogger('GithubAPI') 40 | 41 | __copyright__ = "2015 Salesforce.com, Inc" 42 | __status__ = "Prototype" 43 | 44 | class GithubAPI(object): 45 | def __init__(self, server, credentials): 46 | self.server = server 47 | self.credentials = credentials 48 | self._no_more_requests_until = None 49 | 50 | def fetch(self, url, params=None, post_data=None): 51 | if self._no_more_requests_until: 52 | if self._no_more_requests_until < datetime.datetime.utcnow(): 53 | return None 54 | self._no_more_requests_until = None 55 | r = None 56 | if post_data: 57 | raise NotImplementedError("GithubAPI Post unimplemented") 58 | return 59 | else: 60 | if self.credentials: 61 | r = requests.get(url, params=params, headers={ "Authorization":self.credentials.authorizationHeaderValue() }) 62 | else: 63 | r = requests.get(url, params=params) 64 | if r.headers.get('x-ratelimit-remaining'): 65 | remaining_requests = int(r.headers['x-ratelimit-remaining']) 66 | if (remaining_requests == 0): 67 | logger.warning("Github API hit the rate limiter") 68 | self._no_more_requests_until = datetime.datetime.fromtimestamp(float(r.headers.get('x-ratelimit-reset'))); 69 | return None 70 | if(r.ok): 71 | results = r.json() 72 | return results 73 | logger.warning("Github fetch of %s failed\n%s\n",r.url,r.text) 74 | return None 75 | 76 | def fetch_raw(self, url): 77 | if self._no_more_requests_until: 78 | if self._no_more_requests_until < datetime.datetime.utcnow(): 79 | return None 80 | self._no_more_requests_until = None 81 | r = None 82 | if self.credentials: 83 | r = requests.get(url, headers={ "Authorization":self.credentials.authorizationHeaderValue(),"Accept":"application/vnd.github.v3.raw" }) 84 | else: 85 | r = requests.get(url) 86 | if r.headers.get('x-ratelimit-remaining'): 87 | remaining_requests = int(r.headers['x-ratelimit-remaining']) 88 | if (remaining_requests == 0): 89 | logger.warning("Github API hit the rate limiter") 90 | self._no_more_requests_until = datetime.datetime.fromtimestamp(float(r.headers.get('x-ratelimit-reset'))); 91 | return None 92 | if(r.ok): 93 | results = r.text 94 | return results 95 | logger.warning("Github fetch of %s failed\n%s\n",r.url,r.text) 96 | return None 97 | 98 | def baseURL(self, org_name=None, repo_name=None): 99 | baseurl = 'https://%s' % (self.server) 100 | if repo_name is not None: 101 | baseurl += "/repos/%s/%s" % (org_name, repo_name) 102 | elif org_name is not None: 103 | baseurl += "/orgs/%s" % (org_name) 104 | return baseurl 105 | 106 | if __name__ == "__main__": 107 | creds = Credentials("github") 108 | git = GithubAPI(GithubRepo('api.github.com', 'salesforce','providence'), creds) 109 | bugs = git.issues(params={"labels":"bug,security","state":"all","since":"2015-02-01T00:00:00Z"}) 110 | import json 111 | print json.dumps(bugs, indent=2) 112 | if bugs: 113 | for bug in bugs: 114 | print bug["title"], bug["state"] 115 | -------------------------------------------------------------------------------- /Empire/cloudservices/github/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | GitHub 18 | """ 19 | 20 | __copyright__ = "2015 Salesforce.com, Inc" 21 | __status__ = "Prototype" 22 | 23 | # TODO: 24 | # - Pagination the github way 25 | # - Groups / Users / Org 26 | # - Security 27 | # - Stale branches 28 | 29 | import sys 30 | import os.path 31 | sys.path.append( 32 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 33 | from creds import CredentialManager 34 | import requests 35 | import urllib 36 | import datetime 37 | import pytz 38 | from cloudservices.github.GithubAPI import GithubAPI 39 | 40 | __copyright__ = "2015 Salesforce.com, Inc" 41 | __status__ = "Prototype" 42 | 43 | class GithubOrg(object): 44 | def __init__(self, server, orgname, creds): 45 | self.server = server 46 | self.name = orgname 47 | self.creds = creds 48 | self.github_api = GithubAPI(server, creds) 49 | 50 | def repositories(self, _type="all"): 51 | url = "%s/repos" % (self.github_api.baseURL(self.name)) 52 | params = {"type":_type} 53 | results = self.github_api.fetch(url, params=params) 54 | if results is None: 55 | return None 56 | repos = [] 57 | for result in results: 58 | if result.get("name") is None: 59 | continue 60 | repo = GithubRepo(self, result.get("name"), repo_json=result) 61 | repos.append(repo) 62 | return repos 63 | 64 | def repository(self, repository_name): 65 | url = "%s" % (self.github_api.baseURL(self.name), repository_name) 66 | result = self.github_api.fetch(url) 67 | if result is None: 68 | return None 69 | repo = GithubRepo(self, result.get("name"), repo_json=result) 70 | 71 | def members(self, _filter="all"): 72 | url = "%s/members" % (self.github_api.baseURL(self.name)) 73 | params = {"filter":_filter, "per_page":200} 74 | results = self.github_api.fetch(url, params=params) 75 | if results is None: 76 | return None 77 | members = [] 78 | for result in results: 79 | if result.get("id") is None: 80 | continue 81 | member = GithubMember(self, result.get("id"), member_json=result) 82 | members.append(member) 83 | return members 84 | 85 | class GithubRepo(object): 86 | def __init__(self, github_org, name, repo_json=None): 87 | self.github_org = github_org 88 | self.name = name 89 | self.json = repo_json 90 | 91 | def issues(self, params=None): 92 | url = "%s/issues" % (self.github_org.github_api.baseURL(self.github_org.name, self.name)) 93 | results = self.github_org.github_api.fetch(url, params=params) 94 | if results is None: 95 | return None 96 | issues = [] 97 | for result in results: 98 | if result.get("number") is None: 99 | continue 100 | issue = GithubIssue(self.github_org, result.get("number"), github_repo=self, issue_json=result) 101 | issues.append(issue) 102 | return issues 103 | 104 | def issue(self, issue_number): 105 | url = "%s/issues/%s" % (self.github_org.github_api.baseURL(self.github_org.name, self.name), issue_number) 106 | result = self.github_org.github_api.fetch(url) 107 | if result is None: 108 | return None 109 | issue = GithubIssue(self.github_org, result.get("number"), github_repo=self, issue_json=result) 110 | return issue 111 | 112 | def commits(self, commits_since_datetime, path=None): 113 | commits_since_datetime_utc = commits_since_datetime.astimezone(pytz.utc) 114 | since_iso_string = commits_since_datetime_utc.strftime("%Y-%m-%dT%H:%M:%SZ") 115 | until_iso_string = datetime.datetime.utcnow().isoformat('T') + "Z" 116 | 117 | url = "%s/commits" % (self.github_org.github_api.baseURL(self.github_org.name, self.name)) 118 | params = {"since":since_iso_string} 119 | if path: 120 | params["path"] = path 121 | results = self.github_org.github_api.fetch(url, params=params) 122 | if results is None: 123 | return None 124 | commits = [] 125 | for result in results: 126 | if result.get("sha") is None: 127 | continue 128 | commit = GithubCommit(self, result.get("sha"), commit_json=result) 129 | commits.append(commit) 130 | return commits 131 | 132 | def commit(self, commit_sha): 133 | url = "%s/commits/%s" % (self.github_org.github_api.baseURL(self.github_org.name, self.name), commit_sha) 134 | result = self.github_org.github_api.fetch(url) 135 | if result is None: 136 | return None 137 | commit = GithubCommit(self, result.get("sha"), commit_json=result) 138 | return commit 139 | 140 | def get_raw_file(self, file_path, commit_sha=None): 141 | url = "%s/contents/%s" % (self.github_org.github_api.baseURL(self.github_org.name, self.name), file_path) 142 | file_content = self.github_org.github_api.fetch_raw(url) 143 | return file_content 144 | 145 | class GithubIssue(object): 146 | def __init__(self, github_org, number, github_repo=None, issue_json=None): 147 | self.github_org = github_org 148 | self.github_repo = github_repo 149 | self.number = number 150 | self.json = issue_json 151 | 152 | class GithubCommit(object): 153 | def __init__(self, github_repo, sha, commit_json=None): 154 | self.github_repo = github_repo 155 | self.sha = sha 156 | self.json = commit_json 157 | self.url = None 158 | self.html_url = None 159 | self.message = None 160 | self.date = None 161 | self.committer_name = None 162 | self.committer_email = None 163 | self.committer_login = None 164 | self.committer_type = None 165 | self.files = None 166 | if self.json is not None: 167 | self.parseJSON(self.json) 168 | 169 | def parseJSON(self, commits_json): 170 | self.url = commits_json.get('url') 171 | self.sha = commits_json.get('sha') 172 | self.html_url = commits_json.get('html_url') 173 | 174 | commit = commits_json.get('commit') 175 | if commit is None: 176 | commit = {} 177 | self.message = commit.get("message") 178 | commit_committer = commit.get('committer') 179 | if commit_committer is None: 180 | commit_committer = {} 181 | self.date = commit_committer.get('date') 182 | self.committer_email = commit_committer.get('email') 183 | 184 | committer = commits_json.get('committer') 185 | if committer is None: 186 | committer = {} 187 | self.committer_login = committer.get('login') 188 | self.committer_name = committer.get('name') 189 | self.committer_type = committer.get('type') 190 | self.files = commits_json.get('files') 191 | 192 | class GithubMember(object): 193 | def __init__(self, github_org, _id, member_json=None): 194 | self.github_org = github_org 195 | self.id = _id 196 | self.json = member_json 197 | 198 | if __name__ == "__main__": 199 | 200 | credentials_file = "credentials.json" 201 | credential_key = os.environ.get('CREDENTIAL_KEY') 202 | if credential_key is None: 203 | credential_key = getpass.getpass('Credential Key:') 204 | credential_manager = CredentialManager(credentials_file, credential_key) 205 | creds = credential_manager.get_or_create_credentials_for("github-pardot-orgowner","password") 206 | 207 | git = GithubOrg('api.github.com', 'Pardot', creds) 208 | members = git.members("2fa_disabled") 209 | for member in members: 210 | print member.json["login"] 211 | 212 | bugs = git.issues(params={"labels":"bug,security","state":"all","since":"2015-02-01T00:00:00Z"}) 213 | import json 214 | print json.dumps(bugs, indent=2) 215 | if bugs: 216 | for bug in bugs: 217 | print bug["title"], bug["state"] 218 | -------------------------------------------------------------------------------- /Empire/creds/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Credential Manager Base Classes 18 | 19 | CredentialManager - Manage the location, collection and encryption/decryption of credentials 20 | """ 21 | 22 | import sys 23 | import os.path 24 | import json 25 | sys.path.append( 26 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) 27 | from creds.credentials import Credentials 28 | from creds.credentials import CredentialsInvalidError, OAuthCredentials 29 | import helpers 30 | 31 | CRYPTO_FERNET=True 32 | try: 33 | from cryptography.fernet import Fernet 34 | except: 35 | CRYPTO_FERNET=False 36 | import fernet 37 | fernet.Configuration.enforce_ttl = False 38 | 39 | __copyright__ = "2015 Salesforce.com, Inc" 40 | __status__ = "Prototype" 41 | 42 | class CredentialManager(object): 43 | def __init__(self, filename, key=None): 44 | def decrypt(ciphertext): 45 | if key is None: 46 | return ciphertext 47 | try: 48 | value = ciphertext 49 | if CRYPTO_FERNET == True: 50 | f = Fernet(key) 51 | value = f.decrypt(ciphertext) 52 | else: 53 | verifier = fernet.verifier(key, ciphertext) 54 | if verifier.valid() == True: 55 | value = verifier.message 56 | return value 57 | except Exception as e: 58 | pass 59 | return ciphertext 60 | 61 | def encrypt(value): 62 | if key is None: 63 | return value 64 | if CRYPTO_FERNET == True: 65 | f = Fernet(key) 66 | token = f.encrypt(bytes(value)) 67 | else: 68 | token = fernet.generate(key, value) 69 | return token 70 | 71 | self.filename = filename 72 | self.encrypt = encrypt 73 | self.decrypt = decrypt 74 | 75 | def credential_object_for_type(self, identifier, type): 76 | if (type == "password"): 77 | return Credentials(identifier) 78 | raise NotImplementedError("Credential type '%s' is not supported" % (type)) 79 | 80 | def get_credentials_for(self, identifier, check_valid=True): 81 | file_creds = {} 82 | try: 83 | with open(self.filename) as fp: 84 | file_creds = json.load(fp,object_hook=helpers.convert) 85 | except IOError as ioe: 86 | raise NameError("Credential file not found. " + str(ioe)) 87 | 88 | user_creds = file_creds.get(identifier) 89 | if user_creds is None: 90 | raise NameError(identifier) 91 | return 92 | 93 | #XXX: Why bother type checking here if we're specifying that in the function 94 | #XXX: call in providence.py? 95 | type = user_creds.get("type") 96 | credentials = self.credential_object_for_type(identifier, type) 97 | credentials.credential_manager = self 98 | credentials.encrypt = self.encrypt 99 | credentials.decrypt = self.decrypt 100 | credentials.read(user_creds) 101 | 102 | if check_valid == True and credentials.valid() == False: 103 | if isinstance(creds, OAuthCredentials): 104 | try: 105 | credentials.refreshToken() 106 | if credentials.valid(): 107 | if write == True: 108 | credentials.write() 109 | return credentials 110 | except NotImplementedError, nie: 111 | pass 112 | raise CredentialsInvalidError("Invalid Credentials for %s" % identifier) 113 | return credentials 114 | 115 | def get_or_create_credentials_for(self, identifier, type, write=True, check_valid=True): 116 | try: 117 | creds = self.get_credentials_for(identifier, check_valid=False) 118 | except NameError, ne: 119 | creds = self.new_credentials_for(identifier, type) 120 | if write == True: 121 | creds.write() 122 | if check_valid == True and creds.valid() == False: 123 | # If it's an Oauth token maybe it needs refreshed? 124 | if isinstance(creds, OAuthCredentials): 125 | try: 126 | creds.refreshToken() 127 | if creds.valid(): 128 | if write == True: 129 | creds.write() 130 | return creds 131 | except NotImplementedError, nie: 132 | pass 133 | # Lets give them 3 attempts to login 134 | for attempt_numer in range(3): 135 | creds.new_credentials() 136 | if creds.valid(): 137 | if write == True: 138 | creds.write() 139 | return creds 140 | # All attempts failed, return creds are bad 141 | raise CredentialsInvalidError("Invalid Credentials for %s" % identifier) 142 | return creds 143 | 144 | def new_credentials_for(self, identifier, type, server_data=None): 145 | credentials = self.credential_object_for_type(identifier, type) 146 | credentials.credential_manager = self 147 | credentials.encrypt = self.encrypt 148 | credentials.decrypt = self.decrypt 149 | if (server_data is not None): 150 | credentials.server_data = server_data 151 | credentials.new_credentials() 152 | 153 | return credentials 154 | 155 | def write_back_credentials(self, credentials): 156 | file_creds = {} 157 | try: 158 | file_data = open(self.filename) 159 | file_creds = json.load(file_data) 160 | except IOError as ioe: 161 | print "Credential file not found, will create..." 162 | 163 | user_creds = file_creds.get(credentials.identifier) 164 | if user_creds is None: 165 | user_creds = {} 166 | user_creds = credentials._populate_user_creds(user_creds) 167 | file_creds[credentials.identifier] = user_creds 168 | 169 | with open(self.filename, 'w') as outfile: 170 | json.dump(file_creds, outfile, indent=2, separators=(',', ': ')) 171 | 172 | 173 | if __name__ == '__main__': 174 | import os 175 | credential_key = os.environ.get('CREDENTIAL_KEY') 176 | credentials_file = "credentials2.json" 177 | credential_manager = CredentialManager(credentials_file, credential_key) 178 | -------------------------------------------------------------------------------- /Empire/creds/credentials.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Credential Method Specific Classes 18 | 19 | Credentials - Base class to be inherited 20 | OAuthCredential(Credentials) - OAuth Methods 21 | """ 22 | import json 23 | import base64 24 | import requests 25 | import copy 26 | import getpass 27 | import urllib 28 | import logging 29 | import httplib2 30 | import csv 31 | 32 | from rauth import OAuth2Service 33 | 34 | __copyright__ = "2015 Salesforce.com, Inc" 35 | __status__ = "Prototype" 36 | 37 | logger = logging.getLogger(__name__) 38 | 39 | class CredentialsInvalidError(Exception): 40 | pass 41 | 42 | class Credentials(object): 43 | def __init__(self, identifier): 44 | self.type = "password" 45 | self.identifier = identifier 46 | self.server_data = {} 47 | self.username = None 48 | self.password = None 49 | 50 | def read(self, user_creds): 51 | self.username = user_creds.get("username") 52 | self.password = user_creds.get("password") 53 | if self.password: 54 | self.password = self.decrypt(self.password) 55 | 56 | def _populate_user_creds(self, user_creds): 57 | user_creds = {} 58 | user_creds["type"] = self.type 59 | if self.username is not None: 60 | user_creds["username"] = self.username 61 | if self.password is not None: 62 | user_creds["password"] = self.encrypt(self.password) 63 | return user_creds 64 | 65 | def new_credentials(self): 66 | print "Need credentials for [", self.identifier, "]" 67 | username = raw_input('Enter your username: ') 68 | password = getpass.getpass('Enter your password: ') 69 | self.username = username 70 | self.password = password 71 | 72 | def authorizationHeaderValue(self): 73 | b64creds = base64.b64encode("%s:%s" % (self.username, self.password)) 74 | return "Basic %s" % (b64creds) 75 | 76 | def write(self): 77 | self.credential_manager.write_back_credentials(self) 78 | 79 | def valid(self): 80 | return True 81 | 82 | class OAuthCredentials(Credentials): 83 | def __init__(self, identifier): 84 | super(OAuthCredentials, self).__init__(identifier) 85 | self.type = "oauth" 86 | self.token_data = {} 87 | 88 | def read(self, user_creds): 89 | super(OAuthCredentials, self).read(user_creds) 90 | self.token_data = user_creds.get("token_data",{}) 91 | if self.token_data.get("access_token") is not None: 92 | self.token_data["access_token"] = self.decrypt(self.token_data["access_token"]) 93 | self.server_data = user_creds.get("server_data",{}) 94 | if self.server_data is not None: 95 | self.server_data = user_creds.get("server_data",{}) 96 | if self.server_data.get("client_secret") is not None: 97 | self.server_data["client_secret"] = self.decrypt(self.server_data["client_secret"]) 98 | 99 | def _populate_user_creds(self, user_creds): 100 | user_creds = super(OAuthCredentials, self)._populate_user_creds(user_creds) 101 | if self.token_data and self.token_data.get("access_token") is not None: 102 | user_creds["token_data"] = copy.deepcopy(self.token_data) 103 | user_creds["token_data"]["access_token"] = self.encrypt(self.token_data["access_token"]) 104 | if self.server_data is not None: 105 | user_creds["server_data"] = copy.deepcopy(self.server_data) 106 | if self.server_data.get("client_secret") is not None: 107 | user_creds["server_data"]["client_secret"] = self.encrypt(self.server_data["client_secret"]) 108 | return user_creds 109 | 110 | def authorizationHeaderValue(self): 111 | if self.token_data is None or self.token_data.get("access_token") is None: 112 | self.refreshToken() 113 | return "Bearer %s" % (self.token_data.get("access_token")) 114 | 115 | def refreshToken(self): 116 | raise NotImplementedError 117 | -------------------------------------------------------------------------------- /Empire/creds/encrypt-cred.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Fernet encryption test 18 | """ 19 | 20 | import argparse 21 | from cryptography.fernet import Fernet 22 | import getpass 23 | import os 24 | 25 | __copyright__ = "2015 Salesforce.com, Inc" 26 | __status__ = "Prototype" 27 | 28 | if __name__ == '__main__': 29 | credentials_file = "credentials.json" 30 | credential_key = os.environ.get('CREDENTIAL_KEY') 31 | if credential_key is None: 32 | credential_key = getpass.getpass('Credential Key:') 33 | 34 | value = getpass.getpass('Enter a value to encrypt: ') 35 | f = Fernet(credential_key) 36 | token = f.encrypt(value) 37 | print token 38 | -------------------------------------------------------------------------------- /Empire/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | """ 17 | Helper functions and classes that are utilized across the application but do not belong in any one home 18 | """ 19 | 20 | __copyright__ = "2015 Salesforce.com, Inc" 21 | __status__ = "Prototype" 22 | 23 | def convert(input): 24 | """ 25 | Convert a Unicode encoded dictionary or list into a utf-8 encoded one. 26 | Copypasta'd from: 27 | http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-of-unicode-ones-from-json-in-python 28 | """ 29 | if isinstance(input, dict): 30 | return {convert(key): convert(value) for key, value in input.iteritems()} 31 | elif isinstance(input, list): 32 | return [convert(element) for element in input] 33 | elif isinstance(input, unicode): 34 | return input.encode('utf-8') 35 | else: 36 | return input 37 | -------------------------------------------------------------------------------- /License.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Salesforce.com, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | 8 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 9 | 10 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Providence 2 | ========== 3 | Providence is a system for code commit & bug system monitoring. It is deployed within an organization to monitor code commits for security (or other) concerns, via customizable plugins. A plugin performs logic whenever a commit occurs. 4 | 5 | ##### Cool Stuff 6 | * Build plugin to run a something everytime a commit happens 7 | * `Empire` contains some useful tools for credential management 8 | * Find our slides from our AppSec presentation [here](http://www.slideshare.net/salesforceeng/providence-rapid-vulnerability-prevention) 9 | 10 | ## Requirements 11 | python2.7 12 | postgresql 9.4+ 13 | 14 | ## Steps: Local install with pip and virtualenv 15 | 16 | 17 | ### 1. OS X Prerequisites 18 | * Homebrew (http://brew.sh) 19 | * XCode (once installed, open up and accept license) 20 | * Xcode Command Line Tools (from Terminal.app) 21 | xcode-select --install 22 | 23 | ### 2. Setup Postgresql on your server 24 | 25 | ### 3. Setup the Database 26 | Create a database named 'providence' 27 | 28 | ### 4. Checkout Providence and Submodules 29 | git clone https://github.com/Salesforce/Providence --recursive 30 | cd Providence 31 | 32 | ### 5. Install the VirtualEnv in your Providence Directory & Install Dependencies 33 | #### Linux 34 | sudo apt-get install python-pip python-virtualenv 35 | 36 | #### OS X 37 | sudo easy_install virtualenv 38 | brew install swig postgresql wget 39 | 40 | #### Configuration On All Systems 41 | 42 | virtualenv venv 43 | source venv/bin/activate 44 | pip install -r requirements.txt 45 | 46 | For OSX users, you may have issues installing the cryptography dependency. If the above steps fail due to the cryptography module, update your pip to the latest version in virtualenv 47 | 48 | pip -V 49 | pip 7.1.2 from /Users/joe_smith/Desktop/Providence/venv/lib/python2.7/site-packages (python 2.7) 50 | (upgrade pip to latest) 51 | pip install --upgrade pip 52 | 53 | See https://cryptography.io/en/latest/installation/ if upgrading pip does not solve the problem 54 | 55 | If you need to deactivate the virtualenv just type `deactivate` 56 | 57 | #### If you would like to use Perforce monitoring 58 | (follow steps above, make sure you have run 'source venv/bin/activate') 59 | pip install p4python 60 | 61 | ### 6. Configuration 62 | The config.json.example file contains the settings for which to run Providence with. 63 | 64 | #### credentials_file 65 | Name of the credential storage file. There should be no need to modify this. 66 | 67 | #### logging 68 | Format of the Providence.log logging file. Change the `loglevel` if the log file is too large 69 | 70 | #### postgresql 71 | Edit this section to point to your Postgres server. `credential-identifier` is the name used in credentials.json for the Postgres username and password. 72 | 73 | #### repos 74 | Edit this section to point to the repositories you want to monitor with Providence. Github (including Enterprise) and Perforce are currently supported. 75 | 76 | { 77 | "type": "(github or perforce)", 78 | "name": "(name used by Providence, not the actual repo name)", 79 | "server": "(server URL)", 80 | "owner": "(github only - owner)", 81 | "directory": "(directory of the repo)" 82 | } 83 | 84 | #### cron 85 | `watcher_interval` sets the time in minutes between each scheduled processing, for watcher plugins. 86 | 87 | ### 7. Adjust which plugins you want to run 88 | Enable plugins in your new config.json file, several examples are listed in the example file. 89 | 90 | #### pmd_path 91 | If using the PMD plugin, fill out the full directory to `run.sh` file for your PMD installation. 92 | 93 | ### 8. Generate a Credentials Key 94 | ``` 95 | dd if=/dev/urandom bs=32 count=1 2>/dev/null | openssl base64 96 | ``` 97 | This key can be stored in the environmental variable $CREDENTIAL_KEY or entered when Providence is first run. It's highly 98 | recommended you don't keep the key on the same server as the credentials.json file, and use something like LastPass for 99 | keeping it safe. 100 | 101 | ### 9. Entering Credentials 102 | When you start up Providence it will try to connect to the repositories set up in config.json, and ask you for credentials that aren't found. Alternatively you can edit the credentials.json file yourself (useful if one github account works for several repositories). 103 | 104 | #### Manually create the credentials file. 105 | 106 | You can encrypt a passwords using the command: 107 | ``` 108 | python Empire/creds/encrypt-cred.py 109 | ``` 110 | 111 | copy credentials.json.example to credentials.json and update it as needed: 112 | ```json 113 | { 114 | "plsqlcreds": { 115 | "type":"password", 116 | "username":"", 117 | "password":"" 118 | }, 119 | "github": { 120 | "type":"password", 121 | "username":"", 122 | "password":"" 123 | } 124 | } 125 | ``` 126 | 127 | ### 10. Run Providence! 128 | ``` 129 | python providence.py 130 | ``` 131 | 132 | 133 | -------------------------------------------------------------------------------- /alerts/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | class Alert(object): 17 | DEBUG = -10 18 | INFO = 0 19 | LOW = 10 20 | MEDIUM = 100 21 | HIGH = 1000 22 | CRITICAL = 10000 23 | def __init__(self, message, message_html=None, subject=None, level=0): 24 | self.message=message 25 | self.subject = subject 26 | if self.subject is None: 27 | if message: 28 | self.subject = message[:30] 29 | self.level=level 30 | self.message_html = message_html -------------------------------------------------------------------------------- /alerts/email_alert.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import smtplib 17 | import re 18 | from email.mime.text import MIMEText 19 | from email.mime.multipart import MIMEMultipart 20 | from alerts import Alert 21 | 22 | import jinja2 23 | import logging 24 | import config 25 | import os 26 | 27 | logger = logging.getLogger('alert_email') 28 | 29 | class EmailAlert(object): 30 | @staticmethod 31 | def email_templates(templateVars, textTemplateFilename, htmlTemplateFilename=None): 32 | templateLoader = jinja2.FileSystemLoader( searchpath="/" ) 33 | templateEnv = jinja2.Environment( loader=templateLoader ) 34 | 35 | text_template_file = os.path.join(os.path.dirname(__file__),textTemplateFilename) 36 | text_template = templateEnv.get_template( text_template_file ) 37 | text = text_template.render( templateVars ) 38 | 39 | html = None 40 | if htmlTemplateFilename is not None: 41 | html_template_file = os.path.join(os.path.dirname(__file__),htmlTemplateFilename) 42 | html_template = templateEnv.get_template( html_template_file ) 43 | html = html_template.render( templateVars ) 44 | 45 | return (text, html) 46 | 47 | def __init__(self, alert=None, creds=None): 48 | configuration = config.Configuration('config.json') 49 | 50 | self.server = configuration.get(('email', 'host')) 51 | self.to_email = configuration.get(('email', 'to')) 52 | self.from_email = self.to_email 53 | self.creds=creds 54 | self.alert=alert 55 | 56 | def send(self, alert=None, to_email=None, from_email=None): 57 | if alert is None: 58 | alert = self.alert 59 | if to_email is None: 60 | to_email = self.to_email 61 | if from_email is None: 62 | from_email = self.from_email 63 | 64 | logger.debug("Preparing email to %s", to_email) 65 | 66 | if type(to_email) is not list: 67 | to_email = [to_email,] 68 | 69 | self.emails = ",".join(to_email) 70 | 71 | self.from_email = from_email 72 | self.subject = alert.subject 73 | self.text = alert.message 74 | self.html = alert.message_html 75 | 76 | msg = MIMEMultipart('alternative') 77 | msg['Subject'] = "" + self.subject 78 | msg['From'] = self.from_email 79 | msg['To'] = self.emails 80 | 81 | part1 = MIMEText(self.text, 'plain') 82 | msg.attach(part1) 83 | 84 | if self.html: 85 | part2 = MIMEText(self.html.encode('utf-8'), 'html', 'utf-8') 86 | msg.attach(part2) 87 | 88 | s = smtplib.SMTP(self.server) 89 | 90 | safe_to_email_addresses = [] 91 | for email_address in to_email: 92 | break_up_email = re.match("^([\w\d\.-]+)(\+.*)?@(.*)", email_address) 93 | if break_up_email is None: 94 | logger.error("Email address [%s] could not be parsed", str(email_address)) 95 | return False 96 | if len(break_up_email.groups()) != 3: 97 | logger.error("Email address [%s] did not parse correctly", str(email_address)) 98 | return False 99 | safe_to_email = "@".join( (break_up_email.groups()[0], break_up_email.groups()[2]) ) 100 | safe_to_email_addresses.append(safe_to_email) 101 | s.sendmail(from_email, safe_to_email_addresses, msg.as_string()) 102 | logger.info("Email sent to [%s] with subject [%s]", msg['To'], msg['Subject']) 103 | s.quit() 104 | return True 105 | 106 | if __name__ == "__main__": 107 | templateVars = { "title" : "Test Example 3", 108 | "description" : "A simple inquiry of function.", 109 | "rule" : "Rule test X", 110 | "github_link" : "#", 111 | "code" : "\n".join(["a","b","c"]) 112 | } 113 | (text, html) = EmailAlert.email_templates(templateVars, "templates/test_email.txt", "templates/test.html") 114 | EmailAlert().send(Alert(message=text, message_html=html), to_email=config.Configuration('config.json').get(('email', 'to'))) 115 | 116 | 117 | -------------------------------------------------------------------------------- /alerts/templates/test_email.txt: -------------------------------------------------------------------------------- 1 | {{ title }} 2 | 3 | Description 4 | ----------- 5 | {{ description }} 6 | 7 | Rule 8 | ---- 9 | {{ rule }} 10 | 11 | Link 12 | ---- 13 | {{ github_link }} 14 | 15 | Code 16 | ---- 17 | {{ code }} -------------------------------------------------------------------------------- /config.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "credentials_file": "credentials.json", 3 | "logging": { 4 | "filename": "providence.log", 5 | "loglevel": "debug", 6 | "stringfmt": "%(asctime)s %(name)-12s %(levelname)-8s:%(message)s", 7 | "datefmt": "%m/%d/%Y %I:%M:%S %p", 8 | "formatter": "%(name)-12s: %(levelname)-8s %(message)s" 9 | }, 10 | "postgresql": { 11 | "server":"localhost:5432", 12 | "database":"providence", 13 | "credential-identifier":"psqlcreds" 14 | }, 15 | "email": { 16 | "host": "mail.example.com", 17 | "to": "johndoe@example.com", 18 | "subject": "Providence Alert: ", 19 | "credential-identifier":null 20 | }, 21 | "repos": [ 22 | { 23 | "type": "perforce", 24 | "name": "PerforceRepo", 25 | "server": "ssl:perforce.com:8888", 26 | "directory": "//home/..." 27 | }, 28 | { 29 | "type": "github", 30 | "name": "PublicGithubTest", 31 | "server": "api.github.com", 32 | "owner": "providencedemo", 33 | "directory": "appsecdemo" 34 | } 35 | ], 36 | "cron": { 37 | "watcher_interval": "1" 38 | }, 39 | "plugins": { 40 | "enabled": [{ 41 | "basic_plugin_register":true, 42 | "basic_plugin_watcher":true, 43 | "forcedotcom_apex": true, 44 | "forcedotcom_aura_cmp_ui": true, 45 | "forcedotcom_aura_js": true, 46 | "forcedotcom_vf": true, 47 | "forcedotcom_pmd": true 48 | }] 49 | }, 50 | "pmd_path": "/path_to_pmd/pmd-dist/target/pmd-bin-5.6.0-SNAPSHOT/bin/run.sh" 51 | } 52 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | #-- Imports 17 | import json 18 | from lib.helpers import * 19 | 20 | class Configuration(object): 21 | config_filename = "config.json" 22 | 23 | def __init__(self,configFile=None): 24 | if configFile is None: 25 | configFile = Configuration.config_filename 26 | self.__configFile = configFile 27 | self.__load() 28 | 29 | def __load(self): 30 | with open(self.__configFile) as fp: 31 | self.__configData = json.load(fp, object_hook=convert) 32 | 33 | def get(self,keys,data=None,default=None): 34 | if type(keys) is tuple and len(keys) > 1: 35 | if not data: 36 | retData = self.get(keys[1:],self.__configData.get(keys[0])) 37 | if retData is None: 38 | return default 39 | return retData 40 | else: 41 | retData = self.get(keys[1:],data[keys[0]]) 42 | if retData is None: 43 | return default 44 | return retData 45 | else: 46 | if type(keys) is str: 47 | retData = data.get(keys, default) if data else self.__configData.get(keys, default) 48 | else: 49 | retData = data.get(keys[0], default) if data else self.__configData.get(keys[0], default) 50 | if type(retData) is dict: 51 | raise Exception("Cannot return multiple entries") 52 | else: 53 | return retData 54 | 55 | def set(self,keys,value,data=None): 56 | pass 57 | 58 | def persist(self): 59 | pass 60 | 61 | credential_manager = None 62 | -------------------------------------------------------------------------------- /credentials.json.example: -------------------------------------------------------------------------------- 1 | { 2 | "psqlcreds": { 3 | "type":"password", 4 | "username":"", 5 | "password":"" 6 | }, 7 | "github": { 8 | "type":"password", 9 | "username":"", 10 | "password":"" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | from sqlalchemy.engine import create_engine 17 | 18 | 19 | import config 20 | configuration = config.Configuration() 21 | 22 | def get_engine(): 23 | credentials_id = configuration.get(('postgresql','credential-identifier')) 24 | server_name = configuration.get(('postgresql','server')) 25 | database_name = configuration.get(('postgresql','database')) 26 | 27 | cred_url_part = "" 28 | if credentials_id is not None: 29 | creds = config.credential_manager.get_or_create_credentials_for(credentials_id, "password") 30 | cred_url_part = "%s:%s@" % (creds.username, creds.password) 31 | connectionurl = 'postgresql://%s%s/%s' % (cred_url_part, server_name, database_name) 32 | return create_engine(connectionurl) 33 | 34 | engine = get_engine() 35 | 36 | from sqlalchemy.orm import sessionmaker 37 | from sqlalchemy.orm.exc import NoResultFound 38 | from sqlalchemy.exc import IntegrityError 39 | Session = sessionmaker(bind=engine) 40 | Session.configure(bind=engine) 41 | import copy 42 | 43 | def get_one_or_create(session, 44 | model, 45 | create_method='', 46 | create_method_kwargs=None, 47 | **kwargs): 48 | try: 49 | return session.query(model).filter_by(**kwargs).one(), False 50 | except NoResultFound: 51 | kwargs.update(create_method_kwargs or {}) 52 | created = getattr(model, create_method, model)(**kwargs) 53 | try: 54 | session.add(created) 55 | session.flush() 56 | return created, True 57 | except IntegrityError: 58 | session.rollback() 59 | return session.query(model).filter_by(**kwargs).one(), False -------------------------------------------------------------------------------- /detection/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /detection/behavior/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | 17 | -------------------------------------------------------------------------------- /detection/behavior/membertracker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import re 17 | import os 18 | from time import sleep 19 | from models import Repo, RepoDocument 20 | from db import Session, get_one_or_create 21 | import config 22 | import copy 23 | 24 | class MemberTracker(object): 25 | def __init__(self, repository_name, new_members=True, odd_hours=False): 26 | self.session = Session(); 27 | self.repository_name = repository_name; 28 | self.new_members = new_members; 29 | 30 | @classmethod 31 | def doc_name(cls, member_identifier): 32 | return "%s" % (member_identifier,) 33 | 34 | @classmethod 35 | def tool_name(cls): 36 | return "Detection::Behavior::Members" 37 | 38 | def check_if_new(self, member_identifier, add_if_new=True, alert_function=None): 39 | if member_identifier is None: 40 | return False 41 | name = MemberTracker.doc_name(member_identifier) 42 | tool = MemberTracker.tool_name() 43 | repo, did_add = get_one_or_create(self.session, Repo, name=self.repository_name) 44 | members_doc, did_add = get_one_or_create(self.session, RepoDocument, repo=repo, tool=tool, name=name) 45 | if did_add: 46 | members_doc.data = {} 47 | if alert_function: 48 | alert_function(member_identifier) 49 | self.session.commit() 50 | return True 51 | self.session.commit() 52 | return False 53 | -------------------------------------------------------------------------------- /detection/pattern/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /detection/pattern/regex_rule_engine/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import re 17 | import cgi 18 | import jinja2 19 | import os 20 | from email.mime.text import MIMEText 21 | from email.mime.multipart import MIMEMultipart 22 | 23 | from alerts import Alert 24 | from alerts.email_alert import EmailAlert 25 | 26 | import config 27 | import logging 28 | logger = logging.getLogger(__name__) 29 | 30 | class RegexRuleEngine(object): 31 | def __init__(self, config=None, test_mode=False): 32 | self.alert_queue = [] 33 | if config is None: 34 | config={} 35 | self._alert_configs = None 36 | self._log_configs = None 37 | self._regexes = None 38 | self._rules = None 39 | self._tests = None 40 | self._name = None 41 | self._version = None 42 | 43 | self.load_config(config) 44 | self.test_mode = test_mode 45 | 46 | def load_config(self, config): 47 | if self._alert_configs is None: 48 | self._alert_configs = {} 49 | if self._log_configs is None: 50 | self._log_configs = [] 51 | if self._rules is None: 52 | self._rules = [] 53 | if self._regexes is None: 54 | self._regexes = {} 55 | if self._tests is None: 56 | self._tests = {} 57 | 58 | self._name = config.get("name",self._name) 59 | self._version = config.get("version",self._version) 60 | 61 | self._alert_configs.update(config.get("alert configs",[])) 62 | self._log_configs.extend(config.get("log configs",[])) 63 | 64 | self._regexes.update(config.get("regexes",{})) 65 | self._rules.extend(config.get("regex_rules",[])) 66 | 67 | self._tests.update(config.get("tests",{})) 68 | 69 | def _if_rule(self, data, rule, alert_callback): 70 | if_conditions = rule.get("if") 71 | then_rules = rule.get("then") 72 | else_rules = rule.get("else") 73 | 74 | if data is None or if_conditions is None: 75 | return 76 | 77 | if_condition_met = True 78 | for if_condition in if_conditions: 79 | if_condition = self._regexes.get(if_condition, if_condition) 80 | ignorecase = re.compile(if_condition, re.IGNORECASE | re.MULTILINE | re.DOTALL) 81 | if ignorecase.search(data) is None: 82 | if_condition_met = False 83 | break 84 | 85 | if if_condition_met: 86 | if then_rules is not None: 87 | for then_rule in then_rules: 88 | self._process(data, then_rule, alert_callback) 89 | else: 90 | if else_rules is not None: 91 | for else_rule in else_rules: 92 | self._process(data, else_rule, alert_callback) 93 | 94 | def _alert_rule(self, data, rule, alert_callback): 95 | alert_actions = rule.get("alert") 96 | for alert_action in alert_actions: 97 | if alert_callback: 98 | alert_callback(alert_action) 99 | 100 | def _process(self, data, rule, alert_callback): 101 | if rule.get("if") is not None: 102 | self._if_rule(data, rule, alert_callback) 103 | if rule.get("alert"): 104 | self._alert_rule(data, rule, alert_callback) 105 | 106 | def process(self, data, alert_callback=None): 107 | for rule in self._rules: 108 | self._process(data, rule, alert_callback) 109 | 110 | class RepoPatchRegexRuleEngine(RegexRuleEngine): 111 | def create_alert_email(self, subject, alert_message, repo_commit): 112 | alert_message = alert_message.decode("utf8") 113 | templateVars = { "title" : subject, 114 | "description" : "Description", 115 | "rule" : "Rule", 116 | "github_link" : repo_commit.url, 117 | "code" : alert_message } 118 | 119 | templateEnv = jinja2.Environment(autoescape=True, 120 | loader=jinja2.FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates'))) 121 | 122 | #templateLoader = jinja2.FileSystemLoader( os.path.join(os.path.dirname(__file__)) ) 123 | #templateEnv = jinja2.Environment( loader=templateLoader ) 124 | 125 | html_template_file = os.path.join("ruleengine_github.html") 126 | text_template_file = os.path.join("ruleengine_github.txt") 127 | 128 | html_template = templateEnv.get_template( html_template_file ) 129 | text_template = templateEnv.get_template( text_template_file ) 130 | 131 | html = html_template.render( templateVars ) 132 | text = text_template.render( templateVars ) 133 | return (text, html) 134 | 135 | def process(self, repo_patch, alert_callback=None): 136 | data = u'\n'.join(repo_patch.diff.additions).encode('utf-8').strip() 137 | def _alert_callback(alert_action): 138 | alert_config_key = alert_action.get("alert config") 139 | alert_config = self._alert_configs.get(alert_config_key) 140 | if alert_config is None: 141 | logger.error("Alert config for [%s] is None", alert_config_key); 142 | return 143 | if alert_config.get("email"): 144 | default_email = config.Configuration('config.json').get(('email', 'to')) 145 | to_email = alert_config.get("email", default_email) 146 | patch_lines = u'\n'.join(repo_patch.diff.additions).encode('utf-8').strip() 147 | subject = alert_action.get("subject","Unknown Subject") 148 | (text, html) = self.create_alert_email(subject, data, repo_patch.repo_commit) 149 | ea = EmailAlert(Alert(subject=subject, 150 | message=text.encode('utf-8'), 151 | message_html=html.encode('utf-8')), 152 | to_email=to_email) 153 | if (self.test_mode == True): 154 | print ea 155 | else: 156 | ea.send() 157 | else: 158 | logger.warn("Alert type unknown %s" % (alert_config)) 159 | 160 | if alert_callback is None: 161 | alert_callback = _alert_callback 162 | #data = repo_patch 163 | for rule in self._rules: 164 | self._process(data, rule, alert_callback) 165 | 166 | -------------------------------------------------------------------------------- /detection/pattern/regex_rule_engine/templates/ruleengine_github.txt: -------------------------------------------------------------------------------- 1 | {{ title }} 2 | 3 | Description 4 | ----------- 5 | {{ description }} 6 | 7 | Rule 8 | ---- 9 | {{ rule }} 10 | 11 | Link 12 | ---- 13 | {{ github_link }} 14 | 15 | Code 16 | ---- 17 | {{ code }} -------------------------------------------------------------------------------- /detection/pattern/regexruleengine.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import re 17 | import cgi 18 | import jinja2 19 | import os 20 | from email.mime.text import MIMEText 21 | from email.mime.multipart import MIMEMultipart 22 | 23 | from alerts import Alert 24 | from alerts.email_alert import EmailAlert 25 | 26 | import config 27 | 28 | class RegexRuleEngine(object): 29 | def __init__(self, config=None): 30 | if config is None: 31 | config={} 32 | self._alert_configs = config.get("alert configs",[]); 33 | self._log_configs = config.get("log configs",[]); 34 | self._rules = config.get("regex_rules",[]) 35 | self._tests = config.get("tests",{}) 36 | self._name = config.get("name","unknown") 37 | self._version = config.get("version",0) 38 | 39 | def load_config(self, config): 40 | if self._alert_configs is None: 41 | self._alert_configs = [] 42 | if self._log_configs is None: 43 | self._log_configs = [] 44 | if self._rules is None: 45 | self._rules = [] 46 | if self._tests is None: 47 | self._tests = {} 48 | 49 | self._alert_configs += config.get("alert configs",[]) 50 | self._log_configs += config.get("log configs",[]) 51 | self._rules += config.get("regex_rules",[]) 52 | self._tests.update(config.get("tests",{})) 53 | self._name = config.get("name",self._name) 54 | self._version = config.get("version",self._version) 55 | 56 | 57 | def send_alert_email(self, from_email, to_email, subject, alert_message, repo_commit): 58 | alert_message = alert_message.decode("utf8") 59 | templateVars = { "title" : subject, 60 | "description" : "Description", 61 | "rule" : "Rule", 62 | "github_link" : repo_commit.url, 63 | "code" : alert_message } 64 | 65 | templateLoader = jinja2.FileSystemLoader( searchpath="/" ) 66 | templateEnv = jinja2.Environment( loader=templateLoader ) 67 | 68 | html_template_file = os.path.join(os.path.dirname(__file__),"templates/ruleengine_github.html") 69 | text_template_file = os.path.join(os.path.dirname(__file__),"templates/ruleengine_github.txt") 70 | 71 | html_template = templateEnv.get_template( html_template_file ) 72 | text_template = templateEnv.get_template( text_template_file ) 73 | 74 | html = html_template.render( templateVars ) 75 | text = text_template.render( templateVars ) 76 | 77 | EmailAlert().send(Alert(subject=subject, message=text.encode('utf-8'), message_html=html.encode('utf-8')), to_email=to_email) 78 | 79 | def _process_ruleset(self, all_lines, repo_patch, ruleset, run_tests=False, custom_match_callback=None, offending_line=None): 80 | if_conditions = ruleset.get("if") 81 | return_actions = {"actions":[],"logs":[], "matches":[]} 82 | if if_conditions: 83 | for line in all_lines: 84 | line = line.strip() 85 | if line.startswith("//"): 86 | continue 87 | if_condition_met = True 88 | matches = [] 89 | for if_condition in if_conditions: 90 | ignorecase = re.compile(if_condition, re.IGNORECASE | re.MULTILINE | re.DOTALL) 91 | if ignorecase.search(line) is None: 92 | if_condition_met = False 93 | break 94 | matches.append(if_condition) 95 | if if_condition_met: 96 | return_actions["matches"].extend(matches) 97 | then_rulesets = ruleset.get("then") 98 | if (then_rulesets): 99 | for then_ruleset in then_rulesets: 100 | sub_actions = self._process_ruleset(all_lines, repo_patch, then_ruleset, run_tests=run_tests, custom_match_callback=custom_match_callback, offending_line=line) 101 | return_actions["actions"] += sub_actions["actions"] 102 | return_actions["logs"] += sub_actions["logs"] 103 | return_actions["matches"] += sub_actions["matches"] 104 | break 105 | else_rulesets = ruleset.get("else") 106 | if (else_rulesets and if_condition_met == False): 107 | for else_ruleset in else_rulesets: 108 | sub_actions += self._process_ruleset(all_lines, repo_patch, else_ruleset, run_tests=run_tests, custom_match_callback=custom_match_callback) 109 | return_actions["actions"] += sub_actions["actions"] 110 | return_actions["logs"] += sub_actions["logs"] 111 | return_actions["matches"] += sub_actions["matches"] 112 | return return_actions 113 | alert_actions = ruleset.get("alert") 114 | if alert_actions: 115 | for alert_action in alert_actions: 116 | return_actions["actions"].append(alert_action) 117 | if (run_tests == True): 118 | continue 119 | alert_config_key = alert_action.get("alert config") 120 | alert_config = self._alert_configs.get(alert_config_key) 121 | if alert_config is None: 122 | logger.error("Alert config for [%s] is None", alert_config_key); 123 | continue 124 | if alert_config.get("email"): 125 | default_email = config.Configuration('config.json').get(('email', 'to')) 126 | to = alert_config.get("email", default_email) 127 | patch_lines = u'\n'.join(repo_patch.diff.additions).encode('utf-8').strip() 128 | self.send_alert_email(default_email, 129 | to, 130 | alert_action.get("subject","Unknown Subject"), 131 | patch_lines, 132 | repo_patch.repo_commit); 133 | if alert_config.get("custom"): 134 | if custom_match_callback: 135 | custom_match_callback(alert_config, alert_action, repo_patch, all_lines=all_lines, offending_line=offending_line) 136 | return return_actions 137 | log_actions = ruleset.get("log") 138 | if log_actions: 139 | for log_action in log_actions: 140 | pass 141 | return return_actions 142 | 143 | def test(self): 144 | results = [] 145 | results.append("************************* Rule Engine Test *****************************") 146 | results.append("Test suite for %s version %s" % (self._name, self._version)); 147 | if self._tests is None: 148 | results.append("[ No tests found ]"); 149 | return (False, "\n".join(results)) 150 | fail_count = 0 151 | pass_count = 0 152 | for test,expected_result in self._tests.items(): 153 | results.append(" %s..." % (test,)); 154 | try: 155 | result = {"actions":[],"logs":[],"matches":[]} 156 | for ruleset in self._rules: 157 | sub_results = self._process_ruleset([test], None, ruleset, run_tests=True) 158 | result["actions"] += sub_results["actions"] 159 | result["logs"] += sub_results["logs"] 160 | result["matches"] += sub_results["matches"] 161 | if (sorted(result['matches']) == sorted(expected_result)): 162 | results.append(" ...passed"); 163 | pass_count += 1 164 | else: 165 | results.append(" ...failed"); 166 | fail_count += 1 167 | except Exception as e: 168 | results.append("!exception! %s" % (str(e),)); 169 | results.append(" passed: %d / failed: %d" % (pass_count, fail_count)); 170 | results.append("************************************************************************"); 171 | if fail_count > 0: 172 | return (False, "\r\n".join(results)) 173 | return (True, "\n".join(results)) 174 | 175 | def match(self, all_lines, repo_patch, custom_match_callback=None): 176 | action_sets = [] 177 | for ruleset in self._rules: 178 | action_sets.append(self._process_ruleset(all_lines, repo_patch, ruleset, custom_match_callback=custom_match_callback)) 179 | -------------------------------------------------------------------------------- /detection/pattern/templates/ruleengine_github.txt: -------------------------------------------------------------------------------- 1 | {{ title }} 2 | 3 | Description 4 | ----------- 5 | {{ description }} 6 | 7 | Rule 8 | ---- 9 | {{ rule }} 10 | 11 | Link 12 | ---- 13 | {{ github_link }} 14 | 15 | Code 16 | ---- 17 | {{ code }} -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | 17 | -------------------------------------------------------------------------------- /lib/helpers.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | #-- Functions for 17 | # Because unicode can die in a fire... 18 | # http://stackoverflow.com/questions/956867/how-to-get-string-objects-instead-of-unicode-ones-from-json-in-python 19 | def convert(input): 20 | if isinstance(input, dict): 21 | return {convert(key): convert(value) for key, value in input.iteritems()} 22 | elif isinstance(input, list): 23 | return [convert(element) for element in input] 24 | elif isinstance(input, unicode): 25 | return input.encode('utf-8') 26 | else: 27 | return input 28 | -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, UniqueConstraint, Text 17 | from sqlalchemy.ext.declarative import declarative_base 18 | from sqlalchemy.orm import relationship, backref 19 | Base = declarative_base() 20 | from sqlalchemy.dialects.postgresql import JSONB, JSON 21 | 22 | class Plugin(Base): 23 | __tablename__ = 'plugin' 24 | id = Column(Integer, primary_key=True) 25 | name = Column(String()) 26 | last_attempted_run = Column(DateTime) 27 | last_successful_run = Column(DateTime) 28 | type = Column(String(50)) 29 | documents = relationship("PluginDocument", backref="plugin") 30 | __table_args__ = ( 31 | UniqueConstraint('name'), 32 | ) 33 | 34 | class Repo(Base): 35 | __tablename__ = 'repo' 36 | id = Column(Integer, primary_key=True) 37 | name = Column(String()) 38 | last_attempted_run = Column(DateTime) 39 | last_successful_run = Column(DateTime) 40 | last_identifier = Column(String()) 41 | documents = relationship("RepoDocument", backref="repo") 42 | __table_args__ = ( 43 | UniqueConstraint('name'), 44 | ) 45 | 46 | class Document(Base): 47 | __tablename__ = 'document' 48 | id = Column(Integer, primary_key=True) 49 | name = Column(String()) 50 | data = Column(JSONB) 51 | text = Column(Text()) 52 | __table_args__ = ( 53 | UniqueConstraint('name'), 54 | ) 55 | 56 | class RepoDocument(Base): 57 | __tablename__ = 'repo_document' 58 | id = Column(Integer, primary_key=True) 59 | repo_id = Column(Integer, ForeignKey('repo.id')) 60 | name = Column(String()) 61 | data = Column(JSONB) 62 | text = Column(Text()) 63 | tool = Column(String(50)) 64 | date = Column(DateTime(timezone=True)) 65 | counter = Column(Integer) 66 | __table_args__ = ( 67 | UniqueConstraint('repo_id','tool','name'), 68 | ) 69 | 70 | class PluginDocument(Base): 71 | __tablename__ = 'plugin_document' 72 | id = Column(Integer, primary_key=True) 73 | plugin_id = Column(Integer, ForeignKey('plugin.id')) 74 | name = Column(String()) 75 | data = Column(JSONB) 76 | text = Column(Text()) 77 | date = Column(DateTime(timezone=True)) 78 | counter = Column(Integer) 79 | __table_args__ = ( 80 | UniqueConstraint('name','plugin_id'), 81 | ) 82 | -------------------------------------------------------------------------------- /plugins/README.md: -------------------------------------------------------------------------------- 1 | Plugins 2 | ========== 3 | Providence plugins live in this directory. 4 | 5 | ## Directory Structure 6 | ``` 7 | plugins 8 | |── basic_plugin_register 9 | | └── main.py 10 | |── basic_plugin_watcher 11 | | |── main.py 12 | | |── js.json 13 | | └── java.json 14 | |── forcedotcom_base_watcher 15 | | └── __init__.py 16 | |── forcedotcom_apex 17 | | |-- main.py 18 | | └── regex.json 19 | |── other forcedotcom plugins 20 | └── base.py 21 | ``` 22 | 23 | ## Plugin Roles 24 | Plugins can contain any of the following Roles, and Providence will execute them as described. One plugin can handle any number of Roles, but the sample plugins here separates them for ease-of-maintenance. 25 | 26 | Plugins inherits the parent Plugin object from base.py and must define the method 27 | ``` 28 | test() 29 | ``` 30 | 31 | ### Repository Registers 32 | Repository register plugins sets up the code commit/bug systems for monitoring. It links each repository listed in `config.json` to its corresponding credential 33 | and sets up the monitoring interval. 34 | 35 | Must define the method 36 | ``` 37 | register_repositories() 38 | ``` 39 | 40 | ### Watchers 41 | Watcher plugins are for monitoring commits. These plugins use regex rules to search for patterns in a commit, and sends an alert if there is a match. The alert logic (such as sending email or creating a new work in your favorite bug system) is also defined here. The plsql database keeps a timestamp of the last time commits were processed, and will pull commits since that timestamp every time it executes. You can specify the execution interval in "watcher_interval" in `config.json` 42 | 43 | Must define the methods 44 | ``` 45 | register_watcher() 46 | commit_started() 47 | patch() 48 | commit_finished() 49 | ``` 50 | 51 | ### Hourly/Seven minutes 52 | 53 | Hourly and Seven Minute plugins run as often as their name states. These plugins are better suited for bug system monitoring instead of commit monitoring. 54 | 55 | Must have one of these defined 56 | ``` 57 | run_hourly() 58 | run_seven_minutes() 59 | ``` 60 | 61 | 62 | ## Sample Plugins 63 | ### basic_plugin_register 64 | Registers the Perforce and Github repos 65 | 66 | ### basic_plugin_watcher 67 | Looks for potential vulnerabilities in .js and .java files 68 | 69 | 1. `java.json` will alert for possible instances of XXE in java 70 | 2. `js.json` contains warnings for risky JS coding 71 | 72 | ### forcedotcom_base_watcher 73 | Parent class of other forcedotcom watcher plugins. Contains the hooks (`register_watcher()`, `commit_started()` etc) for providence.py but does not have any actual rules. The child classes contain the actual regex and rules. You do not need to include this file in your `config.json` 74 | 75 | This and the child forcedotcom plugins are a great way to monitor issues that we check for during the AppExchange security review. 76 | 77 | ### forcedotcom_apex 78 | Looks for common security issues in Apex classes 79 | 80 | ### forcedotcom_aura_cmp_ui 81 | Looks for common security issues on the UI side of Aura components 82 | 83 | ### forcedotcom_aura_js 84 | Looks for common security issues on the controller/helper sides of Aura components 85 | 86 | ### forcedotcom_vf 87 | Looks for issues on VisualForce and javascript pages 88 | 89 | ### forcedotcom_pmd 90 | Looks for common security issues in Apex classes and Visualforce pages, using the source code analyzer [PMD](https://pmd.github.io/). PMD is run against each commit, and the resulting findings are sent out as Providence alerts. 91 | This plugin requires additional [setup](forcedotcom_pmd/README.md) 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /plugins/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import imp 17 | import os 18 | import sys 19 | import importlib 20 | import config 21 | import logging 22 | 23 | logger = logging.getLogger(__name__) 24 | 25 | PluginFolder = "./plugins" 26 | MainModule = "main" 27 | 28 | path = "plugins/" 29 | 30 | class Plugins(object): 31 | def __init__(self,configuration): 32 | self._pluginFolder = configuration.get(('plugins','folder'), default="./plugins") 33 | self._mainModule = configuration.get(('plugins','main_module'), default="main.py") 34 | self._pluginTypes = configuration.get(('plugins','plugin_types')) 35 | self._registeredPlugins = {} 36 | self._found_plugins = [] 37 | _enabled_plugin_names = configuration.get(('plugins','enabled'), default=[{}]) 38 | self._enabled_plugin_names = _enabled_plugin_names[0] 39 | 40 | def _findPlugins(self, enabled_dict): 41 | _found_plugins = [] 42 | return _found_plugins 43 | 44 | def enabled_plugins(self): 45 | return self.all_plugins(self._enabled_plugin_names) 46 | 47 | def all_plugins(self, enabled_dict=None): 48 | plugins = {"repositories":[], 49 | "watchers":[], 50 | "hourly":[], 51 | "seven_minutes":[] } 52 | possiblePlugins = os.listdir(self._pluginFolder) 53 | for possible_plugin in possiblePlugins: 54 | if not os.path.isdir(os.path.join(self._pluginFolder,possible_plugin)): 55 | continue 56 | location = os.path.join(self._pluginFolder,possible_plugin) 57 | for pluginFile in os.listdir(location): 58 | if pluginFile != self._mainModule: 59 | continue 60 | plugin_path = location.split("/") 61 | plugin_name = plugin_path[-1] 62 | enabled = True 63 | if enabled_dict is not None: 64 | enabled = enabled_dict.get(plugin_name, False) 65 | if enabled != True: 66 | continue 67 | plugin_mod_name = location[2:] 68 | main = imp.load_source(plugin_mod_name,os.path.join(location,"main.py")) 69 | plugin = main.Plugin() 70 | if hasattr(plugin, 'test') == False: 71 | print "test() function required for ", info 72 | continue 73 | if hasattr(plugin, 'request_credentials'): 74 | creds_requested = plugin.request_credentials() 75 | if creds_requested is not None: 76 | for cred_requested in creds_requested: 77 | if type(cred_requested) is dict: 78 | cred_name = cred_requested["name"] 79 | cred_type = cred_requested["type"] 80 | creds = config.credential_manager.get_or_create_credentials_for(cred_name, cred_type) 81 | if creds is None: 82 | continue 83 | if hasattr(plugin, 'register_repositories'): 84 | plugins["repositories"].append(plugin) 85 | if hasattr(plugin, 'register_watcher'): 86 | plugins["watchers"].append(plugin) 87 | if hasattr(plugin, 'run_hourly'): 88 | plugins["hourly"].append(plugin) 89 | if hasattr(plugin, 'run_seven_minutes'): 90 | plugins["seven_minutes"].append(plugin) 91 | return plugins 92 | 93 | def get_repositories(self, repository_plugins, tests=False): 94 | repositories = {} 95 | for plugin in repository_plugins: 96 | try: 97 | if hasattr(plugin, 'register_repositories'): 98 | plugin_repos = plugin.register_repositories(); 99 | repositories.update(plugin_repos); 100 | except Exception, e: 101 | logger.exception(e) 102 | return repositories 103 | 104 | def get_watchers(self, watcher_plugins, tests=False): 105 | watchers = {} 106 | wildcards = [] 107 | for plugin in watcher_plugins: 108 | try: 109 | repo_watchers = plugin.register_watcher(); 110 | for repo_watcher in repo_watchers: 111 | if repo_watcher.repo_identifier == "*": 112 | repo_watcher.path = None 113 | wildcards.append(repo_watcher) 114 | continue 115 | ident = repo_watcher.repo_identifier 116 | if watchers.get(ident): 117 | watchers[ident].append(repo_watcher) 118 | else: 119 | watchers[ident] = [repo_watcher] 120 | except Exception, e: 121 | logger.exception(e) 122 | 123 | for repo_identifier, repo_watchers in watchers.iteritems(): 124 | categorized_by_path = RepoWatcher.categorize_by_path(repo_watchers) 125 | for path, path_watchers in categorized_by_path.iteritems(): 126 | path_watchers.extend(wildcards) 127 | watchers[repo_identifier] = categorized_by_path 128 | return watchers 129 | 130 | 131 | def _validateSinglePlugin(self): 132 | pass 133 | 134 | def _loadSinglePlugin(self): 135 | pass 136 | 137 | 138 | class RepoWatcher(object): 139 | ALL=0 140 | GITHUB=1 141 | PERFORCE=2 142 | STASH=3 143 | 144 | def __init__(self, plugin, repo_type, repo_identifier, path=None): 145 | self.plugin = plugin 146 | self.repo_type = repo_type 147 | self.repo_identifier = repo_identifier 148 | self.path = path 149 | self.start_time_utc = None 150 | self.end_time_utc = None 151 | self.owner = None 152 | self.subpaths = [] 153 | 154 | def search_path(self): 155 | if self.repo_type == RepoWatcher.PERFORCE: 156 | return self.path + "..." 157 | return self.path 158 | 159 | 160 | @staticmethod 161 | def _list_loop(repo_watcher, repo_watchers): 162 | reduced_paths = [repo_watcher] 163 | while len(repo_watchers) > 0: 164 | nextpath = repo_watchers[-1].path 165 | if nextpath is None: 166 | nextpath = "" 167 | if repo_watcher.path is None and nextpath is not None: 168 | return reduced_paths 169 | elif repo_watcher.path != nextpath: 170 | nextpath = nextpath + "/" 171 | prefix = repo_watcher.path + "/" 172 | if nextpath.startswith(prefix) == False: 173 | return reduced_paths 174 | next_watcher = repo_watchers.pop() 175 | reduced_paths.append(next_watcher) 176 | #reduced_paths[path] = list_loop(path, paths) 177 | return reduced_paths 178 | 179 | @staticmethod 180 | def categorize_by_path(repo_watchers): 181 | reduced_paths = {} 182 | repo_watchers.sort(key=lambda x: x.path, reverse=True) 183 | while len(repo_watchers) > 0: 184 | repo_watcher = repo_watchers.pop() 185 | if (repo_watcher.path is None): 186 | repo_watcher.path = "" 187 | reduced_paths[repo_watcher.path] = RepoWatcher._list_loop(repo_watcher, repo_watchers) 188 | return reduced_paths 189 | 190 | 191 | if __name__ == "__main__": 192 | repo_watchers = [] 193 | repo_watchers.append(RepoWatcher(None, RepoWatcher.GITHUB, "*","/a")) 194 | repo_watchers.append(RepoWatcher(None, RepoWatcher.GITHUB, "x","/a")) 195 | repo_watchers.append(RepoWatcher(None, RepoWatcher.GITHUB, "x","/b/c/d")) 196 | repo_watchers.append(RepoWatcher(None, RepoWatcher.GITHUB, "x","/b")) 197 | 198 | watchers = {} 199 | wildcards = [] 200 | for repo_watcher in repo_watchers: 201 | if repo_watcher.repo_identifier == "*": 202 | repo_watcher.path = None 203 | wildcards.append(repo_watcher) 204 | continue 205 | ident = repo_watcher.repo_identifier 206 | if watchers.get(ident): 207 | watchers[ident].append(repo_watcher) 208 | else: 209 | watchers[ident] = [repo_watcher] 210 | 211 | for repo_identifier, repo_watchers in watchers.iteritems(): 212 | print repo_identifier 213 | 214 | categorized_by_path = RepoWatcher.categorize_by_path(repo_watchers) 215 | for path, path_watchers in categorized_by_path.iteritems(): 216 | path_watchers.extend(wildcards) 217 | watchers[repo_identifier] = categorized_by_path 218 | 219 | print watchers 220 | 221 | """ 222 | def list_loop(prefix, paths): 223 | reduced_paths = [prefix] 224 | while len(paths) > 0: 225 | if paths[-1].startswith(prefix) == False: 226 | return reduced_paths 227 | path = paths.pop() 228 | reduced_paths.append(path) 229 | #reduced_paths[path] = list_loop(path, paths) 230 | return reduced_paths 231 | 232 | def reduce_paths(paths): 233 | reduced_paths = {} 234 | paths.sort(reverse=True) 235 | while len(paths) > 0: 236 | path = paths.pop() 237 | reduced_paths[path] = list_loop(path, paths) 238 | print reduced_paths 239 | 240 | paths = ['/a/b/c','/x/b','/a/b','/a/g','/a/g/d/e'] 241 | reduce_paths(paths) 242 | """ 243 | -------------------------------------------------------------------------------- /plugins/base.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | class PluginTestError(Exception): 17 | pass 18 | 19 | class Plugin(object): 20 | def __init__(self): 21 | self.credentials = {} 22 | self.version = 0.0 23 | 24 | def __getattribute__(self, name): 25 | if name == "name": 26 | return self.__module__ 27 | else: 28 | return object.__getattribute__(self, name) 29 | 30 | def test(self): 31 | raise PluginTestError("No Test Function Defined") 32 | 33 | def request_credentials(credential_identifiers): 34 | return None 35 | 36 | def write_credential(creds): 37 | pass -------------------------------------------------------------------------------- /plugins/basic_plugin_register/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import sys 17 | import os.path 18 | sys.path.append( 19 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 20 | from repos.perforce import PerforceSource 21 | from repos.github import GithubSource 22 | from detection.pattern.regexruleengine import RegexRuleEngine 23 | from Empire.creds.credentials import Credentials 24 | from plugins import base, RepoWatcher 25 | import logging 26 | import config 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | class Plugin(base.Plugin): 31 | # Example of a repository register plugin. Only contains the register_repositories() method. 32 | 33 | def register_repositories(self): 34 | # links each repo in config.json to their corresponding credentials 35 | logger.debug("registering repository") 36 | 37 | configuration = config.Configuration('config.json') 38 | 39 | repos = {} 40 | for repo in configuration.get('repos'): 41 | repo_type = repo.get('type') 42 | if repo_type == 'perforce': 43 | repo_name = repo.get('name') 44 | creds = config.credential_manager.get_or_create_credentials_for(repo_name, "password") 45 | if creds is None: 46 | logger.error("Failed to load credentials") 47 | return {} 48 | repo_source = PerforceSource(creds=creds, port=repo.get('server'), directory=repo.get('directory')) 49 | repos[repo_name] = {"source":repo_source, "check-every-x-minutes":10} 50 | elif repo_type == 'github': 51 | repo_name = repo.get('name') 52 | creds = config.credential_manager.get_or_create_credentials_for(repo_name, "password") 53 | if creds is None: 54 | logger.error("Failed to load credentials") 55 | return {} 56 | repo_source = GithubSource(creds=creds, host=repo.get('server'), owner=repo.get('owner'), repo=repo.get('directory')) 57 | repos[repo_name] = {"source":repo_source, "check-every-x-minutes":10} 58 | else: 59 | print "Repo Type not supported yet: " + repo_type 60 | 61 | return repos 62 | 63 | def test(self): 64 | logger.warn("No tests") 65 | 66 | if __name__ == "__main__": 67 | test(); 68 | -------------------------------------------------------------------------------- /plugins/basic_plugin_watcher/java.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert configs":{ 3 | "alert group":{"custom":true} 4 | }, 5 | "regex_rules":[ 6 | {"if":["import\\s+javax\\.xml\\.((?!Exception)(?!bind\\.annotation\\.XmlAttribute).)*$"], 7 | "then":[ 8 | {"alert":[{"alert config":"alert group","subject":"XML Used"}]} 9 | ] 10 | }, 11 | {"if":["import\\s+org\\.xml\\.sax\\.(?!SAXException)"], 12 | "then":[ 13 | {"alert":[{"alert config":"alert group","subject":"XML Used"}]} 14 | ] 15 | }, 16 | {"if":["import\\s+org\\.jcp\\.xml\\."], 17 | "then":[ 18 | {"alert":[{"alert config":"alert group","subject":"XML Used"}]} 19 | ] 20 | }, 21 | {"if":["import\\s+sun\\.util\\.xml\\."], 22 | "then":[ 23 | {"alert":[{"alert config":"alert group","subject":"XML Used"}]} 24 | ] 25 | }, 26 | {"if":["import\\s+com\\.sun\\.org\\.apache\\.xml\\."], 27 | "then":[ 28 | {"alert":[{"alert config":"alert group","subject":"XML Used"}]} 29 | ] 30 | }, 31 | {"if":["import\\s+org\\.apache\\.commons\\.digester\\.Digester"], 32 | "then":[ 33 | {"alert":[{"alert config":"alert group","subject":"Digester Used"}]} 34 | ] 35 | } 36 | ], 37 | "tests":{ 38 | "import javax.xml.XMLReader":["import\\s+javax\\.xml\\.((?!Exception)(?!bind\\.annotation\\.XmlAttribute).)*$"] 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /plugins/basic_plugin_watcher/js.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert configs":{ 3 | "alert group":{"custom":true} 4 | }, 5 | "regex_rules":[ 6 | {"if":["http(s)?://.*\\.js"], 7 | "then":[ 8 | {"alert":[{"alert config":"alert group","subject":"External Included Script"}]} 9 | ] 10 | }, 11 | {"if":["eval\\(.*\\)"], 12 | "then":[ 13 | {"alert":[{"alert config":"alert group","subject":"Eval used"}]} 14 | ] 15 | }, 16 | {"if":["\\.postMessage\\("], 17 | "then":[ 18 | {"alert":[{"alert config":"alert group","subject":"Cross-domain communication"}]} 19 | ] 20 | }, 21 | {"if":["localStorage\\.setItem\\(['\"](password|token|key|secret)"], 22 | "then":[ 23 | {"alert":[{"alert config":"alert group","subject":"Using local storage for secrets"}]} 24 | ] 25 | } 26 | ], 27 | "tests":{ 28 | "":["\\s+escape\\s*=\\s*(\"|')?false\\1"], 29 | "":["http(s)?://.*\\.js"], 30 | "eval(abcsasdscsdfsdf12334%);":["eval\\(.*\\)"] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /plugins/basic_plugin_watcher/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import sys 17 | import os.path 18 | import json 19 | sys.path.append( 20 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 21 | from detection.pattern.regexruleengine import RegexRuleEngine 22 | from Empire.creds.credentials import Credentials 23 | from plugins import base, RepoWatcher 24 | import logging 25 | import config 26 | import re 27 | from alerts.email_alert import EmailAlert 28 | from alerts import Alert 29 | from Empire.creds import CredentialManager 30 | 31 | logger = logging.getLogger(__name__) 32 | class Plugin(base.Plugin): 33 | # Example of a watcher plugin. Implements the four required methods as described in readme 34 | 35 | # basic configurations 36 | configuration = config.Configuration('config.json') 37 | EMAIL_ALERT_RECIPIENTS = configuration.get(('email', 'to')) 38 | EMAIL_ALERT_SUBJECT = configuration.get(('email', 'subject')) 39 | if EMAIL_ALERT_SUBJECT is None or EMAIL_ALERT_SUBJECT is '': 40 | EMAIL_ALERT_SUBJECT = '[Providence Email Alert]' 41 | 42 | # some regex filename constants 43 | JAVA_SOURCE_FILE_PATTERN = "\.java$" 44 | JS_SOURCE_FILE_PATTERN = "\.js$" 45 | 46 | # anchor for last repo commit 47 | last_alert = None 48 | 49 | def register_watcher(self): 50 | # hooks each repository listed in config.json to the main process in providence.py 51 | repo_watchers = [] 52 | for repo in self.configuration.get('repos'): 53 | 54 | repo_type = repo.get('type') 55 | 56 | if repo_type == 'perforce': 57 | repo_type = RepoWatcher.PERFORCE 58 | elif repo_type == 'github': 59 | repo_type = RepoWatcher.GITHUB 60 | else: 61 | logger.warning('Repo Type \'%s\' not supported yet', repo_type) 62 | repo_type = RepoWatcher.ALL 63 | 64 | repo_watchers.append(RepoWatcher(self, repo_type, repo.get('name'))) 65 | 66 | return repo_watchers 67 | 68 | def commit_started(self, repository_name, repo_commit): 69 | # method called when Providence begins the commit processing 70 | logger.debug("processing repo %s on commit %s", repository_name, repo_commit.identifier) 71 | 72 | def patch(self, repository_name, repo_patch): 73 | # method called during the commit processing. Defines when an alert should be sent 74 | if re.search(self.JAVA_SOURCE_FILE_PATTERN, repo_patch.filename): 75 | regex_file_path = os.path.join(os.path.dirname(__file__),"java.json") 76 | else: 77 | regex_file_path = os.path.join(os.path.dirname(__file__),"js.json") 78 | logger.debug("filename %s processed", (repo_patch.filename)) 79 | all_lines = [] 80 | all_lines.extend(repo_patch.diff.additions) 81 | rule_engine = RegexRuleEngine(json.load(open(regex_file_path))) 82 | 83 | def custom_match_callback(alert_config, alert_action, repo_patch, all_lines, offending_line): 84 | # method called when the commit matches a regex rule. The actual sending of alerts (via email or bug tracking system) goes here 85 | self.send_alert(repo_patch, repo_patch.repo_commit, alert_action.get("subject"), offending_line) 86 | 87 | # Each line in the commit goes through the rule engine. If the line matches a regex rule, custom_match_callback() is called 88 | rule_engine.match(all_lines, repo_patch, custom_match_callback=custom_match_callback) 89 | return 90 | 91 | def commit_finished(self, repository_name, repo_commit): 92 | # method called when Providence finishes the commit processing 93 | pass 94 | 95 | def simple_html_encode(self, string): 96 | return string.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"').replace("'", ''') 97 | 98 | def send_alert(self, repo_patch, repo_commit, subject, offending_line): 99 | # Example method for sending alert whenever a rule is matched. 100 | url = repo_commit.url; 101 | filename = 'NOFILE' 102 | if repo_patch != None: 103 | filename = repo_patch.filename 104 | subject = self.EMAIL_ALERT_SUBJECT + " " + subject + ' in ' + repo_commit.identifier 105 | 106 | # skip if the alert is duplicate 107 | if url + filename + subject + offending_line == Plugin.last_alert: 108 | return 109 | 110 | # remember the current alert 111 | Plugin.last_alert = url + filename + subject + offending_line 112 | 113 | message = '' + url + ''+\ 114 | '

OFFENDING LINE
' + self.simple_html_encode(offending_line) +\ 115 | '

FILE NAME
' + filename 116 | 117 | # email alert 118 | alert = Alert(message='', message_html=message, subject=subject, level=Alert.HIGH) 119 | email_recipients = [self.EMAIL_ALERT_RECIPIENTS] 120 | EmailAlert().send(alert, to_email=email_recipients) 121 | 122 | 123 | def test(self, verbose=True): 124 | # Don't test non-regex based plugins 125 | if self.regex_file_path == "": 126 | return 127 | rule_engine = RegexRuleEngine() 128 | rule_engine.load_config(json.load(open(self.regex_file_path))) 129 | (success, results) = rule_engine.test() 130 | if verbose == True: 131 | logger.debug( results ) 132 | if success == False: 133 | raise base.PluginTestError(results) 134 | 135 | if __name__ == "__main__": 136 | test(); 137 | -------------------------------------------------------------------------------- /plugins/forcedotcom_apex/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import sys 17 | import os.path 18 | import json 19 | sys.path.append( 20 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 21 | from plugins.forcedotcom_base_watcher import ForceDotComBasePlugin 22 | import logging 23 | import config 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | class Plugin(ForceDotComBasePlugin): 28 | 29 | def __init__(self): 30 | regex_file_path = os.path.join(os.path.dirname(__file__),"regex.json") 31 | super(Plugin, self).__init__(ForceDotComBasePlugin.CLS_SOURCE_FILE_PATTERN, regex_file_path, self, logger) 32 | 33 | if __name__ == "__main__": 34 | test(); 35 | -------------------------------------------------------------------------------- /plugins/forcedotcom_apex/regex.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert configs":{ 3 | "alert group":{"custom":true} 4 | }, 5 | "regex_rules":[ 6 | {"if":["(?":["', '>').replace('"', '"').replace("'", ''') 94 | 95 | def send_alert(self, repo_patch, repo_commit, subject ,offending_line, message=''): 96 | # parameter setup 97 | url = repo_commit.url; 98 | filename = 'NOFILE' 99 | if repo_patch != None: 100 | filename = repo_patch.filename 101 | subject = self.EMAIL_ALERT_SUBJECT + " " + subject + ' in ' + repo_commit.identifier 102 | 103 | # skip if the alert is duplicate 104 | if url + filename + subject + offending_line == ForceDotComBasePlugin.last_alert: 105 | return 106 | 107 | # remember the current alert 108 | ForceDotComBasePlugin.last_alert = url + filename + subject + offending_line 109 | 110 | message = '' + url + ''+\ 111 | '

OFFENDING LINE
' + self.simple_html_encode(offending_line) +\ 112 | '

FILE NAME
' + filename +\ 113 | message 114 | 115 | # email alert 116 | alert = Alert(message='', message_html=message, subject=subject, level=Alert.HIGH) 117 | email_recipients = [self.EMAIL_ALERT_RECIPIENTS] 118 | EmailAlert().send(alert, to_email=email_recipients) 119 | 120 | 121 | def test(self, verbose=True): 122 | # Don't test non-regex based plugins 123 | if self.regex_file_path == "": 124 | return 125 | rule_engine = RegexRuleEngine() 126 | rule_engine.load_config(json.load(open(self.regex_file_path))) 127 | (success, results) = rule_engine.test() 128 | if verbose == True: 129 | self.logger.debug( results ) 130 | if success == False: 131 | raise base.PluginTestError(results) 132 | 133 | if __name__ == "__main__": 134 | test(); 135 | -------------------------------------------------------------------------------- /plugins/forcedotcom_pmd/README.md: -------------------------------------------------------------------------------- 1 | # PMD plugin 2 | This plugin uses a [fork](https://github.com/sgorbaty/pmd) of PMD to analyze Apex and Visualforce commits, and sends the resulting findings as Providence alerts. Since PMD is a source code analyzer, the findings here are generally more reliable than simple Regex matches provided by other plugins. Thanks to [Sergey Gorbaty](https://github.com/sgorbaty) for writing the Apex and Visualforce PMD rulesets. 3 | 4 | ## Requirements 5 | Java JRE 1.7+ 6 | Maven 7 | 8 | ## Setup 9 | ### 1. Clone the Apex/VF ruleset PMD fork 10 | git clone git@github.com:sgorbaty/pmd.git 11 | 12 | ### 2. Checkout the `stable` branch 13 | cd pmd 14 | git checkout stable 15 | 16 | ### 3. Configure PMD Toolchains 17 | See [How to Build PMD?](https://github.com/sgorbaty/pmd/blob/master/BUILDING.md) 18 | 19 | ### 4. Build PMD using Maven 20 | from the pmd directory 21 | 22 | mvn clean package 23 | 24 | ### 5. Unzip PMD project 25 | cd pmd-dist/target 26 | unzip pmd-bin-5.6.0-SNAPSHOT.zip 27 | 28 | ### 6. Update your config.json to point to the PMD startup script 29 | "pmd_path": "/path/to/pmd/pmd-dist/target/pmd-bin-5.6.0-SNAPSHOT/bin/run.sh" 30 | 31 | ### 7. Enable the plugin! 32 | "plugins": { 33 | "enabled": [{ 34 | ... 35 | "forcedotcom_pmd":true 36 | }] 37 | } 38 | 39 | ## About PMD 40 | 41 | [**PMD**](https://pmd.github.io/) is a source code analyzer. It finds common programming flaws like unused variables, empty catch blocks, 42 | unnecessary object creation, and so forth. It supports Java, JavaScript, Salesforce.com Apex, PLSQL, Apache Velocity, 43 | XML, XSL. 44 | 45 | Additionally it includes **CPD**, the copy-paste-detector. CPD finds duplicated code in 46 | Java, C, C++, C#, Groovy, PHP, Ruby, Fortran, JavaScript, PLSQL, Apache Velocity, Scala, Objective C, 47 | Salesforce.com Apex, Perl, Swift, Matlab, Python. 48 | 49 | -------------------------------------------------------------------------------- /plugins/forcedotcom_pmd/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | import sys 16 | import os.path 17 | sys.path.append( 18 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 19 | from plugins.forcedotcom_base_watcher import ForceDotComBasePlugin 20 | import logging 21 | import config 22 | import re 23 | from lxml import etree 24 | import tempfile 25 | import subprocess 26 | 27 | logger = logging.getLogger(__name__) 28 | class Plugin(ForceDotComBasePlugin): 29 | EMAIL_SUBJECT = "PMD rule '%s' triggered" 30 | pmd_path = "" 31 | 32 | def __init__(self): 33 | super(Plugin, self).__init__("", "", self, logger) 34 | self.pmd_path = self.configuration.get("pmd_path") 35 | if not self.pmd_path or self.pmd_path == '': 36 | logger.error("pmd_path not defined in config.json") 37 | raise Exception('pmd_path not defined!') 38 | 39 | def patch(self, repository_name, repo_patch): 40 | filename = repo_patch.filename 41 | # CLS_SOURCE_FILE_PATTERN = "\.(cls|apex)$" 42 | if re.search(ForceDotComBasePlugin.CLS_SOURCE_FILE_PATTERN, filename): 43 | file_type = '.cls' 44 | # VF page pattern = "\.page$" 45 | elif re.search("\.page$", filename): 46 | file_type = '.page' 47 | else: 48 | # Filetype not supported by pmd, do nothing 49 | return 50 | logger.debug("filename %s processing...", (filename)) 51 | file_content = repo_patch.repo_commit.repo_source.retrieve_file(filename) 52 | if file_content == '' or file_content is None: 53 | logger.error("%s repo probably did not implement retrieve_file()", repository_name) 54 | return 55 | # append the correct file extension 56 | temp = tempfile.NamedTemporaryFile(mode='w+t', suffix=file_type) 57 | pmd_output = None 58 | try: 59 | temp.write(file_content.encode('utf-8')) 60 | temp.seek(0) 61 | pmd_output = self.run_pmd(temp.name, file_type) 62 | except Exception as e: 63 | logger.exception("Error in salesforce_perforce_pmd %s", e) 64 | finally: 65 | temp.close() 66 | 67 | if pmd_output is not None: 68 | logger.debug("pmd found violations in %s", filename) 69 | lines = file_content.splitlines() 70 | pmd_xml = etree.fromstring(pmd_output) 71 | for v in pmd_xml.findall(".//violation"): 72 | begin_line = int(v.get("beginline")) - 1 73 | # only send alert if pmd results match change additions 74 | if lines[begin_line] in repo_patch.diff.additions: 75 | rule = v.get("rule") 76 | link = v.get("externalInfoUrl") 77 | info = v.text 78 | alert_msg = '

LINE NUMBERS
' + v.get("beginline") + ' - ' + v.get("endline") +\ 79 | '

RULE INFO
' + info 80 | super(Plugin, self).send_alert(repo_patch, repo_patch.repo_commit, self.EMAIL_SUBJECT % rule, lines[begin_line], alert_msg) 81 | 82 | def run_pmd(self, file, file_type): 83 | # returns the raw pmd output as string 84 | try: 85 | if file_type == '.cls': 86 | results = subprocess.check_output([self.pmd_path, "pmd", "-R", "apex-security", "-l", "apex", "-f", "xml", "-d", file]) 87 | elif file_type == '.page': 88 | results = subprocess.check_output([self.pmd_path, "pmd", "-R", "vf-security", "-l", "vf", "-f", "xml", "-d", file]) 89 | except subprocess.CalledProcessError as e: 90 | # pmd exits with 4 if violations are found 91 | if e.returncode == 4: 92 | return e.output 93 | else: 94 | logger.exception("Error executing pmd with code %s", e.returncode) 95 | return None 96 | 97 | if __name__ == "__main__": 98 | pass -------------------------------------------------------------------------------- /plugins/forcedotcom_vf/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import sys 17 | import os.path 18 | import json 19 | sys.path.append( 20 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) 21 | from plugins.forcedotcom_base_watcher import ForceDotComBasePlugin 22 | import logging 23 | import config 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | class Plugin(ForceDotComBasePlugin): 28 | 29 | def __init__(self): 30 | regex_file_path = os.path.join(os.path.dirname(__file__),"regex.json") 31 | super(Plugin, self).__init__(ForceDotComBasePlugin.APEXP_OR_JS_SOURCE_FILE_PATTERN, regex_file_path, self, logger) 32 | 33 | if __name__ == "__main__": 34 | test(); 35 | -------------------------------------------------------------------------------- /plugins/forcedotcom_vf/regex.json: -------------------------------------------------------------------------------- 1 | { 2 | "alert configs":{ 3 | "alert group":{"custom":true} 4 | }, 5 | "regex_rules":[ 6 | {"if":["\\s+escape\\s*=\\s*(\"|')?false\\1"], 7 | "then":[ 8 | {"alert":[{"alert config":"alert group","subject":"Unescaped Output"}]} 9 | ] 10 | }, 11 | {"if":["unescapeHTML\\("], 12 | "then":[ 13 | {"alert":[{"alert config":"alert group","subject":"Unescaped Output"}]} 14 | ] 15 | }, 16 | {"if":["['\"]http(s)?://.*\\.js['\"]"], 17 | "then":[ 18 | {"alert":[{"alert config":"alert group","subject":"External Included Script"}]} 19 | ] 20 | }, 21 | {"if":["\\.eval\\(.*\\)"], 22 | "then":[ 23 | {"alert":[{"alert config":"alert group","subject":"Eval used"}]} 24 | ] 25 | }, 26 | {"if":["\\.postMessage\\(.*?,"], 27 | "then":[ 28 | {"alert":[{"alert config":"alert group","subject":"Cross-domain communication - sender"}]} 29 | ] 30 | }, 31 | {"if":["\\.addEventListener\\(\"message\""], 32 | "then":[ 33 | {"alert":[{"alert config":"alert group","subject":"Cross-domain communication - listener"}]} 34 | ] 35 | }, 36 | {"if":["localStorage\\.setItem\\(['\"](password|token|key|secret)"], 37 | "then":[ 38 | {"alert":[{"alert config":"alert group","subject":"Using local storage for secrets"}]} 39 | ] 40 | }, 41 | {"if":["{!\\$Api\\.Session_ID}"], 42 | "then":[ 43 | {"alert":[{"alert config":"alert group","subject":"Session ID being retrieved"}]} 44 | ] 45 | }, 46 | {"if":["GETSESSIONID\\(\\)"], 47 | "then":[ 48 | {"alert":[{"alert config":"alert group","subject":"Session ID being retrieved"}]} 49 | ] 50 | } 51 | ], 52 | "tests":{ 53 | "":["\\s+escape\\s*=\\s*(\"|')?false\\1"], 54 | "":["http(s)?://.*\\.js"], 55 | "eval(abcsasdscsdfsdf12334%);":["eval\\(.*\\)"] 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /providence.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | #-- Load libraries 17 | import datetime 18 | import imp 19 | import os, sys 20 | import getpass 21 | import json 22 | import pytz 23 | import logging 24 | import argparse 25 | 26 | from uuid import uuid4 27 | from apscheduler.scheduler import Scheduler 28 | from alerts import Alert 29 | from Empire.creds import CredentialManager 30 | #-- Load configuration 31 | #-- This method may change in the future 32 | import config 33 | #-- remember command line settings 34 | import settings 35 | 36 | def set_global_config(): 37 | configuration = config.Configuration('config.json') 38 | config.providence_configuration = configuration 39 | return configuration 40 | 41 | def set_logging_from_configuration(configuration): 42 | #-- Setup Logging 43 | logging.basicConfig(filename=configuration.get(('logging','filename')), 44 | filemode='w', 45 | level=logging.DEBUG if configuration.get(('logging','loglevel')) == 'debug' else logging.INFO, 46 | format=configuration.get(('logging','stringfmt')), 47 | datefmt=configuration.get(('logging','datefmt'))) 48 | console = logging.StreamHandler() 49 | console.setLevel(logging.INFO) 50 | formatter = logging.Formatter(configuration.get(('logging','formatter'))) 51 | console.setFormatter(formatter) 52 | logging.getLogger('').addHandler(console) 53 | 54 | if __name__ == "__main__": 55 | configuration = set_global_config() 56 | set_logging_from_configuration(configuration) 57 | logger = logging.getLogger(__name__) 58 | 59 | 60 | parser = argparse.ArgumentParser(description='Providence Monitor Framework') 61 | parser.add_argument('--tests','-t', help="run plugin tests", action='store_true') 62 | parser.add_argument('--mode', help="specify production for production mode, or anything otherwise. Database will be reset if not in production, and providence will start from the current commit") 63 | parser.add_argument('--p4change', help="specify the p4 change number to debug. Must not be in production mode") 64 | parser.add_argument('--timestamp', help="timestamp in PST to pull commits from, in the format YYYY-MM-DD HH:MM:SS") 65 | args = parser.parse_args() 66 | 67 | settings.init(args.mode, args.p4change) 68 | 69 | #-- Basic Credentials setup 70 | credentials_file = configuration.get('credentials_file') 71 | credential_key = os.environ.get('CREDENTIAL_KEY') 72 | if credential_key is None: 73 | credential_key = getpass.getpass('Credential Key:') 74 | credential_manager = CredentialManager(credentials_file, credential_key) 75 | config.credential_manager = credential_manager 76 | 77 | from models import Base 78 | from db import engine 79 | #-- reset db if not in production or timestamp specified 80 | if not settings.in_production() or args.timestamp: 81 | Base.metadata.drop_all(engine) 82 | Base.metadata.create_all(engine) 83 | from repos import repotracker 84 | from plugins import RepoWatcher, Plugins 85 | from plugins.base import PluginTestError 86 | 87 | #-- Load plugins 88 | loaded_plugins = Plugins(configuration) 89 | 90 | if args.tests: 91 | print "======================= Loading plugins =======================" 92 | plugins = loaded_plugins.enabled_plugins() 93 | 94 | print "======================= Running Plugin Tests =======================" 95 | for plugin_area in plugins: 96 | for plugin in plugins[plugin_area]: 97 | print "Running test for ", plugin.__module__ 98 | try: 99 | plugin.test() 100 | except PluginTestError, pte: 101 | print "Test Failed for ", plugin.__module__ 102 | print pte.message 103 | sys.exit(1) 104 | print "======================= Tests Successful =======================" 105 | sys.exit(0) 106 | 107 | def run_watchers(startTime=None): 108 | # run watcher plugins 109 | logger.info("Running watchers") 110 | 111 | plugins = loaded_plugins.enabled_plugins() 112 | 113 | repositories = loaded_plugins.get_repositories(plugins["repositories"]) 114 | watchers = loaded_plugins.get_watchers(plugins["watchers"]) 115 | tracker = repotracker.RepoTracker(); 116 | 117 | for repository_name, repository_data in repositories.items(): 118 | repository_watchers_by_path = watchers.get(repository_name) 119 | logger.info("In repo %s", repository_name) 120 | if repository_watchers_by_path is None: 121 | continue 122 | for repository_path, repo_watchers in repository_watchers_by_path.items(): 123 | repository_db_identifier = repository_name 124 | if repository_path is not None: 125 | repository_db_identifier += "::" + repository_path 126 | def commit_started_callback(repo_commit): 127 | if repo_watchers: 128 | for repo_watcher in repo_watchers: 129 | plugin = repo_watcher.plugin 130 | if hasattr(plugin, 'commit_started'): 131 | plugin.commit_started(repository_name, repo_commit) 132 | def commit_finished_callback(repo_commit): 133 | if repo_watchers: 134 | for repo_watcher in repo_watchers: 135 | plugin = repo_watcher.plugin 136 | if hasattr(plugin, 'commit_finished'): 137 | plugin.commit_finished(repository_name, repo_commit) 138 | if repo_commit.identifier: 139 | new_identifier = repo_commit.identifier 140 | tracker.update_identifier(repository_db_identifier, new_identifier) 141 | def patch_callback(repo_patch): 142 | if repo_watchers: 143 | for repo_watcher in repo_watchers: 144 | plugin = repo_watcher.plugin 145 | if hasattr(plugin, 'patch'): 146 | plugin.patch(repository_name, repo_patch) 147 | 148 | last_run_completed = tracker.last_run_completed(repository_db_identifier) 149 | if repository_data.get("check-every-x-minutes"): 150 | now = datetime.datetime.utcnow() 151 | if last_run_completed: 152 | if (now - last_run_completed) < datetime.timedelta(minutes=repository_data.get("check-every-x-minutes")): 153 | pass; 154 | try: 155 | last_identifier = tracker.last_identifier(repository_db_identifier) 156 | if not last_identifier and startTime: 157 | last_identifier = startTime + " PST" 158 | repository_data["source"].processSinceIdentifier(last_identifier, 159 | commit_started_callback=commit_started_callback, 160 | patch_callback=patch_callback, 161 | commit_finished_callback=commit_finished_callback, 162 | path=repository_path); 163 | 164 | tracker.update_last_run_completed(repository_db_identifier, datetime.datetime.utcnow()) 165 | except Exception, e: 166 | logger.exception("Repo Error: name=%s error=%s", repository_name, e, exc_info=True) 167 | 168 | def run_hourly(): 169 | # run hourly plugins 170 | hour = datetime.datetime.now().hour 171 | logger.info("Running hourly") 172 | plugins = loaded_plugins.enabled_plugins() 173 | 174 | hourly_plugins = plugins.get("hourly") 175 | for plugin in hourly_plugins: 176 | try: 177 | hour = 1 178 | plugin.run_hourly(hour) 179 | except Exception, e: 180 | logger.exception("Exception running hourly: %s" % (plugin)) 181 | 182 | def run_seven_minutes(): 183 | # run seven minute plugins 184 | hour = datetime.datetime.now().hour 185 | logger.info("Running 7 minutes") 186 | 187 | plugins = loaded_plugins.enabled_plugins() 188 | seven_minute_plugins = plugins.get("seven_minutes") 189 | 190 | for plugin in seven_minute_plugins: 191 | try: 192 | plugin.run_seven_minutes() 193 | except Exception, e: 194 | logger.exception("Exception running 7 minutes: %s" % (plugin)) 195 | 196 | if args.timestamp: 197 | run_watchers(args.timestamp) 198 | else: 199 | run_watchers() 200 | # run_seven_minutes() 201 | # run_hourly() 202 | 203 | sched = Scheduler(standalone=True) 204 | watcher_interval = "*/" + configuration.get(("cron", "watcher_interval")) 205 | sched.add_cron_job(run_watchers, minute=watcher_interval); 206 | # un-comment the following two lines if you'd like to use seven-minute or hourly plugins 207 | # sched.add_cron_job(run_seven_minutes, minute="*/7"); 208 | # sched.add_cron_job(run_hourly, hour="*", minute="5"); 209 | try: 210 | sched.start() 211 | except (KeyboardInterrupt, SystemExit): 212 | pass 213 | -------------------------------------------------------------------------------- /remove_pyc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | find . -type f -name '*.pyc' -not -path "./venv/*" -exec rm {} \; 3 | -------------------------------------------------------------------------------- /repos/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /repos/base.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import smtplib 17 | import re 18 | from email.mime.text import MIMEText 19 | from email.mime.multipart import MIMEMultipart 20 | 21 | from alerts import Alert 22 | from alerts.email_alert import EmailAlert 23 | 24 | import cgi 25 | 26 | class RepoSource(object): 27 | def __init__(self): 28 | pass 29 | 30 | def processSinceIdentifier(self, identifier, commit_callback, patch_callack): 31 | raise NotImplementedError("processSinceIdentifier was not defined") 32 | 33 | def processSinceDate(self, since_datetime, patch_callack): 34 | pass 35 | 36 | def processCommits(self, commits_json, commit_callback, patch_callack): 37 | pass 38 | 39 | def check_for_match(self, lines, search_re, ignore_case=True): 40 | all_lines = [u' '.join(lines).encode('utf-8').strip()] + lines 41 | ignorecase = re.compile(search_re, re.IGNORECASE | re.MULTILINE | re.DOTALL) 42 | for line in all_lines: 43 | if ignorecase.search(line): 44 | return True 45 | return False 46 | 47 | def retrieve_file(self, file_path): 48 | return '' 49 | 50 | def send_alert_email(self, from_email, to_email, subject, alert_message, repo_commit): 51 | # me == my email address 52 | # you == recipient's email address 53 | # Create the body of the message (a plain-text and an HTML version). 54 | url_html = "" 55 | if repo_commit and repo_commit.sha: #XXX: modify below to use config vars, 56 | #XXX: github.com/%s/%s/commit % (config.get('repos','github','owner'),...) 57 | url_html = "" + repo_commit.sha + "
" 58 | html = """ 59 | 60 | 61 | 62 | """ + url_html + """ 63 | 
""" + cgi.escape(alert_message) + """
 64 |
65 | 66 | 67 | """ 68 | EmailAlert().send(Alert(subject=subject, message=body, message_html=html), to_email=to_email) 69 | 70 | class RepoCommit(object): 71 | def __init__(self, 72 | sha=None, 73 | committer_email=None, 74 | committer_name=None, 75 | message=None, 76 | url=None, 77 | is_admin=None, 78 | role=None, 79 | repo_source=None, 80 | patches=[]): 81 | self.identifier = None 82 | self.sha = sha 83 | self.committer_name = committer_name 84 | self.committer_email = committer_email 85 | self.message = message 86 | self.url = url 87 | self.is_admin = is_admin 88 | self.role = role 89 | self.repo_source = repo_source 90 | self.patches = patches 91 | 92 | class RepoPatch: 93 | def __init__(self, 94 | repo_commit=None, 95 | filename=None, 96 | url=None, 97 | diff=None, 98 | lines_removed=None, 99 | lines_added=None): 100 | self.repo_commit = repo_commit 101 | self.filename = filename 102 | self.url = url 103 | self.lines_added = lines_added 104 | self.lines_removed =lines_removed 105 | self.diff = diff 106 | -------------------------------------------------------------------------------- /repos/diffparser.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | class DiffParser: 17 | def __init__(self, difftext): 18 | self.additions = [] 19 | self.deletions = [] 20 | self.after = [] 21 | for line in difftext.splitlines(): 22 | if len(line) > 0: 23 | if line[0] == '+': 24 | self.additions.append(line[1:]) 25 | self.after.append(line[1:]) 26 | elif line[0] == '-': 27 | self.deletions.append(line[1:]) 28 | else: 29 | self.after.append(line[1:]) 30 | 31 | 32 | 33 | if __name__ == "__main__": 34 | sampleDiff = ''' 35 | @@ -25,11 +25,15 @@ 36 | ################################################################################ 37 | 38 | import base64 39 | +import sys 40 | 41 | import github.GithubObject 42 | import github.Repository 43 | 44 | 45 | +atLeastPython3 = sys.hexversion >= 0x03000000 46 | + 47 | + 48 | class ContentFile(github.GithubObject.CompletableGithubObject): 49 | """ 50 | This class represents ContentFiles as returned for example by http://developer.github.com/v3/todo 51 | ''' 52 | x = DiffParser(sampleDiff) 53 | print x.additions 54 | print x.deletions 55 | -------------------------------------------------------------------------------- /repos/github.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import requests 17 | import json 18 | import datetime 19 | import dateutil.parser 20 | import urllib 21 | import os 22 | import pytz 23 | import sys 24 | sys.path.append( 25 | os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))) 26 | from Empire.cloudservices.github import GithubOrg, GithubRepo, GithubCommit 27 | from repos.base import RepoSource, RepoCommit, RepoPatch 28 | from repos.diffparser import DiffParser 29 | import logging 30 | logger = logging.getLogger('repos_github') 31 | 32 | class GithubSource(RepoSource): 33 | def __init__(self, creds=None, host='api.github.com', owner='Salesforce', repo='providence'): 34 | # default repository to github.com/salesforce/providence 35 | self._last_date = None 36 | self._last_identifier = None 37 | self._no_more_requests_until = None 38 | github_org = GithubOrg(host, owner, creds) 39 | self._github_repo = GithubRepo(github_org, repo) 40 | 41 | def processSinceIdentifier(self, identifier, commit_started_callback, patch_callback, commit_finished_callback, path=None): 42 | since_datetime = datetime.datetime.utcnow() 43 | since_datetime = since_datetime.replace(tzinfo=pytz.UTC, hour=0, minute=0, second=0) 44 | if identifier: 45 | try: 46 | since_datetime = dateutil.parser.parse(identifier) 47 | except Exception as e: 48 | logger.error("Error parsing datetime from %s", identifier) 49 | commits = self._github_repo.commits(since_datetime, path=path) 50 | self.processCommits(commits, 51 | commit_started_callback=commit_started_callback, 52 | patch_callback=patch_callback, 53 | commit_finished_callback=commit_finished_callback); 54 | if self._last_date: 55 | return self._last_date.isoformat() 56 | if since_datetime: 57 | return since_datetime.isoformat() 58 | return None 59 | 60 | def processCommits(self, commits, commit_started_callback, patch_callback, commit_finished_callback): 61 | if commits is None: 62 | return None 63 | 64 | # Process oldest first 65 | commits = commits[::-1] 66 | logger.debug("processing %d commits", len(commits)) 67 | for github_commit in commits: 68 | logger.debug("sha: %s", github_commit.sha) 69 | if github_commit.sha: 70 | github_commit = self._github_repo.commit(github_commit.sha) 71 | 72 | repo_commit = RepoCommit(); 73 | repo_commit.url = github_commit.html_url 74 | repo_commit.repo_source = self 75 | 76 | if github_commit.date is not None: 77 | self._last_date = dateutil.parser.parse(github_commit.date) 78 | 79 | repo_commit.date = self._last_date 80 | self._last_date += datetime.timedelta(seconds=1) 81 | repo_commit.identifier = self._last_date.isoformat() 82 | 83 | repo_commit.committer_email = github_commit.committer_email 84 | repo_commit.committer_name = github_commit.committer_name 85 | repo_commit.username = github_commit.committer_login 86 | repo_commit.message = github_commit.message 87 | 88 | repo_commit.sha = github_commit.sha 89 | commit_started_callback(repo_commit) 90 | 91 | if github_commit.files: 92 | for file_info in github_commit.files: 93 | if file_info.get('patch'): 94 | filename = committer_username = None 95 | diff = DiffParser(file_info['patch']) 96 | repo_patch = RepoPatch(repo_commit=repo_commit) 97 | repo_patch.diff = diff 98 | repo_patch.filename = file_info.get("filename") 99 | patch_callback(repo_patch) 100 | 101 | commit_finished_callback(repo_commit) 102 | logger.debug("commit sha: %s processing complete", github_commit.sha) 103 | #logger.debug("done") 104 | 105 | def retrieve_file(self, file_path): 106 | file_content = self._github_repo.get_raw_file(file_path) 107 | if file_content is None: 108 | file_content = '' 109 | return file_content 110 | 111 | if __name__ == "__main__": 112 | from Empire.creds import CredentialManager 113 | credentials_file = "credentials.json" 114 | credential_key = os.environ.get('CREDENTIAL_KEY') 115 | if credential_key is None: 116 | credential_key = getpass.getpass('Credential Key:') 117 | credential_manager = CredentialManager(credentials_file, credential_key) 118 | creds = credential_manager.get_or_create_credentials_for("github","password") 119 | 120 | test = GithubSource(creds); 121 | def cstart(commit): 122 | print commit 123 | def pstart(patch): 124 | print "touched ", patch.filename 125 | def cend(commit): 126 | pass 127 | test.processSinceIdentifier("2016-11-12T00:00:00Z", cstart, pstart, cend); 128 | -------------------------------------------------------------------------------- /repos/repotracker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import logging 17 | import dateutil.parser 18 | from time import sleep 19 | from models import Repo 20 | from db import Session, get_one_or_create 21 | from datetime import datetime, timedelta 22 | 23 | class RepoTracker(): 24 | def __init__(self): 25 | self.session = Session() 26 | 27 | def last_run_completed(self, repository_name): 28 | repo_tracker, did_add = get_one_or_create(self.session, Repo, name=repository_name) 29 | return repo_tracker.last_successful_run 30 | 31 | def update_last_run_completed(self, repository_name, last_successful_run): 32 | repo_tracker, did_add = get_one_or_create(self.session, Repo, name=repository_name) 33 | repo_tracker.last_successful_run = last_successful_run 34 | self.session.commit() 35 | 36 | def last_identifier(self, repository_name): 37 | repo_tracker, did_add = get_one_or_create(self.session, Repo, name=repository_name) 38 | return repo_tracker.last_identifier 39 | 40 | def update_identifier(self, repository_name, last_identifier): 41 | repo_tracker, did_add = get_one_or_create(self.session, Repo, name=repository_name) 42 | repo_tracker.last_identifier = last_identifier 43 | self.session.commit() 44 | 45 | if __name__ == "__main__": 46 | repo = RepoTracker() 47 | print repo.last_identifier("abcz") 48 | repo.update_identifier("abcz", "xxx") 49 | print repo.last_identifier("abcz") 50 | repo.update_identifier("abcz", "yyy") 51 | 52 | print repo.last_run_completed("abc") 53 | repo.update_last_run_completed("abc", datetime.utcnow()) 54 | print repo.last_run_completed("abc") 55 | repo.update_last_run_completed("abc", None) 56 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | APScheduler==2.1.2 2 | SQLAlchemy==0.9.8 3 | argparse==1.2.1 4 | distribute==0.6.24 5 | httplib2==0.9 6 | py==1.4.20 7 | pycrypto==2.6.1 8 | cryptography 9 | python-dateutil==2.2 10 | python-simple-hipchat==0.3.3 11 | pytz==2014.4 12 | rauth==0.7.0 13 | requests==2.3.0 14 | should-dsl==2.1.2 15 | six==1.7.3 16 | urllib3 17 | boto==2.33.0 18 | Jinja2==2.7.3 19 | ndg-httpsclient 20 | pyasn1 21 | lxml==3.4.1 22 | psycopg2==2.7 23 | p4python -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | _mode = None 17 | _change_id = None 18 | 19 | def init(mode, change_id): 20 | global _change_id 21 | global _mode 22 | _mode = mode 23 | if _mode == None: 24 | _mode = "production" 25 | _change_id = change_id 26 | 27 | def in_production(): 28 | if _mode == "production": 29 | return True 30 | else: 31 | return False 32 | 33 | def get_change_id(): 34 | return _change_id 35 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /tests/detection/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /tests/detection/regex_rule_engine/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /tests/detection/regex_rule_engine/config_one.json: -------------------------------------------------------------------------------- 1 | { 2 | "name":"Detection Test 1", 3 | "version":2, 4 | "alert configs":{ 5 | "alert m.feldman normal":{"email":"johndoe+alert_normal@example.com" 6 | }, 7 | "regexes":{ 8 | "exec() function":"\\.exec(\\s+)?\\(", 9 | "System property":"System\\.getProperty" 10 | }, 11 | "regex_rules":[ 12 | {"if":["exec"], 13 | "then":[ 14 | {"alert":[{"alert config":"alert m.feldman normal","subject":"ESC_RAW Matched"}]} 15 | ] 16 | }, 17 | {"if":["System property"], 18 | "then":[ 19 | {"alert":[{"alert config":"alert m.feldman high","subject":"DANGEROUS FUNCTION (system) MATCHED"}]} 20 | ] 21 | } 22 | ], 23 | "tests":{ 24 | "x.exec ('a')":["exec() function"], 25 | "System.getProperty(a)":["System property"], 26 | "Safe":[] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tests/detection/regex_rule_engine/test_regexruleengine.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import unittest 17 | import os 18 | 19 | from repos.base import RepoPatch, RepoCommit 20 | from repos.diffparser import DiffParser 21 | 22 | import json 23 | from detection.pattern.regex_rule_engine import RepoPatchRegexRuleEngine 24 | 25 | class RegexRuleEngineTest(unittest.TestCase): 26 | 27 | def setUp(self): 28 | pass 29 | 30 | def tearDown(self): 31 | pass 32 | 33 | def test_config_one(self): 34 | testDiff = '''@@ -25,11 +25,15 @@ 35 | ################################################################################ 36 | +Test exec abc 37 | -Removed This Line 38 | ''' 39 | diff = DiffParser(testDiff) 40 | 41 | test_commit = RepoCommit(url="http://www.example.com/commit/1") 42 | test_patch = RepoPatch(repo_commit=test_commit, diff=diff) 43 | 44 | filename = os.path.join(os.path.dirname(__file__),"config_one.json") 45 | rule_engine = RepoPatchRegexRuleEngine(json.load(open(filename)), test_mode=True) 46 | rule_engine.process(test_patch) 47 | 48 | -------------------------------------------------------------------------------- /tests/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /tests/plugins/test_pluginloader.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": { 3 | "folder":"./tests/plugins/test_pluginloader_testplugins/", 4 | "enabled": [{ 5 | "debug":false 6 | }] 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /tests/plugins/test_pluginloader.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import unittest 17 | import os 18 | import config 19 | from plugins import Plugins 20 | 21 | class PluginLoaderTest(unittest.TestCase): 22 | 23 | def setUp(self): 24 | configuration = config.Configuration('tests/plugins/test_pluginloader.config.json') 25 | self.plugins = Plugins(configuration) 26 | 27 | def tearDown(self): 28 | pass 29 | 30 | def test_config_one(self): 31 | plugins = self.plugins.all_plugins() 32 | watchers = plugins.get("watchers") 33 | watcher_names = [watcher.name for watcher in watchers] 34 | watcher_names.sort() 35 | 36 | self.assertEqual(watcher_names, ["tests/plugins/test_pluginloader_testplugins/debug"]) 37 | 38 | self.assertIsNotNone(plugins.get("hourly")) 39 | self.assertIsNotNone(plugins.get("seven_minutes")) 40 | -------------------------------------------------------------------------------- /tests/plugins/test_pluginloader_testplugins/debug/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import sys 17 | import os.path 18 | import json 19 | from plugins import base, RepoWatcher 20 | 21 | class Plugin(base.Plugin): 22 | def register_watcher(self): 23 | return [RepoWatcher(self, RepoWatcher.ALL, "*")] 24 | 25 | def commit_started(self, repository_name, repo_commit): 26 | pass 27 | 28 | def commit_finished(self, repository_name, repo_commit): 29 | pass 30 | 31 | def test(self): 32 | pass -------------------------------------------------------------------------------- /tests/repos/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | -------------------------------------------------------------------------------- /tests/repos/test_diffparser.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import unittest 17 | import os 18 | 19 | from repos.diffparser import DiffParser 20 | import json 21 | 22 | 23 | 24 | class DiffParserTest(unittest.TestCase): 25 | 26 | def setUp(self): 27 | self.test_diff=''' 28 | @@ -25,11 +25,15 @@ 29 | ################################################################################ 30 | +import sys 31 | This will still be around 32 | +atLeastPython3 = sys.hexversion >= 0x03000000 33 | + 34 | + 35 | -Removed Line 36 | class ContentFile(github.GithubObject.CompletableGithubObject): 37 | ''' 38 | 39 | def tearDown(self): 40 | pass 41 | 42 | def test_config_one(self): 43 | 44 | diff = DiffParser(self.test_diff) 45 | self.assertEqual(diff.additions, ["import sys","atLeastPython3 = sys.hexversion >= 0x03000000","",""]) 46 | self.assertEqual(diff.deletions, ["Removed Line"]) 47 | after = ['@ -25,11 +25,15 @@', 48 | '###############################################################################', 49 | 'import sys', 'This will still be around ', 50 | 'atLeastPython3 = sys.hexversion >= 0x03000000', 51 | '', 52 | '', 53 | 'class ContentFile(github.GithubObject.CompletableGithubObject):'] 54 | self.assertEqual(diff.after, after) -------------------------------------------------------------------------------- /tests/repos/test_repotracker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import unittest 17 | import os 18 | 19 | import json 20 | import string 21 | import random 22 | import getpass 23 | 24 | from datetime import datetime 25 | 26 | from Empire.creds import CredentialManager 27 | import config 28 | configuration = config.Configuration() 29 | credentials_file = configuration.get('credentials_file') 30 | credential_key = os.environ.get('CREDENTIAL_KEY') 31 | if credential_key is None: 32 | credential_key = getpass.getpass('Credential Key:') 33 | credential_manager = CredentialManager(credentials_file, credential_key) 34 | 35 | config.credential_manager = credential_manager 36 | from repos.repotracker import RepoTracker 37 | 38 | def id_generator(size=6, chars=string.ascii_uppercase + string.digits): 39 | return ''.join(random.choice(chars) for _ in range(size)) 40 | 41 | class RepoTrackerTest(unittest.TestCase): 42 | 43 | def setUp(self): 44 | pass 45 | 46 | def tearDown(self): 47 | pass 48 | 49 | def test_last_identifier(self): 50 | repo = RepoTracker() 51 | 52 | temp_identifier = id_generator() 53 | repo.update_identifier("test-identifier", temp_identifier) 54 | fetched_identifier = repo.last_identifier("test-identifier") 55 | self.assertEqual(temp_identifier, fetched_identifier) 56 | 57 | def test_last_run_completed(self): 58 | repo = RepoTracker() 59 | 60 | temp_last_run = datetime.utcnow() 61 | repo.update_last_run_completed("test-last-run",temp_last_run) 62 | fetched_last_run = repo.last_run_completed("test-last-run") 63 | self.assertEqual(temp_last_run, fetched_last_run) 64 | -------------------------------------------------------------------------------- /tests/test_providence.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Copyright (c) 2015, Salesforce.com, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | ''' 15 | 16 | import unittest 17 | import os 18 | import config 19 | import providence 20 | 21 | class ProvidenceTest(unittest.TestCase): 22 | 23 | def setUp(self): 24 | pass 25 | 26 | def tearDown(self): 27 | pass 28 | 29 | def test_global_configuration(self): 30 | configuration = providence.set_global_config() 31 | self.assertIsNotNone(config.providence_configuration) 32 | self.assertEqual(configuration, config.providence_configuration) --------------------------------------------------------------------------------