├── local_pull_aliases.py ├── local_pull_hashtags.py ├── .gitignore ├── LICENSE.txt ├── README.md ├── local_pull_urls.py ├── local_tweet_to_words.py ├── follow_redirects.py └── MaltegoTransform.py /local_pull_aliases.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Description: Locally extract aliases/usernames from Tweets 5 | # Installation: http://dev.paterva.com/developer/getting_started/building_your_own_local_transform.php 6 | # Author: Michael Henriksen (@michenriksen) 7 | 8 | from MaltegoTransform import * 9 | from ttp import ttp 10 | import sys 11 | 12 | transform = MaltegoTransform() 13 | transform.parseArguments(sys.argv) 14 | 15 | tweet = transform.getVar("content").decode('utf-8') 16 | parser = ttp.Parser() 17 | parsed_tweet = parser.parse(tweet) 18 | 19 | for alias in parsed_tweet.users: 20 | transform.addEntity("maltego.Alias", alias) 21 | 22 | transform.returnOutput() 23 | -------------------------------------------------------------------------------- /local_pull_hashtags.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | from MaltegoTransform import * 4 | from ttp import ttp 5 | import sys 6 | 7 | # Description: Locally extract hashtags from Tweets 8 | # Installation: http://dev.paterva.com/developer/getting_started/building_your_own_local_transform.php 9 | # Author: Michael Henriksen (@michenriksen) 10 | 11 | transform = MaltegoTransform() 12 | transform.parseArguments(sys.argv) 13 | 14 | tweet = transform.getVar("content").decode('utf-8') 15 | parser = ttp.Parser() 16 | parsed_tweet = parser.parse(tweet) 17 | 18 | for hashtag in parsed_tweet.tags: 19 | hashtag = "#" + hashtag 20 | transform.addEntity("maltego.hashtag", hashtag) 21 | 22 | transform.returnOutput() 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | 44 | # Translations 45 | *.mo 46 | *.pot 47 | 48 | # Django stuff: 49 | *.log 50 | 51 | # Sphinx documentation 52 | docs/_build/ 53 | 54 | # PyBuilder 55 | target/ 56 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015 Michael Henriksen 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Maltego 2 | 3 | A collection of custom local transforms for the [Maltego](http://www.paterva.com/web6/products/maltego.php) 4 | OSINT gathering tool. 5 | 6 | ## local_pull_aliases.py, local_pull_hashtags.py, local_pull_urls.py, local_tweet_to_words.py 7 | Extracts entities from Tweets just like the built-in transforms, but faster as they 8 | don't send the Tweets off the the remote servers, but extract the entities locally in code. 9 | 10 | ### Requirements: 11 | Depends on [twitter-text-python](https://github.com/edburnett/twitter-text-python) which 12 | can be installed with `pip install twitter-text-python`. 13 | 14 | **local_tweet_to_words.py** also depends on [Natural Language Toolkit](http://www.nltk.org/) 15 | which can be installed with `pip install nltk`. After installation, execute 16 | `python -c "import nltk; nltk.download()"` in a terminal and download the `stopwords` 17 | package under the Corpora tab. 18 | 19 | ## follow_redirects.py 20 | URL transform that follows up to 15 HTTP redirects and returns the final URL. 21 | Convenient if a transform returns a shortened or obfuscated URL. 22 | 23 | **Note:** Running this transform will potentially engage target systems, 24 | so if you're doing passive reconnaissance, this transform should not be used! 25 | -------------------------------------------------------------------------------- /local_pull_urls.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | from MaltegoTransform import * 4 | from ttp import ttp 5 | from urlparse import urlparse 6 | import sys, httplib 7 | 8 | # Description: Locally extract URLs from Tweets 9 | # Installation: http://dev.paterva.com/developer/getting_started/building_your_own_local_transform.php 10 | # Author: Michael Henriksen (@michenriksen) 11 | 12 | transform = MaltegoTransform() 13 | transform.parseArguments(sys.argv) 14 | 15 | tweet = transform.getVar("content").decode('utf-8') 16 | parser = ttp.Parser() 17 | parsed_tweet = parser.parse(tweet) 18 | 19 | for url in parsed_tweet.urls: 20 | parsed_url = urlparse(url) 21 | 22 | # Expand Twitter shortened URLs 23 | if parsed_url.hostname == "t.co": 24 | transform.addUIMessage("Expanding t.co URL: " + url, "Inform") 25 | 26 | # Perform a HEAD request on t.co via secure connection 27 | connection = httplib.HTTPSConnection(parsed_url.hostname) 28 | connection.request("HEAD", parsed_url.path) 29 | response = connection.getresponse() 30 | 31 | # Loop through each header until we find the Location header 32 | for header in response.getheaders(): 33 | if header[0] == 'location': 34 | url = header[1] 35 | break 36 | 37 | ent = transform.addEntity("maltego.URL", url) 38 | ent.addAdditionalFields("url", "URL", True, url) 39 | 40 | transform.returnOutput() 41 | -------------------------------------------------------------------------------- /local_tweet_to_words.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | from MaltegoTransform import * 4 | from nltk.corpus import stopwords 5 | from nltk.stem.snowball import SnowballStemmer 6 | import sys, string 7 | 8 | # Description: Locally extract words from Tweets 9 | # Installation: http://dev.paterva.com/developer/getting_started/building_your_own_local_transform.php 10 | # Author: Michael Henriksen (@michenriksen) 11 | 12 | transform = MaltegoTransform() 13 | transform.parseArguments(sys.argv) 14 | 15 | tweet = transform.getVar("content") 16 | stemmer = SnowballStemmer("english") 17 | stop_words = stopwords.words("english") 18 | with open("top100Kenglishwords.txt") as f: 19 | common_words = [x.strip().strip().lower().decode("unicode_escape").encode("ascii", "ignore") for x in f.readlines()] 20 | words = [] 21 | 22 | for word in tweet.split(): 23 | if word.startswith('#') or word.startswith('@') or word.startswith('\\'): 24 | continue 25 | 26 | # Normalize word and strip out silliness 27 | word = word.strip().lower().decode("unicode_escape").encode("ascii", "ignore") 28 | word = ''.join(ch for ch in word if ch not in string.punctuation) 29 | 30 | if word == '' or word == 'rt' or word.startswith('http'): 31 | continue 32 | 33 | # Ignore stop-words and common words 34 | if word in stop_words or word in common_words: 35 | continue 36 | 37 | word = stemmer.stem(word) 38 | 39 | # Check word again after stemming 40 | if word in stop_words or word in common_words: 41 | continue 42 | 43 | words.append(word) 44 | 45 | for word in words: 46 | transform.addEntity("maltego.Phrase", word) 47 | 48 | transform.returnOutput() 49 | -------------------------------------------------------------------------------- /follow_redirects.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | from MaltegoTransform import * 4 | from urlparse import urlparse 5 | import sys, httplib 6 | 7 | # Description: Follows HTTP redirects and returns the final URL. 8 | # Installation: http://dev.paterva.com/developer/getting_started/building_your_own_local_transform.php 9 | # Author: Michael Henriksen (@michenriksen) 10 | 11 | def follow_redirects(url, maximum_redirects = 15): 12 | parsed_url = urlparse(url) 13 | if parsed_url.scheme == 'https': 14 | connection = httplib.HTTPSConnection(parsed_url.hostname) 15 | else: 16 | connection = httplib.HTTPConnection(parsed_url.hostname) 17 | 18 | connection.request("HEAD", parsed_url.path) 19 | response = connection.getresponse() 20 | 21 | if response.status >= 300 and response.status <= 399: 22 | if maximum_redirects == 0: 23 | transform.addUIMessage("Too many redirects; giving up.", "FatalError") 24 | return url 25 | 26 | transform.addUIMessage("Got redirect response from " + parsed_url.hostname + "; following...", "Inform") 27 | for header in response.getheaders(): 28 | if header[0] == 'location': 29 | return follow_redirects(header[1], maximum_redirects - 1) 30 | transform.addUIMessage("Location header was not found...", "PartialError") 31 | return url 32 | else: 33 | transform.addUIMessage("Got non-redirect response from " + parsed_url.hostname + ": " + str(response.status) + " " + response.reason, "Inform") 34 | return url 35 | 36 | transform = MaltegoTransform() 37 | 38 | url = sys.argv[1] 39 | final_url = follow_redirects(url) 40 | 41 | if url != final_url: 42 | transform.addUIMessage(url + " redirected to: " + final_url, "Inform") 43 | 44 | ent = transform.addEntity("maltego.URL", urlparse(final_url).hostname) 45 | ent.addAdditionalFields("url", "URL", True, final_url) 46 | else: 47 | transform.addUIMessage(url + " did not redirect.", "Inform") 48 | 49 | transform.returnOutput() 50 | -------------------------------------------------------------------------------- /MaltegoTransform.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | ####################################################### 3 | # Maltego Python Local Transform Helper # 4 | # Version 0.2 # 5 | # # 6 | # Local transform specification can be found at: # 7 | # http://ctas.paterva.com/view/Specification # 8 | # # 9 | # For more help and other local transforms # 10 | # try the forum or mail me: # 11 | # # 12 | # http://www.paterva.com/forum # 13 | # # 14 | # Andrew MacPherson [ andrew <> Paterva.com ] # 15 | # # 16 | ####################################################### 17 | import sys, re 18 | from xml.sax.saxutils import escape 19 | 20 | class MaltegoEntity(object): 21 | value = ""; 22 | weight = 100; 23 | displayInformation = None; 24 | additionalFields = []; 25 | iconURL = ""; 26 | entityType = "Phrase" 27 | 28 | def __init__(self,eT=None,v=None): 29 | if (eT is not None): 30 | self.entityType = eT; 31 | if (v is not None): 32 | self.value = sanitise(v); 33 | self.additionalFields = []; 34 | self.displayInformation = None; 35 | 36 | def setType(self,eT=None): 37 | if (eT is not None): 38 | self.entityType = eT; 39 | 40 | def setValue(self,eV=None): 41 | if (eV is not None): 42 | self.value = sanitise(eV); 43 | 44 | def setWeight(self,w=None): 45 | if (w is not None): 46 | self.weight = w; 47 | 48 | def setDisplayInformation(self,di=None): 49 | if (di is not None): 50 | self.displayInformation = di; 51 | 52 | def addAdditionalFields(self,fieldName=None,displayName=None,matchingRule=False,value=None): 53 | self.additionalFields.append([sanitise(fieldName),sanitise(displayName),matchingRule,sanitise(value)]); 54 | 55 | def setIconURL(self,iU=None): 56 | if (iU is not None): 57 | self.iconURL = iU; 58 | 59 | def returnEntity(self): 60 | print ""; 61 | print "" + str(self.value) + ""; 62 | print "" + str(self.weight) + ""; 63 | if (self.displayInformation is not None): 64 | print ""; 65 | if (len(self.additionalFields) > 0): 66 | print ""; 67 | for i in range(len(self.additionalFields)): 68 | if (str(self.additionalFields[i][2]) <> "strict"): 69 | print "" + str(self.additionalFields[i][3]) + ""; 70 | else: 71 | print "" + str(self.additionalFields[i][3]) + ""; 72 | print ""; 73 | if (len(self.iconURL) > 0): 74 | print "" + self.iconURL + ""; 75 | print ""; 76 | 77 | class MaltegoTransform(object): 78 | entities = [] 79 | exceptions = [] 80 | UIMessages = [] 81 | values = {}; 82 | 83 | def __init__(self): 84 | values = {}; 85 | value = None; 86 | 87 | def parseArguments(self,argv): 88 | if (argv[1] is not None): 89 | self.value = argv[1]; 90 | 91 | if (len(argv) > 2): 92 | if (argv[2] is not None): 93 | # Michael Henriksen: Bug fix: The simple splitting fails 94 | # if the tweet content contains a hashtag. Only split on 95 | # pound symbols that are not preceded by a backslash... 96 | # vars = argv[2].split('#'); 97 | vars = re.compile('(?"; 128 | print ""; 129 | print "" 130 | 131 | for i in range(len(self.exceptions)): 132 | print "" + self.exceptions[i] + ""; 133 | print "" 134 | print ""; 135 | print ""; 136 | exit(); 137 | 138 | def returnOutput(self): 139 | print ""; 140 | print ""; 141 | 142 | print "" 143 | for i in range(len(self.entities)): 144 | self.entities[i].returnEntity(); 145 | print "" 146 | 147 | print "" 148 | for i in range(len(self.UIMessages)): 149 | print "" + self.UIMessages[i][1] + ""; 150 | print "" 151 | 152 | print ""; 153 | print ""; 154 | 155 | def writeSTDERR(self,msg): 156 | sys.stderr.write(str(msg)); 157 | 158 | def heartbeat(self): 159 | self.writeSTDERR("+"); 160 | 161 | def progress(self,percent): 162 | self.writeSTDERR("%" + str(percent)); 163 | 164 | def debug(self,msg): 165 | self.writeSTDERR("D:" + str(msg)); 166 | 167 | 168 | 169 | def sanitise(value): 170 | return escape(value) 171 | replace_these = ["&",">","<"]; 172 | replace_with = ["&",">","<"]; 173 | tempvalue = value; 174 | for i in range(0,len(replace_these)): 175 | tempvalue = tempvalue.replace(replace_these[i],replace_with[i]); 176 | return tempvalue; 177 | --------------------------------------------------------------------------------