├── LICENSE ├── README.md ├── main.py ├── requirements.txt └── src ├── template.html └── utitlis.py /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |

Keep an eye on your targets with Websy to get quickly notified for any change they push on their Web Server 3 |

4 | 5 | ### Installation 6 | ```bash 7 | > git clone https://github.com/cYph3r1337/websy 8 | > cd websy 9 | > pip3 install -r requirements.txt 10 | > python3 main.py --help 11 | ``` 12 | ### Features 13 |

14 | 15 | - Fast and simple to use 16 | - Outputs html file with **Search and Sort ability** for easy analysis 17 | - Lightweight on resources 18 | 19 | ### Usage 20 | 21 | ```python 22 | python3 main.py --help 23 | ``` 24 |

25 | 26 | ``` 27 | Run Websy on a set of urls regularly through cronjobs and it 28 | will automatically generate "output.html" with every run 29 | 30 | https://code.tutsplus.com/tutorials/scheduling-tasks-with-cron-jobs--net-8800 31 | ``` 32 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import time,sys,argparse,datetime,os.path 2 | from progress.bar import Bar 3 | from concurrent.futures import ThreadPoolExecutor,wait,as_completed 4 | from src.utitlis import get_old_data,update_file,collect,output_save,date_stamp,not_empty 5 | 6 | parser = argparse.ArgumentParser(description='This is a monitoring tool, it checks for any change in website') 7 | parser.add_argument('-f', action="store",dest="file",help="file containing urls to monitor") 8 | parser.add_argument('-t', action="store",dest="threads",default=20, help="No. of threads", type=int) 9 | parser.add_argument('-o', action="store",dest="output",default="./", help="Directory in which output should be saved") 10 | parser.add_argument('-db', action="store",dest="db",default="./", help="Directory in which db.json should be saved") 11 | 12 | input = parser.parse_args() 13 | 14 | diff=[] 15 | 16 | url_file_path = input.file 17 | thread = input.threads 18 | outDir = input.output 19 | if not url_file_path: raise Exception("Url file not provided") 20 | output_file_name = os.path.join(outDir, "output.html") 21 | data_json_path= (os.path.join(input.db,'data.json') or os.path.join(os.path.abspath(os.path.dirname(__file__)),'data.json')) 22 | 23 | # fetch old data from data.json 24 | data= get_old_data(data_json_path) 25 | futures,html=[],"" 26 | 27 | try: 28 | urls = list(filter(not_empty,open(url_file_path).read().split('\n'))) 29 | except Exception as e: 30 | raise e 31 | 32 | bar = Bar('Progress', max=len(urls)) 33 | 34 | executor = ThreadPoolExecutor(max_workers=thread) 35 | futures = [executor.submit(collect,url,data,diff) for url in urls] 36 | 37 | for future in as_completed(futures): 38 | [data,diff] = future.result() 39 | bar.next() 40 | 41 | header = f'{(data.get("date") or [datetime.datetime.now().strftime("%x")])[-1]} - {datetime.datetime.now().strftime("%x")}' 42 | data = date_stamp(data) 43 | update_file(data_json_path,data) 44 | 45 | # Constructing output.html with diff data 46 | properties = ["url","code","length", "lines", "words", "location"] 47 | for d in diff: 48 | html += "" 49 | for p in properties: 50 | val = (d.get(p) or "No Change") 51 | if val != "No Change" and p!="url": 52 | html += f'{val}' 53 | else: 54 | html += f'{val}' 55 | html += "" 56 | 57 | output_save(html,header,output_file_name) 58 | 59 | bar.finish() 60 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.20.0 2 | urllib3==1.26.5 3 | progress==1.5 4 | -------------------------------------------------------------------------------- /src/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 16 | 17 | 18 |

HEADER

19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | FILL_WITH_DIFF_RESULT 32 | 33 |
UrlCodelengthwordslinesLocation
34 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/utitlis.py: -------------------------------------------------------------------------------- 1 | import base64,json,datetime 2 | import requests,os 3 | from urllib3.exceptions import InsecureRequestWarning 4 | 5 | ## supress ssl WARNING 6 | requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning) 7 | 8 | def collect(url,data,diff): 9 | if url: 10 | re = req(url) 11 | if data.get(url): 12 | [data,diff] = Compare_and_add(data,re,url,diff) 13 | else: 14 | # this adds the "re" object to the json file 15 | data.update(re) 16 | return [data,diff] 17 | 18 | # request url and create json object with different properties 19 | def req(u): 20 | db,prop={},{} 21 | try: 22 | response = requests.get(u, allow_redirects=False, verify=False) 23 | prop['code'] = [response.status_code] 24 | prop['length'] = [len(response.text)] 25 | prop['lines'] = [len(response.text.split('\n'))] 26 | prop['words'] = [len(response.text.split(' '))] 27 | prop['location'] = [response.headers.get('Location')] 28 | db[u] = prop 29 | except requests.exceptions.Timeout: 30 | print(f'{u} Timeout') 31 | except requests.exceptions.RequestException as e: 32 | print(f'{u} is Not reachable') 33 | 34 | return db 35 | 36 | 37 | # read data.json 38 | def get_old_data(data_json_path,data={}): 39 | if not os.path.exists(data_json_path): 40 | f=open(data_json_path, "w+") 41 | f.write("{}") 42 | else: 43 | with open(data_json_path, 'r') as json_file: 44 | data = json.load(json_file) 45 | return data 46 | 47 | # compare data[url] json with req[url], add to the data.json and diff 48 | def Compare_and_add(data,req,url,diff): 49 | temp={} 50 | for prop in data[url].keys(): 51 | old,new=data[url][prop],req[url][prop] 52 | if old[-1]!=new[0]: 53 | temp['url'] = url 54 | if prop != "location" and prop != "code": 55 | temp[prop] = abs(int(old[-1])-int(new[0])) 56 | else: 57 | temp[prop] = f'{(old[-1] or 0)} --> {(new[0] or 0)}' 58 | data[url][prop].append(req[url][prop][0]) 59 | if temp.get('url'): diff.append(temp) 60 | return [data,diff] 61 | 62 | # update the data.json with new date 63 | def update_file(data_json_path,data): 64 | with open(data_json_path, "w") as jsonFile: 65 | json.dump(data, jsonFile) 66 | 67 | # create `output.html` from `template.html` and `diff data` 68 | def output_save(html,header,name="./output.html"): 69 | template_file = os.path.join(os.path.dirname(__file__), 'template.html') 70 | with open(template_file, 'r') as template: 71 | result = template.read() 72 | result = result.replace("FILL_WITH_DIFF_RESULT",html) 73 | result = result.replace("HEADER",header) 74 | f = open(name,"w") 75 | f.write(result) 76 | f.close() 77 | 78 | # Add today's date in data.json 79 | def date_stamp(data): 80 | today=datetime.datetime.now().strftime("%x") 81 | if data.get('date'): 82 | data['date'].append(today) 83 | else: 84 | data['date'] = [today] 85 | return data 86 | 87 | def not_empty(value): 88 | return value != "" --------------------------------------------------------------------------------