├── .gitignore
├── Dockerfile
├── LICENSE
├── Pipfile
├── README.md
├── Vagrantfile
├── __init__.py
├── config.json.example
├── crawler.py
├── docker-compose.yml
├── mapping
    ├── datacenters.json
    ├── regions.json
    └── server_types.json
├── notifiers
    ├── __init__.py
    ├── base_notifier.py
    ├── email_notifier.py
    ├── file_notifier.py
    ├── freemobile_notifier.py
    ├── osx_notifier.py
    ├── popup_notifier.py
    ├── popup_pywin_notifier.py
    ├── pushbullet_notifier.py
    ├── pushover_notifier.py
    ├── smsapi_notifier.py
    ├── telegram_notifier.py
    └── xmpp_notifier.py
└── requirements.txt


/.gitignore:
--------------------------------------------------------------------------------
 1 | venv/
 2 | 
 3 | # vagrant
 4 | .vagrant
 5 | 
 6 | # Byte-compiled / optimized / DLL files
 7 | __pycache__/
 8 | *.py[cod]
 9 | 
10 | # C extensions
11 | *.so
12 | 
13 | # Distribution / packaging
14 | .Python
15 | env/
16 | bin/
17 | build/
18 | develop-eggs/
19 | dist/
20 | eggs/
21 | lib/
22 | lib64/
23 | parts/
24 | sdist/
25 | var/
26 | *.egg-info/
27 | .installed.cfg
28 | *.egg
29 | 
30 | # Installer logs
31 | pip-log.txt
32 | pip-delete-this-directory.txt
33 | 
34 | # Unit test / coverage reports
35 | htmlcov/
36 | .tox/
37 | .coverage
38 | .cache
39 | nosetests.xml
40 | coverage.xml
41 | 
42 | # Translations
43 | *.mo
44 | 
45 | # Mr Developer
46 | .mr.developer.cfg
47 | .project
48 | .pydevproject
49 | 
50 | # Rope
51 | .ropeproject
52 | 
53 | # Django stuff:
54 | *.log
55 | *.pot
56 | 
57 | # Sphinx documentation
58 | docs/_build/
59 | 
60 | # Private files
61 | config.json
62 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3-onbuild
2 | CMD [ "python", "./crawler.py" ]
3 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2014 Anton
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.


--------------------------------------------------------------------------------
/Pipfile:
--------------------------------------------------------------------------------
 1 | [[source]]
 2 | name = "pypi"
 3 | url = "https://pypi.org/simple"
 4 | verify_ssl = true
 5 | 
 6 | [dev-packages]
 7 | 
 8 | [packages]
 9 | tornado = ">=4.0.0"
10 | chump = ">=1.5.0"
11 | python-telegram-bot = "*"
12 | 
13 | [requires]
14 | python_version = "3.8"
15 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Kimsufi/So-you-Start Avaliability Crawler
 2 | ============================
 3 | 
 4 | > Crawler that will notify you when Kimsufi servers (or So-you-Start, or OVH) become available for purchase.
 5 | 
 6 | 
 7 | 
 8 |     git clone https://github.com/MA3STR0/kimsufi-crawler.git
 9 |     cd kimsufi-crawler
10 |     cp config.json.example config.json
11 |     vim config.json   # edit config.json file to set up your nofitication preferences
12 |     sudo pip install -r requirements.txt
13 |     python crawler.py
14 | 
15 | About
16 | -----
17 | 
18 | Dedicated servers on [Kimsufi](http://www.kimsufi.com) and [So you Start](http://www.soyoustart.com) have amazing prices, however they are always out of stock. This crawler will notify you as aoon as servers are available for purchase; it can send emails, show Mac OS notifications, open popup windows, or even trigger sms. Then hurry up, you have around 2 minutes to complete your order :)
19 | 
20 | 
21 | Set it up
22 | ---------
23 | 
24 | **Runs on GNU/Linux, Mac and Widnows with Python 2.7+ or Python 3.4+**
25 | 
26 | - Clone this repo (`git clone https://github.com/MA3STR0/kimsufi-crawler.git`) or download and unpack archive
27 | - Taking `config.json.example` as a template, create a file `config.json` and correct configuration according to your preferences:
28 |   - `servers`: list of servers that should be tracked, eg `["KS-1", "KS-2 SSD", "GAME-2"]` etc. All supported server names can be found in mapping file [mapping/server_types.json](/mapping/server_types.json) as values)
29 | 
30 |   - `region`: desired location of server, `canada` or `europe`
31 | 
32 |   - `notifier`: notification mechanism, choose between:
33 |     - `"email"`: default, requires additional email account settings
34 |     - `"popup"`: simple popup window, platform-independent
35 |     - `"popup_pywin"`: simple popup window, displays on top of all other windows (Windows only, requires pywin32 package)
36 |     - `"osx"`: Mac OS-X desktop notifications (using terminal-notifier)
37 |     - `"smsapi"`: sms through smsapi.pl gateway, requires account
38 |     - `"xmpp"`: send jabber/xmpp message, requires account - needs xmpppy
39 |     - `"pushover"`: send Pushover message, requires account - needs chump
40 |     - `"pushbullet"`: send Pushbullet message, requires account - needs pushbullet
41 |     - `"freemobile"`: sends sms to freemobile customer, requires account - needs requests
42 |     - `"telegram"`: sends message to a Telegram chat - needs python-telegram-bot
43 | 
44 |   - `to_email`: your email to receive notifications
45 |   - `from_email`: email account of the crawler.
46 |   - `from_pwd`, `from_smtp_host`: smtp configuration of crawler email account
47 |   - `xmpp_jid`, `xmpp_password`, `xmpp_recipient`, `xmpp_send_test`: xmpp configuration of sending jabber account
48 |   - `pushover_application_id`, `pushover_user_id`: your Pushover configuration, messages are sent to all devices assigned to user
49 |   - `pushover_priority`: priority of the notification to receive. -2 is lowest and 2 is highest
50 |   - `pushbullet_apikey`: your Pushbullet API key, notification sent to all devices
51 |   - `freemobile_username`: your freemobile login
52 |   - `freemobile_key`: your freemobile sms notification key
53 |   - `telegram_token`: full token you receive from BotFather after creating a bot
54 |   - `telegram_chat_id`: chat id you want to receive the message in. Retrieve by following for example [this stackoverflow answer](https://stackoverflow.com/a/32572159)
55 | 
56 | - Crawler runs on Python 2.7+ and Tornado framework 4.0+. Assuming that you already have Python/pip, just get Tornado and the notifier dependencies with `sudo pip install -r requirements.txt`. You can also set up virtualenv if you like.
57 | - Run with `python crawler.py`. If no error messages come, you're ready.
58 | - (optional) If your pip install command fails due to xmpppy being on pre-release version (eg. 0.5.0rc1) use: `sudo pip install --pre -r requirements.txt`
59 | - (optional) In case of problems with easygui installation on Ubuntu, you can get it also with `sudo apt-get install python-easygui`
60 | - (optional) It's a good idea to register an account on Kimsufi/OVH in advance; just pick any available server on www.kimsufi.com and sign up without paying. This way later you will only need to log in and enter payment details, which saves a lot of time.
61 | 
62 | **Test everything**
63 | 
64 | You may test the whole chain by setting some popular servers in your
65 | config.json, for example ["KS-3A", "KS-3B", "KS-2E"]. Since those are always
66 | available, you should receive a notification immediately.
67 | 
68 | **Config check**
69 | 
70 | Crawler makes an initial check of the config file during startup, so if you have any syntax errors or missing software dependencies - console logs will let you know.
71 | 
72 | 
73 | Advanced configuration
74 | ----------------------
75 | 
76 | You can add more options to the config.json if you need:
77 | 
78 | - `"crawler_interval": 8` // overriding default periodic callback interval in seconds (should be more than 7.2 to avoid rate-limit)
79 | - `"request_timeout": 30` // http timeout for API requests.
80 | - `"from_smtp_port": 587` // use non-standard smtp port
81 | - `"use_starttls": true` // forcing encrypted SMTP session using TLS (true by default)
82 | - `"use_ssl": false` // forcing encrypted SMTP session using SSL (false by default)
83 | - `"from_user": "sender@domain.com"`  // if smtp user is different from `from_email`
84 | 
85 | **Versions**
86 | 
87 | These instructions are based on Version 2 of the crawler. You can access last stable release of v1 by browsing through [release history](https://github.com/MA3STR0/kimsufi-crawler/releases)
88 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | V_NAME="kimsufi-crawler"
 2 | 
 3 | Vagrant.configure(2) do |config|
 4 |   config.vm.box = "ubuntu/trusty64"
 5 | 
 6 |   config.ssh.forward_agent = true
 7 | 
 8 |   config.vm.network "public_network"
 9 | 
10 |   config.vm.provision "shell", inline: $provision
11 | 
12 |   config.vm.hostname = V_NAME
13 | 
14 |   config.vm.provider "virtualbox" do |v|
15 |     v.name   = V_NAME
16 |     v.memory = 2048
17 |   end
18 | end
19 | 
20 | $provision = <<SCRIPT
21 | # variables
22 | HTML_PATH="/home/vagrant/html/"
23 | HTML_FILE_NAME="kimsufi-crawler.html"
24 | # install required packages
25 | apt-get update
26 | apt-get upgrade -y
27 | apt-get install -y nginx git python python-pip
28 | # nginx
29 | mkdir -p $HTML_PATH
30 | echo 'NOT AVAILABLE' > "$HTML_PATH/$HTML_FILE_NAME"
31 | rm -f /etc/nginx/sites-enabled/default
32 | cat << EOF > /etc/nginx/sites-enabled/kimsufi-crawler.conf
33 | server {
34 |     listen          80;
35 |     server_name     localhost;
36 |     root            $HTML_PATH;
37 | }
38 | EOF
39 | service nginx reload
40 | # python
41 | cd /vagrant
42 | pip install --pre -r requirements.txt
43 | chown -R vagrant:vagrant /home/vagrant/*
44 | # conf
45 | if [ ! -f config.json ]; then
46 | cat << EOF > /vagrant/config.json
47 | {
48 |     "servers": ["KS-2E"],
49 |     "region": "europe",
50 |     "notifier": "file",
51 |     "file_path": "$HTML_PATH/$HTML_FILE_NAME",
52 |     "crawler_interval": 8,
53 |     "request_timeout": 15
54 | }
55 | EOF
56 | fi
57 | echo '*************************************************************************'
58 | ifconfig
59 | SCRIPT


--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MA3STR0/kimsufi-crawler/4c9f1607e120bcb178d3deb5a86a2431d2fab59f/__init__.py


--------------------------------------------------------------------------------
/config.json.example:
--------------------------------------------------------------------------------
 1 | {
 2 |     "servers": ["KS-3A", "KS-3B"],
 3 |     "region": "europe",
 4 |     "notifier": "email",
 5 |     "from_email": "sender@domain.com",
 6 |     "from_pwd": "sender-password",
 7 |     "from_smtp_host": "smtp.gmail.com",
 8 |     "to_email": "mymail@domain.com"
 9 | }
10 | 


--------------------------------------------------------------------------------
/crawler.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # -*- coding: utf-8 -*-
  3 | """Crawler that sends notifications as soon as servers
  4 | on Kimsufi/OVH become available for purchase"""
  5 | 
  6 | import json
  7 | import sys
  8 | import os
  9 | import re
 10 | import logging
 11 | import importlib
 12 | import tornado.ioloop
 13 | import tornado.web
 14 | from tornado.httpclient import AsyncHTTPClient
 15 | from tornado.httpclient import HTTPError
 16 | from tornado.gen import coroutine
 17 | # Python 3 imports
 18 | try:
 19 |     from urllib import quote
 20 | except ImportError:
 21 |     from urllib.parse import quote
 22 | 
 23 | logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
 24 | _logger = logging.getLogger(__name__)
 25 | CURRENT_PATH = os.path.dirname(__file__)
 26 | 
 27 | 
 28 | def parse_json_file(filename):
 29 |     """open file and parse its content as json"""
 30 |     with open(filename, 'r') as jsonfile:
 31 |         content = jsonfile.read()
 32 |         try:
 33 |             result = json.loads(content)
 34 |         except ValueError:
 35 |             _logger.error(
 36 |                 "Parsing file %s failed. Check syntax with a JSON validator:"
 37 |                 "\nhttp://jsonlint.com/?json=%s", filename, quote(content))
 38 |             sys.exit(1)
 39 |     return result
 40 | 
 41 | 
 42 | class Crawler(object):
 43 |     """Crawler responsible for fetching availability and monitoring states"""
 44 | 
 45 |     def __init__(self, state_change_callback):
 46 |         # set properties
 47 |         self.state_change_callback = state_change_callback
 48 | 
 49 |         # load mappings
 50 |         self.SERVER_TYPES = parse_json_file(
 51 |             os.path.join(CURRENT_PATH, 'mapping/server_types.json'))
 52 |         self.REGIONS = parse_json_file(
 53 |             os.path.join(CURRENT_PATH, 'mapping/regions.json'))
 54 | 
 55 |         # set private vars
 56 |         self.API_URL = ("https://ca.ovh.com/engine/api/dedicated/server/availabilities?country=fra")
 57 |         self.STATES = {}
 58 |         self.HTTP_ERRORS = []
 59 |         self.interval = 8   # seconds between interations
 60 |         self.periodic_cb = None
 61 |         self.ioloop = None
 62 |         self.http_client = AsyncHTTPClient()
 63 | 
 64 |     def update_state(self, state, value, message=None):
 65 |         """Update state of particular event"""
 66 |         # if state is new, init it as False
 67 |         if state not in self.STATES:
 68 |             self.STATES[state] = False
 69 |         # compare new value to old value
 70 |         if value is not self.STATES[state]:
 71 |             _logger.debug("State change - %s: %s", state, value)
 72 |         # notify, if state changed from False to True
 73 |         if value and not self.STATES[state]:
 74 |             self.state_change_callback(state, message)
 75 |         # save the new value
 76 |         self.STATES[state] = value
 77 | 
 78 |     def resume_periodic_cb(self):
 79 |         _logger.info("Crawler resumed")
 80 |         self.periodic_cb.start()
 81 | 
 82 |     @coroutine
 83 |     def run(self):
 84 |         """Run a crawler iteration"""
 85 |         progress()
 86 |         try:
 87 |             # request OVH availability API asynchronously
 88 |             resp = yield self.http_client.fetch(self.API_URL,
 89 |                                                 request_timeout=REQUEST_TIMEOUT)
 90 |         except HTTPError as ex:
 91 |             # Internal Server Error
 92 |             self.HTTP_ERRORS.append(ex)
 93 |             if len(self.HTTP_ERRORS) > 5:
 94 |                 if all([e.code == 500 for e in self.HTTP_ERRORS]):
 95 |                     _logger.error("Server continiously returns error 500 and "
 96 |                                   "may be down, check the status manually: %s",
 97 |                                   self.API_URL)
 98 |                 else:
 99 |                     _logger.error("Too many HTTP Errors: %s", self.HTTP_ERRORS)
100 |                 self.HTTP_ERRORS = []
101 |             return
102 |         except Exception as gex:
103 |             # Also catch other errors.
104 |             _logger.error("Socket Error: %s", str(gex))
105 |             return
106 |         if self.HTTP_ERRORS:
107 |             del self.HTTP_ERRORS[:]
108 |         response_json = json.loads(resp.body.decode('utf-8'))
109 |         # check for error
110 |         if not response_json:
111 |             _logger.error("No answer from API: %s", response_json)
112 |             return
113 |         # parse
114 |         for entry in response_json:
115 |             server_type = self.SERVER_TYPES.get(entry['hardware'])
116 |             if not server_type:
117 |                 continue
118 |             available_zones = set([
119 |                 e['datacenter'] for e in entry['datacenters']
120 |                 if e['availability'] not in ['unavailable', 'unknown']])
121 |             _logger.debug('%s is available in %s', server_type, available_zones)
122 |             
123 |             for region, places in self.REGIONS.items():
124 |                 server_available = bool(available_zones.intersection(places))
125 |                 state_id = '%s_available_in_%s' % (server_type.lower(),
126 |                                                    region.lower())
127 |                 message = {
128 |                     'title': "{0} is available".format(server_type),
129 |                     'text': "Server {server} is available in {region}".format(
130 |                         server=server_type, region=region.capitalize()),
131 |                     'url': "https://www.kimsufi.com/en/servers.xml"
132 |                 }
133 |                 if 'sys' in entry['hardware'] or 'bk' in entry['hardware']:
134 |                     message['url'] = 'http://www.soyoustart.com/de/essential-server/'
135 |                 self.update_state(state_id, server_available, message)
136 | 
137 | 
138 | def bell():
139 |     sys.stdout.write('\a')
140 |     sys.stdout.flush()
141 | 
142 | 
143 | def progress():
144 |     sys.stdout.write('.')
145 |     sys.stdout.flush()
146 | 
147 | 
148 | if __name__ == "__main__":
149 |     # load user config
150 |     _CONFIG = parse_json_file(os.path.join(CURRENT_PATH, 'config.json'))
151 | 
152 |     # init notifier
153 |     _NOTIFIERS = {
154 |         'pushover': 'notifiers.pushover_notifier.PushoverNotifier',
155 |         'email': 'notifiers.email_notifier.EmailNotifier',
156 |         'osx': 'notifiers.osx_notifier.OSXNotifier',
157 |         'popup': 'notifiers.popup_notifier.PopupNotifier',
158 |         'popup_pywin': 'notifiers.popup_pywin_notifier.PopupPywinNotifier',
159 |         'smsapi': 'notifiers.smsapi_notifier.SmsApiNotifier',
160 |         'xmpp': 'notifiers.xmpp_notifier.XMPPNotifier',
161 |         'pushbullet': 'notifiers.pushbullet_notifier.PushbulletNotifier',
162 |         'file': 'notifiers.file_notifier.FileNotifier',
163 |         'freemobile': 'notifiers.freemobile_notifier.FreemobileNotifier',
164 |         'telegram': 'notifiers.telegram_notifier.TelegramNotifier',
165 |     }
166 |     # Select notifier, 'email' by default
167 |     if 'notifier' not in _CONFIG:
168 |         _logger.warning("No notifier selected in config, 'email' will be used")
169 |         _CONFIG['notifier'] = 'email'
170 |     # Instantiate notifier class dynamically
171 |     try:
172 |         _NOTIFIER_PATH = _NOTIFIERS[_CONFIG['notifier']]
173 |         _NOTIFIER_FILE, _NOTIFIER_CLASSNAME = _NOTIFIER_PATH.rsplit('.', 1)
174 |         _NOTIFIER_MODULE = importlib.import_module(_NOTIFIER_FILE)
175 |         NOTIFIER = getattr(_NOTIFIER_MODULE, _NOTIFIER_CLASSNAME)(_CONFIG)
176 |     except Exception as ex:
177 |         _logger.exception("Notifier loading failed, check config for errors")
178 |         sys.exit(1)
179 | 
180 |     # prepare states tracked by the user
181 |     TRACKED_STATES = []
182 |     for server in _CONFIG['servers']:
183 |         TRACKED_STATES.append(
184 |             '%s_available_in_%s' % (server.lower(), _CONFIG['region'].lower()))
185 |     _logger.info('Tracking states: %s', TRACKED_STATES)
186 | 
187 |     # define state-change callback to notify the user
188 |     def state_changed(state, message=None):
189 |         """Trigger notifications"""
190 |         message = message or {}
191 |         if state in TRACKED_STATES:
192 |             _logger.info("Will notify: %s", state)
193 |             NOTIFIER.notify(**message)
194 |             bell()
195 | 
196 |     # Check and set request timeout
197 |     REQUEST_TIMEOUT = _CONFIG.get('request_timeout', 30)
198 | 
199 |     # Init the crawler
200 |     crawler = Crawler(state_change_callback=state_changed)
201 |     crawler.periodic_cb = tornado.ioloop.PeriodicCallback(
202 |         crawler.run, crawler.interval * 1000)
203 |     crawler.periodic_cb.start()
204 | 
205 |     # start the IOloop
206 |     _logger.info("Starting main loop")
207 |     crawler.ioloop = tornado.ioloop.IOLoop.instance()
208 |     try:
209 |         crawler.ioloop.start()
210 |     except KeyboardInterrupt:
211 |         _logger.info("Terminated by user. Bye.")
212 |         sys.exit(0)
213 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | app:
2 |    build: .
3 | 


--------------------------------------------------------------------------------
/mapping/datacenters.json:
--------------------------------------------------------------------------------
1 | {
2 |     "bhs": "Beauharnois, Canada (Americas)",
3 |     "gra": "Gravelines, France",
4 |     "rbx": "Roubaix, France (Western Europe)",
5 |     "sbg": "Strasbourg, France (Central Europe)",
6 |     "par": "Paris, France"
7 | }
8 | 


--------------------------------------------------------------------------------
/mapping/regions.json:
--------------------------------------------------------------------------------
1 | {
2 |     "canada": ["bhs"],
3 |     "europe": ["gra", "rbx", "sbg", "par"]
4 | }
5 | 


--------------------------------------------------------------------------------
/mapping/server_types.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "160sk4": "KS-4A",
 3 |     "160sk41": "KS-4B",
 4 |     "160sk42": "KS-4C",
 5 |     "160sk3": "KS-3A",
 6 |     "160sk31": "KS-3B",
 7 |     "160sk32": "KS-3C",
 8 |     "160sk2": "KS-2A",
 9 |     "160sk21": "KS-2B",
10 |     "160sk22": "KS-2C",
11 |     "160sk23": "KS-2D",
12 |     "161sk2": "KS-2E",
13 |     "1804sk12": "KS-1",
14 |     "1801sk13": "KS-2",
15 |     "1801sk14": "KS-3",
16 |     "1801sk15": "KS-4",
17 |     "1801sk16": "KS-5",
18 |     "1801sk17": "KS-6",
19 |     "1801sk18": "KS-7",
20 |     "1801sk19": "KS-8",
21 |     "1801sk20": "KS-9",
22 |     "1801sk21": "KS-10",
23 |     "1801sk22": "KS-11",
24 |     "1801sk23": "KS-12",
25 | 
26 |     "141game1": "GAME-1",
27 |     "141game2": "GAME-2",
28 | 
29 |     "142sys4":  "SYS-IP-1",
30 |     "142sys5":  "SYS-IP-2",
31 |     "142sys8":  "SYS-IP-4",
32 |     "142sys6":  "SYS-IP-5",
33 |     "142sys10": "SYS-IP-5S",
34 |     "142sys7":  "SYS-IP-6",
35 |     "142sys9":  "SYS-IP-6S",
36 | 
37 |     "143sys13": "E3-SSD-1",
38 |     "143sys10": "E3-SSD-2",
39 |     "143sys11": "E3-SSD-3",
40 |     "143sys12": "E3-SSD-4",
41 |     
42 |     "143sys4":  "E3-SAT-1",
43 |     "143sys1":  "E3-SAT-2",
44 |     "143sys2":  "E3-SAT-3",
45 |     "143sys3":  "E3-SAT-4",
46 |     
47 |     "141bk1":   "BK-8T",
48 |     "141bk2":   "BK-24T"
49 | }
50 | 


--------------------------------------------------------------------------------
/notifiers/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MA3STR0/kimsufi-crawler/4c9f1607e120bcb178d3deb5a86a2431d2fab59f/notifiers/__init__.py


--------------------------------------------------------------------------------
/notifiers/base_notifier.py:
--------------------------------------------------------------------------------
 1 | """Generic notifier, to be subclassed"""
 2 | 
 3 | import logging
 4 | _logger = logging.getLogger(__name__)
 5 | 
 6 | 
 7 | class Notifier(object):
 8 |     """Abstract class for notifiers"""
 9 |     def __init__(self, config):
10 |         """Save config and run system check"""
11 |         self.config = config
12 |         self.check_requirements()
13 |         _logger.info("Notification system check passed")
14 | 
15 |     def check_requirements(self):
16 |         """Abstract method to check requirements, config, dependencies etc"""
17 |         raise NotImplementedError
18 | 
19 |     def notify(self, title, text, url=None):
20 |         """Abstract method for notification sending"""
21 |         raise NotImplementedError
22 | 


--------------------------------------------------------------------------------
/notifiers/email_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends email messages through SMTP"""
 2 | 
 3 | import sys
 4 | import logging
 5 | import smtplib
 6 | from email.mime.text import MIMEText
 7 | from email.mime.multipart import MIMEMultipart
 8 | from notifiers.base_notifier import Notifier
 9 | 
10 | _logger = logging.getLogger(__name__)
11 | 
12 | 
13 | class EmailNotifier(Notifier):
14 |     """Notifier class to work with email"""
15 | 
16 |     def __init__(self, config):
17 |         """Override init to make SMTP-settings check"""
18 |         self.use_starttls = config.get('use_starttls', True)
19 |         self.use_ssl = config.get('use_ssl', False)
20 |         self.fromaddr = config['from_email']
21 |         # smtp user may be different from email
22 |         self.fromuser = config.get('from_user', self.fromaddr)
23 |         self.frompwd = config['from_pwd']
24 |         if sys.version_info[0] == 2:
25 |             self.fromuser = self.fromuser.encode('utf-8')
26 |             self.frompwd = self.frompwd.encode('utf-8')
27 |         self.host = config['from_smtp_host']
28 |         self.port = config.get('from_smtp_port',
29 |                                465 if self.use_ssl else 587)
30 |         self.toaddr = config['to_email']
31 |         self.login_required = self.fromuser and config['from_pwd']
32 |         super(EmailNotifier, self).__init__(config)
33 | 
34 |     def check_requirements(self):
35 |         """Logs in to SMTP server to check credentials and settings"""
36 |         if self.use_ssl:
37 |             server = smtplib.SMTP_SSL(self.host, self.port)
38 |         else:
39 |             server = smtplib.SMTP(self.host, self.port)
40 |         server.ehlo()
41 |         if self.use_starttls:
42 |             server.starttls()
43 |             server.ehlo()
44 |         try:
45 |             if self.login_required:
46 |                 server.login(self.fromuser, self.frompwd)
47 |         except Exception as ex:
48 |             _logger.error("Cannot connect to your SMTP account. "
49 |                           "Correct your config and try again. Error details:")
50 |             _logger.error(ex)
51 |             raise
52 |         _logger.info("SMTP server check passed")
53 | 
54 |     def notify(self, title, text, url=None):
55 |         """Send email notification using SMTP"""
56 |         msg = MIMEMultipart()
57 |         msg['From'] = self.fromaddr
58 |         msg['To'] = self.toaddr
59 |         msg['Subject'] = title
60 |         body = text if url else text + '\nURL: ' + url
61 |         msg.attach(MIMEText(body, 'plain'))
62 |         if self.use_ssl:
63 |             server = smtplib.SMTP_SSL(self.host, self.port)
64 |         else:
65 |             server = smtplib.SMTP(self.host, self.port)
66 |         server.ehlo()
67 |         if self.use_starttls:
68 |             server.starttls()
69 |             server.ehlo()
70 |         if self.login_required:
71 |           server.login(self.fromuser, self.frompwd)
72 |         text = msg.as_string()
73 |         server.sendmail(self.fromaddr, self.toaddr, text)
74 | 


--------------------------------------------------------------------------------
/notifiers/file_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that append text to a given file"""
 2 | """
 3 | This notifier should be used with think kind of browser plugin:
 4 | https://chrome.google.com/webstore/detail/distill-web-monitor/inlikjemeeknofckkjolnjbpehgadgge
 5 | (doesn't seem to work directly on KS website)
 6 | You can set up this plugin to be notified with a sound, desktop notifs...
 7 | You can run it somewhere and check the availibility from any web browser!
 8 | """
 9 | 
10 | import os
11 | import datetime
12 | import logging
13 | from notifiers.base_notifier import Notifier
14 | 
15 | _logger = logging.getLogger(__name__)
16 | 
17 | class FileNotifier(Notifier):
18 |     """Notifier class to work with file"""
19 | 
20 |     def __init__(self, config):
21 |         self.file_path = config.get('file_path')
22 |         super(FileNotifier, self).__init__(config)
23 | 
24 |     def check_requirements(self):
25 |         """Check if file exist"""
26 |         if not os.path.isfile(self.file_path):
27 |             _logger.error("File not found")
28 |             _logger.error(ex)
29 |             raise
30 |         _logger.info("File notifier check passed")
31 | 
32 |     def notify(self, title, text, url=None):
33 |         """Update html file"""
34 |         with open(self.file_path, "a") as file:
35 |             ts = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now())
36 |             file.write('<br/>AVAILABLE! - %s' % ts)


--------------------------------------------------------------------------------
/notifiers/freemobile_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends sms to freemobile client using FM notifications API"""
 2 | 
 3 | import logging
 4 | import webbrowser
 5 | from notifiers.base_notifier import Notifier
 6 | 
 7 | _logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | class FreemobileNotifier(Notifier):
11 |     """Notifier class for freemobile sms"""
12 | 
13 |     def __init__(self, config):
14 |         self.username = config.get('freemobile_username', '')
15 |         self.password = config.get('freemobile_key', '')
16 | 
17 |         super(FreemobileNotifier, self).__init__(config)
18 | 
19 |     def check_requirements(self):
20 |         """Check requests library is installed"""
21 |         try:
22 |             __import__('requests')
23 |         except ImportError:
24 |             raise Warning(
25 |                 "requests Python library is required for freemobile notifications. ")
26 | 
27 |     def notify(self, title, text, url=False):
28 |         """"""
29 |         import requests
30 | 
31 |         requests.get("https://smsapi.free-mobile.fr/sendmsg?user={user}&pass={key}&msg={msg}"
32 |             .format(user=self.username, key=self.password, msg='Kimsufi: '+text))
33 | 


--------------------------------------------------------------------------------
/notifiers/osx_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends Mac OSX desktop notifications"""
 2 | 
 3 | import logging
 4 | import subprocess
 5 | import platform
 6 | import os
 7 | from notifiers.base_notifier import Notifier
 8 | 
 9 | _logger = logging.getLogger(__name__)
10 | 
11 | 
12 | class OSXNotifier(Notifier):
13 |     """Notifier class for OSX desktop notifications"""
14 | 
15 |     def check_requirements(self):
16 |         """Check platform and notifier command"""
17 |         if platform.system() != 'Darwin':
18 |             raise Warning("You are not running Mac OS X, "
19 |                           "this notification plugin will not work.")
20 |         try:
21 |             subprocess.call(['terminal-notifier', '--version'],
22 |                             stdout=open(os.devnull, 'w'),
23 |                             stderr=subprocess.STDOUT)
24 |         except OSError:
25 |             raise Warning(
26 |                 "terminal-notifier library is required for OSX notifications. "
27 |                 "You can install it by running in terminal:\n"
28 |                 "sudo gem install terminal-notifier")
29 | 
30 |     def notify(self, title, text, url=False):
31 |         """Send Mac-OS-X notification using terminal-notifier"""
32 |         subprocess.call(['terminal-notifier',
33 |                          '-title', title,
34 |                          '-message', text,
35 |                          '-open', url])
36 | 


--------------------------------------------------------------------------------
/notifiers/popup_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that opens a popup window"""
 2 | 
 3 | import logging
 4 | import webbrowser
 5 | from notifiers.base_notifier import Notifier
 6 | 
 7 | _logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | class PopupNotifier(Notifier):
11 |     """Notifier class for popup windows"""
12 | 
13 |     def check_requirements(self):
14 |         """Check platform and notifier command"""
15 |         try:
16 |             __import__('easygui')
17 |         except ImportError:
18 |             raise Warning(
19 |                 "easygui Python library is required for popup notifications. "
20 |                 "You can install it by running in terminal:\n"
21 |                 "sudo easy_install easygui")
22 | 
23 |     def notify(self, title, text, url=False):
24 |         """Open popup notification window with easygui"""
25 |         import easygui
26 |         if easygui.ccbox(text, title, ["Open web page", "Ignore"]):
27 |             webbrowser.open_new_tab(url)
28 | 


--------------------------------------------------------------------------------
/notifiers/popup_pywin_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that opens a pywin popup window"""
 2 | 
 3 | import logging
 4 | import webbrowser
 5 | from notifiers.base_notifier import Notifier
 6 | 
 7 | _logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | class PopupPywinNotifier(Notifier):
11 |     """Notifier class for popup windows, a windows specific version"""
12 | 
13 |     def check_requirements(self):
14 |         """Check platform and notifier command"""
15 |         lib_name = "win32api"
16 |         try:
17 |             __import__(lib_name)
18 |         except ImportError:
19 |             raise Warning(
20 |                 "{} Python library is required".format(lib_name))
21 | 
22 |     def notify(self, title, text, url=False):
23 |         """Open popup notification window with easygui"""
24 |         import win32api
25 |         # 0x00001000 -- Value represents MB_SYSTEMMODAL
26 |         # This is to allow for the messagebox to sit over every window
27 |         # Something that is not possible using easygui (as far as I'm aware)
28 |         if win32api.MessageBox(0, text, title, 0x00001000) == 1:
29 |             webbrowser.open_new_tab(url)
30 | 


--------------------------------------------------------------------------------
/notifiers/pushbullet_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends messages through Pushbullet"""
 2 | 
 3 | import logging
 4 | from pushbullet import Pushbullet
 5 | from notifiers.base_notifier import Notifier
 6 | 
 7 | _logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | class PushbulletNotifier(Notifier):
11 |     """Notifier class to work with Pushbullet"""
12 | 
13 |     def __init__(self, config):
14 |         """Override init to check settings"""
15 |         self.pushbullet_apikey = config['pushbullet_apikey']
16 |         super(PushbulletNotifier, self).__init__(config)
17 | 
18 |     def check_requirements(self):
19 |         try:
20 |             pb = Pushbullet(self.pushbullet_apikey)
21 |         except Exception as ex:
22 |             _logger.error("Cannot connect to your Pushbullet account. "
23 |                           "Correct your config and try again. Error details:")
24 |             _logger.error(ex)
25 |             raise
26 |         _logger.info("Pushbullet server check passed")
27 | 
28 |     def notify(self, title, text, url=None):
29 |         pb = Pushbullet(self.pushbullet_apikey)
30 |         push = pb.push_link(text, url)


--------------------------------------------------------------------------------
/notifiers/pushover_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends messages through Pushover"""
 2 | 
 3 | import logging
 4 | from chump import Application
 5 | from notifiers.base_notifier import Notifier
 6 | 
 7 | _logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | class PushoverNotifier(Notifier):
11 |     """Notifier class to work with Pushover"""
12 | 
13 |     def __init__(self, config):
14 |         """Override init to make Pushover-settings check"""
15 |         self.application_id = config.get('pushover_application_id')
16 |         self.user_id = config.get('pushover_user_id')
17 |         self.priority = config.get('pushover_priority', 0)
18 |         super(PushoverNotifier, self).__init__(config)
19 | 
20 |     def check_requirements(self):
21 |         app = Application(self.application_id)
22 | 
23 |         try:
24 |             user = app.get_user(self.user_id)
25 |             if not user.is_authenticated:
26 |                 raise ValueError("User could not be authenticated")
27 |         except Exception as ex:
28 |             _logger.error("Cannot connect to your Pushover account. "
29 |                           "Correct your config and try again. Error details:")
30 |             _logger.error(ex)
31 |             raise
32 |         _logger.info("Pushover server check passed")
33 | 
34 |     def notify(self, title, text, url=None):
35 |         app = Application(self.application_id)
36 |         user = app.get_user(self.user_id)
37 | 
38 |         message = user.send_message(message=text,
39 |                                     title=title,
40 |                                     url=url,
41 |                                     priority=self.priority)
42 | 


--------------------------------------------------------------------------------
/notifiers/smsapi_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends messages through smsapi.pl, Polish SMS service"""
 2 | 
 3 | import logging
 4 | 
 5 | from notifiers.base_notifier import Notifier
 6 | from smsapi import SmsApi
 7 | 
 8 | 
 9 | _logger = logging.getLogger(__name__)
10 | 
11 | 
12 | class SmsApiNotifier(Notifier):
13 |     """Notifier class to work with email"""
14 | 
15 |     def __init__(self, config):
16 |         """Override init to check settings"""
17 |         self.smsapi_username = config['smsapi_username']
18 |         self.smsapi_password = config['smsapi_password']
19 |         self.smsapi_recipient = config['smsapi_recipient']
20 | 
21 |         super(SmsApiNotifier, self).__init__(config)
22 | 
23 |     def check_requirements(self):
24 |         """Log in to smsapi and check credentials and settings"""
25 |         sms = SmsApi(self.smsapi_username, self.smsapi_password)
26 |         try:
27 |             total_points = sms.get_points()['points']
28 |         except Exception as ex:
29 |             _logger.error("Cannot connect to your SMSAPI account. "
30 |                           "Correct your config and try again. Error details:")
31 |             _logger.error(ex)
32 |             raise
33 |         _logger.info("SMSAPI connected. You have %s points." % total_points)
34 | 
35 |     def notify(self, title, text, url=False):
36 |         """Send sms notification using smsapi.pl"""
37 |         body = text + ' - ' + url
38 |         sms = SmsApi(self.smsapi_username, self.smsapi_password)
39 |         sms.send_sms(body, recipient=self.smsapi_recipient)
40 |         _logger.info("SMSAPI sent: [%s] %s" % (self.smsapi_recipient, body))
41 | 


--------------------------------------------------------------------------------
/notifiers/telegram_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends a message to the specified chat through Telegram"""
 2 | 
 3 | import logging
 4 | from notifiers.base_notifier import Notifier
 5 | import telegram
 6 | from telegram.error import TelegramError
 7 | 
 8 | _logger = logging.getLogger(__name__)
 9 | 
10 | 
11 | class TelegramNotifier(Notifier):
12 |     """Notifier class for Telegram"""
13 | 
14 |     def __init__(self, config):
15 |         self.chat_id = config.get('telegram_chat_id')
16 |         token = config.get('telegram_token')
17 |         self.bot = telegram.Bot(token=token)
18 |         super().__init__(config)
19 | 
20 |     def check_requirements(self):
21 |         try:
22 |             self.bot.get_me()
23 |             self.bot.send_message(chat_id=self.chat_id, text="Kimsufi Crawler started")
24 |         except TelegramError as te:
25 |             _logger.error("Telegram validation failed: {error}".format(error=te.message))
26 |             raise
27 | 
28 |     def notify(self, title, text, url=None):
29 |         try:
30 |             self.bot.send_message(chat_id=self.chat_id, text=text)
31 |         except TelegramError as te:
32 |             _logger.error("Something went wrong sending the message to Telegram:")
33 |             _logger.error(te)
34 | 


--------------------------------------------------------------------------------
/notifiers/xmpp_notifier.py:
--------------------------------------------------------------------------------
 1 | """Notifier that sends messages through XMPP/JABBER"""
 2 | 
 3 | import logging
 4 | 
 5 | from notifiers.base_notifier import Notifier
 6 | import xmpp
 7 | 
 8 | 
 9 | _logger = logging.getLogger(__name__)
10 | 
11 | 
12 | class XMPPNotifier(Notifier):
13 |     """Notifier class to work with xmpp"""
14 | 
15 |     def __init__(self, config):
16 |         """Override init to check settings"""
17 |         self.xmpp_jid = config['xmpp_jid']
18 |         self.xmpp_password = config['xmpp_password']
19 |         if not isinstance(config['xmpp_recipient'], list):
20 |             self.xmpp_recipient = [ config['xmpp_recipient'] ]
21 |         else:
22 |             self.xmpp_recipient = config['xmpp_recipient']
23 |         self.xmpp_send_test = config.get('xmpp_send_test', False)
24 |         super(XMPPNotifier, self).__init__(config)
25 | 
26 |     def check_requirements(self):
27 |         """Log in to xmpp server and check credentials"""
28 |         jid = xmpp.protocol.JID(self.xmpp_jid)
29 |         try:
30 |             cl = xmpp.Client(jid.getDomain(),debug=[])
31 |             if not cl.connect():
32 |                 raise Exception("Connect failed.")
33 | 
34 |             if not cl.auth(jid.getNode(), self.xmpp_password, resource=jid.getResource()):
35 |                 raise Exception("Authentication failed.")
36 | 
37 |             if self.xmpp_send_test:
38 |                 for recipient in self.xmpp_recipient:
39 |                     if not cl.send(xmpp.protocol.Message(recipient, 'Kimsufi Crawler: Initial Test Message')):
40 |                         raise Exception("Failed to send message to %s" % (recipient, ))
41 |                     _logger.info("XMPP sent: [%s] %s" % (recipient, 'Kimsufi Crawler: Initial Test Message'))
42 | 
43 |         except Exception as ex:
44 |             _logger.error("Cannot connect/auth to XMPP Server. "
45 |                           "Correct your config and try again. Error details:")
46 |             _logger.error(ex)
47 |             raise
48 |         _logger.info("XMPP connected.")
49 | 
50 |     def notify(self, title, text, url=False):
51 |         """Send XMPP notification"""
52 |         body = text + ' - ' + url
53 |         jid = xmpp.protocol.JID(self.xmpp_jid)
54 |         cl = xmpp.Client(jid.getDomain(),debug=[])
55 |         if not cl.connect():
56 |             raise Exception("Connect failed.")
57 | 
58 |         if not cl.auth(jid.getNode(), self.xmpp_password, resource=jid.getResource()):
59 |             raise Exception("Authentication failed.")
60 | 
61 |         for recipient in self.xmpp_recipient:
62 |             if not cl.send(xmpp.protocol.Message(recipient, body)):
63 |                 _logger.error("Failed to send message to %s" % (recipient, ))
64 |             else:
65 |                 _logger.info("XMPP sent: [%s] %s" % (recipient, body))
66 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | tornado>=4.0.0
 2 | 
 3 | # optional
 4 | pushbullet.py>=0.8.0
 5 | chump>=1.5.0
 6 | xmpppy
 7 | easygui
 8 | requests
 9 | python-telegram-bot
10 | 


--------------------------------------------------------------------------------