.+?">(.+?).+?
Uploaded (.+?), Size (.+?), .+?.+?(\d+?) | .+?
(\d+?) | '
103 | for tr in re.compile(regex, re.DOTALL).findall(response):
104 | result = re.compile(regex_tr, re.DOTALL).findall(tr)
105 | self.debug(tr + ' -> ' + str(result))
106 | if result:
107 | (title, link, date, size, seeds, leechers) = result[0]
108 |
109 | info = {}
110 | num = num - 1
111 | original_title = None
112 | year = 0
113 | img = ''
114 | size = size.replace(' ', ' ')
115 | date = self.stripHtml(date.replace(' ', ' '))
116 |
117 | # info
118 |
119 | info['label'] = info['title'] = self.unescape(title)
120 | info['link'] = link
121 | self.log(info['link'])
122 | info['plot'] = info['title'] + '\r\n[I](%s) [S/L: %s/%s] [/I]\r\n%s' % (size, seeds, leechers, date)
123 | contentList.append((
124 | int(int(self.sourceWeight) * (int(num))),
125 | original_title, title, int(year), img, info,
126 | ))
127 | return contentList
128 |
--------------------------------------------------------------------------------
/resources/utorrent/dopal/debug.py:
--------------------------------------------------------------------------------
1 | # File: debug.py
2 | # Library: DOPAL - DO Python Azureus Library
3 | #
4 | # This program is free software; you can redistribute it and/or modify
5 | # it under the terms of the GNU General Public License as published by
6 | # the Free Software Foundation; version 2 of the License.
7 | #
8 | # This program is distributed in the hope that it will be useful,
9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 | # GNU General Public License for more details ( see the COPYING file ).
12 | #
13 | # You should have received a copy of the GNU General Public License
14 | # along with this program; if not, write to the Free Software
15 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 |
17 | '''
18 | Contains functions and objects useful for debugging DOPAL.
19 | '''
20 |
21 |
22 | class DebugObject(object):
23 | pass
24 |
25 |
26 | class LinkDebugObject(DebugObject):
27 | pass
28 |
29 |
30 | class ErrorLinkDebug(LinkDebugObject):
31 | def __init__(self, cgi_path, error):
32 | self.cgi_path = cgi_path
33 | self.error = error
34 |
35 |
36 | class OutgoingExchangeDebug(LinkDebugObject):
37 | def __init__(self, cgi_path, data_to_send):
38 | self.cgi_path = cgi_path
39 | self.data_to_send = data_to_send
40 |
41 |
42 | class ConnectionExchangeDebug(LinkDebugObject):
43 | def __init__(self, cgi_path, data_sent, data_received):
44 | self.cgi_path = cgi_path
45 | self.data_sent = data_sent
46 | self.data_received = data_received
47 |
48 |
49 | class ConnectionDebugObject(DebugObject):
50 | pass
51 |
52 |
53 | class MethodRequestDebug(ConnectionDebugObject):
54 | def __init__(self, object_id, request_method):
55 | self.object_id = object_id
56 | self.request_method = request_method
57 |
58 |
59 | class MethodResponseDebug(ConnectionDebugObject):
60 | def __init__(self, object_id, request_method, response):
61 | self.object_id = object_id
62 | self.request_method = request_method
63 | self.response = response
64 |
65 |
66 | def print_everything(debug_object):
67 | if not isinstance(debug_object, LinkDebugObject):
68 | return
69 |
70 | print
71 | print '---------------'
72 | print
73 |
74 | if isinstance(debug_object, OutgoingExchangeDebug):
75 | print 'Sending to "%s"' % debug_object.cgi_path
76 | print
77 | print debug_object.data_to_send
78 |
79 | elif isinstance(debug_object, ConnectionExchangeDebug):
80 | print 'Recieved from "%s"' % debug_object.cgi_path
81 | print
82 | print debug_object.data_received
83 |
84 | elif isinstance(debug_object, ErrorLinkDebug):
85 | error = debug_object.error
86 | print 'Error from "%s"' % debug_object.cgi_path
87 | print
88 | print '%s: %s' % (error.__class__.__name__, error)
89 |
90 | print
91 | print '---------------'
92 | print
93 |
94 |
95 | def print_everything_with_stack(debug_object):
96 | if isinstance(debug_object, OutgoingExchangeDebug):
97 | import traceback
98 |
99 | print
100 | print '---------------'
101 | print
102 | print 'Stack trace of request:'
103 | traceback.print_stack()
104 | print
105 | print '---------------'
106 | print
107 | print_everything(debug_object)
108 |
109 |
110 | def print_method(debug_object):
111 | from dopal.utils import make_short_object_id as _sid
112 |
113 | if isinstance(debug_object, MethodRequestDebug):
114 | print
115 | print '---------------'
116 | print ' Object:', debug_object.object_id,
117 | if debug_object.object_id is not None:
118 | print "[sid=%s]" % _sid(debug_object.object_id),
119 | print
120 | print ' Method:', debug_object.request_method
121 | print
122 | elif isinstance(debug_object, MethodResponseDebug):
123 | import dopal.core
124 |
125 | if isinstance(debug_object.response, dopal.core.ErrorResponse):
126 | print ' Response Type: ERROR'
127 | print ' Response Data:', debug_object.response.response_data
128 | elif isinstance(debug_object.response, dopal.core.AtomicResponse):
129 | print ' Response Type: VALUE'
130 | print ' Response Data:', debug_object.response.response_data
131 | elif isinstance(debug_object.response, dopal.core.NullResponse):
132 | print ' Response Type: NULL / EMPTY'
133 | print ' Response Data: None'
134 | elif isinstance(debug_object.response, dopal.core.StructuredResponse):
135 | print ' Response Type: STRUCTURE'
136 | print ' Response Data:',
137 |
138 | obj_id = debug_object.response.get_object_id()
139 | if obj_id is not None:
140 | print 'Object [id=%s, sid=%s]' % (obj_id, _sid(obj_id))
141 | else:
142 | print 'Non-object value'
143 | print '---------------'
144 | print
145 |
146 |
147 | def print_method_with_stack(debug_object):
148 | if isinstance(debug_object, MethodResponseDebug):
149 | import traceback
150 |
151 | print
152 | print '---------------'
153 | print
154 | print 'Stack trace of request:'
155 | traceback.print_stack()
156 | print
157 | print '---------------'
158 | print
159 | print_method(debug_object)
160 |
161 |
162 | class DebugGrabber(object):
163 | debug_object = None
164 |
165 | def get_in(self):
166 | if self.debug_object is None:
167 | raise Exception, "not captured any data yet"
168 | return self.debug_object.data_sent
169 |
170 | def get_out(self):
171 | if self.debug_object is None:
172 | raise Exception, "not captured any data yet"
173 | return self.debug_object.data_received
174 |
175 | def __call__(self, debug_object):
176 | if isinstance(debug_object, ConnectionExchangeDebug):
177 | self.debug_object = debug_object
178 |
--------------------------------------------------------------------------------
/resources/scrapers/cache.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import os
4 | import time
5 | import pickle
6 | import threading
7 | import zipfile
8 |
9 | import xbmc
10 | import xbmcvfs
11 | import xbmcgui
12 | import Localization
13 | from net import HTTP
14 |
15 | try:
16 | from sqlite3 import dbapi2 as sqlite
17 | except:
18 | from pysqlite2 import dbapi2 as sqlite
19 |
20 | rtrCache_lock = threading.RLock()
21 |
22 |
23 | class Cache:
24 | def __init__(self, name, version, expire=0, size=0, step=100):
25 | self.name = name
26 | self.version = version
27 | self._connect()
28 | if expire:
29 | self.expire(expire)
30 | if size:
31 | self.size(size, step)
32 |
33 | def get(self, token, callback, *param):
34 | cur = self.db.cursor()
35 | cur.execute('select expire,data from cache where id=? limit 1', (token,))
36 | row = cur.fetchone()
37 | cur.close()
38 |
39 | if row:
40 | if row[0] and row[0] < int(time.time()):
41 | pass
42 | else:
43 | try:
44 | obj = pickle.loads(row[1])
45 | except:
46 | pass
47 | else:
48 | return obj
49 |
50 | response = callback(*param)
51 |
52 | if response[0]:
53 | obj = sqlite.Binary(pickle.dumps(response[1]))
54 | curtime = int(time.time())
55 | cur = self.db.cursor()
56 | if isinstance(response[0], bool):
57 | cur.execute('replace into cache(id,addtime,expire,data) values(?,?,?,?)', (token, curtime, None, obj))
58 | else:
59 | cur.execute('replace into cache(id,addtime,expire,data) values(?,?,?,?)',
60 | (token, curtime, curtime + response[0], obj))
61 | self.db.commit()
62 | cur.close()
63 |
64 | return response[1]
65 |
66 | def expire(self, expire):
67 | # with rtrCache_lock:
68 | cur = self.db.cursor()
69 | cur.execute('delete from cache where addtime.
19 | '''
20 | import Content
21 | from BeautifulSoup import BeautifulSoup
22 |
23 |
24 | class SWESUB(Content.Content):
25 | category_dict = {
26 | 'tvshows': ('TV Shows', '/senaste-tv-serier/', {'page': '/senaste-tv-serier/?page=%d',
27 | 'increase': 1, 'second_page': 2, }),
28 | 'movies': ('Movies', '/senaste-filmer/'),
29 | # , {'page': '/senaste-filmer/?page=%d', 'increase': 1, 'second_page': 2,}),
30 | 'genre': {'genre': 'by Genre',
31 | 'action': ('Action', '/action/', {'page': '/action/?page=%d', 'increase': 1, 'second_page': 2, }),
32 | 'adventure': (
33 | 'Adventure', '/aventyr/', {'page': '/aventyr/?page=%d', 'increase': 1, 'second_page': 2, }),
34 | 'animation': (
35 | 'Animation', '/animerat/', {'page': '/animerat/?page=%d', 'increase': 1, 'second_page': 2, }),
36 | 'comedy': ('Comedy', '/komedi/', {'page': '/komedi/?page=%d', 'increase': 1, 'second_page': 2, }),
37 | 'crime': ('Crime', '/kriminal/', {'page': '/kriminal/?page=%d', 'increase': 1, 'second_page': 2, }),
38 | 'documentary': (
39 | 'Documentary', '/dokumentar/', {'page': '/dokumentar/?page=%d', 'increase': 1, 'second_page': 2, }),
40 | 'drama': ('Drama', '/drama/', {'page': '/drama/?page=%d', 'increase': 1, 'second_page': 2, }),
41 | 'family': ('Family', '/familj/', {'page': '/familj/?page=%d', 'increase': 1, 'second_page': 2, }),
42 | 'fantasy': ('Fantasy', '/fantasy/', {'page': '/fantasy/?page=%d', 'increase': 1, 'second_page': 2, }),
43 | 'horror': ('Horror', '/skrack/', {'page': '/skrack/?page=%d', 'increase': 1, 'second_page': 2, }),
44 | 'music': ('Music', '/dans/', {'page': '/dans/?page=%d', 'increase': 1, 'second_page': 2, }),
45 | 'musical': ('Musical', '/musikal/', {'page': '/musikal/?page=%d', 'increase': 1, 'second_page': 2, }),
46 | 'romance': (
47 | 'Romance', '/romantik/', {'page': '/romantik/?page=%d', 'increase': 1, 'second_page': 2, }),
48 | 'sci_fi': ('Sci-Fi', '/sci-fi/', {'page': '/sci-fi/?page=%d', 'increase': 1, 'second_page': 2, }),
49 | 'sport': ('Sport', '/sport/', {'page': '/sport/?page=%d', 'increase': 1, 'second_page': 2, }),
50 | 'thriller': (
51 | 'Thriller', '/thriller/', {'page': '/thriller/?page=%d', 'increase': 1, 'second_page': 2, }),
52 | 'war': ('War', '/krig/', {'page': '/krig/?page=%d', 'increase': 1, 'second_page': 2, }),
53 | 'western': ('Western', '/western/', {'page': '/western/?page=%d', 'increase': 1, 'second_page': 2, }),
54 | }
55 | }
56 |
57 | baseurl = "http://swesub.tv"
58 | headers = [('User-Agent',
59 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124' + \
60 | ' YaBrowser/14.10.2062.12061 Safari/537.36'),
61 | ('Referer', 'http://swesub.tv/'), ('Accept-Encoding', 'gzip')]
62 | '''
63 | Weight of source with this searcher provided.
64 | Will be multiplied on default weight.
65 | Default weight is seeds number
66 | '''
67 | sourceWeight = 1
68 |
69 | def isLabel(self):
70 | return True
71 |
72 | def isScrappable(self):
73 | return False
74 |
75 | def isInfoLink(self):
76 | return True
77 |
78 | def isPages(self):
79 | return True
80 |
81 | def isSort(self):
82 | return False
83 |
84 | def isSearchOption(self):
85 | return False
86 |
87 | def get_contentList(self, category, subcategory=None, apps_property=None):
88 | contentList = []
89 | url = self.get_url(category, subcategory, apps_property)
90 |
91 | response = self.makeRequest(url, headers=self.headers)
92 | response = response.decode('iso-8859-1')
93 |
94 | if None != response and 0 < len(response):
95 | if category:
96 | contentList = self.mode(response)
97 | # print str(contentList)
98 | return contentList
99 |
100 | def mode(self, response):
101 | contentList = []
102 | num = 51
103 | Soup = BeautifulSoup(response)
104 | result = Soup.findAll('article', {'class': 'box'})
105 | # print str(result)
106 | for article in result:
107 | # main
108 | info = {}
109 | num = num - 1
110 | original_title = None
111 | year = 0
112 |
113 | div = article.find('div', {'class': 'box-img'})
114 | title = div.find('img').get('alt')
115 | img = div.find('img').get('src')
116 | link = div.find('a').get('href').replace(self.baseurl, '').replace('.html', '')
117 |
118 | # info
119 |
120 | info['label'] = info['title'] = self.unescape(title)
121 | info['link'] = self.baseurl + '/downloads' + link + '/'
122 | info['infolink'] = self.baseurl + link + '.html'
123 |
124 | info['plot'] = article.find('div', {'class': 'item-content'}).text
125 |
126 | contentList.append((
127 | int(int(self.sourceWeight) * (int(num))),
128 | original_title, title, int(year), img, info,
129 | ))
130 | return contentList
131 |
--------------------------------------------------------------------------------
/resources/language/English/strings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Interface Language
4 | Lock Folders View Style
5 | Off
6 | Save Files To Folder (not FAT32)
7 | Use magnet-links
8 | Keep downloaded files
9 | Keep seeding of downloaded files
10 | Upload speed limit MBits/sec (0 - unlimited)
11 | Download speed limit MBits/sec (0 - unlimited)
12 | Use only system libtorrent
13 | Predownload and play next episode
14 | Download metadata for Content Lists
15 | Debug (Developer Mode)
16 | Confluence (by slng)
17 | Transperency (by slng)
18 | Confluence (by DiMartino)
19 | Confluence (by RussakHH)
20 | Enable Search History
21 | python-libtorrent
22 | Ace Stream
23 | P2P Player
24 | Rest Settings in Programms - AceStream Client
25 | Search Timeout
26 | Short (10s)
27 | Normal (20s)
28 | Long (30s)
29 | Predownload subtitles from all folders
30 | Keep seeding after watching
31 | Ask to change storage before play
32 | Delete Russian stuff
33 | Return Russian stuff
34 | Action onClick for torrent
35 | Open Torrent File
36 | Open Context Menu
37 | Download via Torrent-client
38 | Download via python-libtorrent
39 | Sort search results by seeds
40 | Custom search option phrase
41 | Encryption
42 | Search Thread Number
43 | Delete files
44 | Save files
45 | Ask to save
46 | Torrent2HTTP (libtorrent via http)
47 | Proxy for Search
48 | Proxy for Content Lists
49 | SOCKS-proxy IP
50 | SOCKS-proxy Port
51 | None
52 | Anti-zapret
53 | Tor
54 | Immunicity
55 | Max. connections (0 - unlimited)
56 | Use random ports
57 | Port for incoming connections
58 | Pre-buffer amount before start playing (MB)
59 | Autoplay next episode (or ask)
60 | Device configuration
61 | Average/Good PC
62 | Below average PC/router
63 | Storage size at least for auto clear (GB)
64 | Pause video after start
65 | Sort search results
66 | by Seeds
67 | Do not sort
68 | by Name
69 | Do not add to Watched History if played more (%)
70 | Confluence (by safonov_ivan)
71 | Aeon Nox (by joyrider)
72 | pyrrent2http (python-libtorrent via http)
73 | Append size to file name
74 | Enable DHT
75 | Estuary (by DiMartino)
76 | Стиль информации о раздаче
77 | In the end
78 | In the beginning
79 | Second string
80 | Arctic: Zephyr (by xbmc00)
81 | Interface
82 | P2P Network
83 | Advanced
84 | Torrent-client
85 | Hentai (fine-tuning)
86 | Search Window Mode
87 | Enabled, hide old style
88 | Enabled, replace old style
89 | Enabled, optional
90 | Disabled
91 | Search Window Transparent Background
92 | Disable Notifications
93 | Save path
94 | Call dialog
95 | Default
96 | Path
97 | Create subdirectory for scrapper
98 | Host
99 | Port
100 | URL
101 | Login
102 | Password
103 | Torrent Client
104 | URL (only No SSL)
105 | Path replacement (remote only)
106 | Close
107 | Open Settings
108 | Torrent Client Browser
109 | Install Trackers
110 | Trackers Settings
111 | Clear Storage
112 | Choose searcher
113 | You don't have external searcher. Please install it first.
114 |
115 |
--------------------------------------------------------------------------------
/resources/language/spanish/strings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Idioma de la interfaz
4 | Modo vista de carpetas
5 | No fijar
6 | Carpeta en la que guardar archivos (no FAT32)
7 | Utilizar enlaces magnet
8 | Almacenaje de archivos descargados
9 | Seguir compartiendo archivos descargados
10 | Máx. vel. de subida en Mbit/s (0 = ilimitada)
11 | Máx. vel. de descarga en Mbit/s (0 = ilimitada)
12 | Utilizar solo libtorrent
13 | Lanzar descarga de episodio siguiente
14 | Descargar metadatos para listas de contenido
15 | Depuración (Modo desarrollador)
16 | Confluence (de slng)
17 | Transperency (de slng)
18 | Confluence (de DiMartino)
19 | Confluence (de RussakHH)
20 | Habilitar el historial de búsquedas
21 | python-libtorrent
22 | Ace Stream
23 | Reproductor P2P
24 | Restablecer ajustes en Add-ons de programas -- AceStream client
25 | Duración de las búsquedas
26 | Breve (10 s)
27 | Normal (20 s)
28 | Larga (30 s)
29 | Predescargar los subtítulos de todas las carpetas
30 | Seguir compartiendo después de ver el vídeo
31 | Preguntar para cambiar de almacén antes de la descarga
32 | Eliminar material ruso
33 | Restituir material ruso
34 | Efecto de clic sobre torrent
35 | Abrir archivo torrent
36 | Abrir menú contextual
37 | Descargar vía cliente BitTorrent
38 | Descargar vía python-libtorrent
39 | Ordenar resultados por fuentes
40 | Términos para búsqueda personalizada
41 | Cifrado
42 | Número de hilos de búsqueda
43 | Eliminar archivos
44 | Guardar archivos
45 | Preguntar para guardar
46 | Torrent2HTTP (libtorrent vía HTTP)
47 | Proxy
48 | Ninguno
49 | Anti-zapret (antirestricción)
50 | Tor
51 | Inmunidad
52 | Núm. máx. de conexiones (0 = ilimitadas)
53 | Utilizar puertos aleatorios
54 | Puerto para conexiones entrantes
55 | Tamaño de precarga para empezar a reproducir (MB)
56 | Autorreproducir episodio siguiente (o preguntar)
57 | Configuración de dispositivo
58 | Promedio/Buen PC
59 | Por debajo de la media, PC/router
60 | Tamaño mínimo de almacenaje para autoborrado (GB)
61 | Diferir inicio del reproductor
62 | Ordenar resultados de la búsqueda
63 | Por fuentes
64 | No ordenar
65 | Por nombre
66 | Añadir al historial de vistos después del (%)
67 | Confluence (de safonov_ivan)
68 | Aeon Nox (de joyrider)
69 | pyrrent2http (python-libtorrent vía HTTP)
70 | Añadir tamaño a nombre de archivo
71 | Activar protocolo DHT
72 | Estuary (de DiMartino)
73 | Modo presentación de datos
74 | Al final
75 | Al principio
76 | En segunda línea
77 | Arctic: Zephyr (de xbmc00)
78 | Interfaz
79 | Red P2P
80 | Avanzado
81 | Cliente de BitTorrent
82 | Hentai (reajustes)
83 | Modo ventana de búsquedas
84 | Activado ocultar estilo anterior
85 | Activado remplazar estilo anterior
86 | Activado opcional
87 | Desactivado
88 | Fondo transparente de ventana de búsquedas
89 | Desactivar notificaciones
90 | Ruta para guardar
91 | Preguntar
92 | Por defecto
93 | Ruta
94 | Crear subdirectorio para scrapper
95 | Host
96 | Puerto
97 | URL
98 | Usuario
99 | Contraseña
100 | Cliente de BitTorrent
101 | URL (no SSL)
102 | Sustitución de ruta (Solo remoto)
103 | Cerrar
104 | Abrir "Ajustes"
105 | Navegador de cliente BitTorrent
106 | Instalar rastreadores
107 | Ajustes de rastreadores
108 | Vaciar almacén
109 | Seleccionar buscador
110 | No hay buscador externo. Instale uno primero.
111 |
112 |
--------------------------------------------------------------------------------
/resources/language/Russian/strings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 | Язык интерфейса
4 | Удерживать стиль отображения
5 | Отключено
6 | Сохранять файлы в директорию (не FAT32)
7 | Использовать магнет-ссылки
8 | Хранить загруженные файлы
9 | Оставаться на раздаче скачанных файлов
10 | Ограничить скорость раздачи МБит/сек (0 - неограничено)
11 | Ограничить скорость закачки МБит/сек (0 - неограничено)
12 | Использовать только пользовательский libtorrent
13 | Скачать и запустить следующий эпизод
14 | Загружать мета-данные для Списков Медиа
15 | Дебаг (Разработчик)
16 | Confluence (от slng)
17 | Transperency (от slng)
18 | Confluence (от DiMartino)
19 | Confluence (от RussakHH)
20 | Включить Историю Поиска
21 | python-libtorrent
22 | Ace Stream
23 | P2P Проигрыватель
24 | Остальные настройки в "Программы - AceStream Client"
25 | Ожидание ответа сервера при Поиске
26 | Короткое (10с)
27 | Среднее (20с)
28 | Долгое (30с)
29 | Предзакачать и подключить субтитры
30 | Продолжать сидировать после просмотра
31 | Предлагать изменить место хранения
32 | Удалить русские трекеры
33 | Вернуть русские трекеры
34 | Действие при нажатии на торрент
35 | Открыть Торрент Файл
36 | Открыть Контекстное меню
37 | Скачать Торрент-клиентом
38 | Скачать python-libtorrent
39 | Сортировать поиск по количеству раздающих
40 | Дополнительная фраза в вариантах поиска
41 | Шифрование
42 | Количество параллельных потоков поиска
43 | Удалять файлы
44 | Сохранять файлы
45 | Спросить о сохранении
46 | Torrent2HTTP (libtorrent по http)
47 | Прокси для поиска
48 | Прокси для списков медиа
49 | IP SOCKS-прокси
50 | Порт SOCKS-прокси
51 | Не использовать
52 | Anti-zapret
53 | Tor
54 | Immunicity
55 | Макс. соединений (0 - безлимит)
56 | Использовать случайные порты
57 | Порт для входящих соединений
58 | Загружаемый объем файла до начала просмотра (МБ)
59 | Автоматически проиграть следующий эпизод (или предложить)
60 | Конфигурация устройства
61 | Нормальный ПК
62 | Плохой ПК/роутер
63 | Минимальный размер хранилища для очистки (ГБ)
64 | Ставить паузу при старте видео
65 | Сортировать результат поиска
66 | по Количеству раздающих
67 | Не сортировать
68 | по Имени
69 | Не добавлять в История Просмотров если больше (%)
70 | Confluence (от safonov_ivan)
71 | Aeon Nox (от joyrider)
72 | pyrrent2http (python-libtorrent по http)
73 | Добавлять размер к имени файла
74 | Включить DHT
75 | Estuary (от DiMartino)
76 | Стиль информации о раздаче
77 | В конце
78 | В начале
79 | Второй строкой
80 | Arctic: Zephyr (от xbmc00)
81 | Интерфейс
82 | P2P Сеть
83 | Дополнительные
84 | Торрент-клиент
85 | Hentai (тонкая настр.)
86 | Режим работы Окна Поиска
87 | Включен, убрать старый вид
88 | Включен, заменить старый вид
89 | Включен, как опция
90 | Отключен
91 | Прозрачность Окна Поиска
92 | Отключить Уведомления
93 | Директория для сохранения файлов
94 | Вызывать диалог
95 | Задать по умолчанию
96 | Путь к директории
97 | Создавать поддиректорию для скрапера
98 | Хост
99 | Порт
100 | URL
101 | Логин
102 | Пароль
103 | URL (только без SSL)
104 | Торрент-клиент
105 | Замена пути (remote only)
106 | Закрыть
107 | Открыть Настройки
108 | Браузер Торрент-клиента
109 | Установить Трекеры
110 | Настройки Трекеров
111 | Очистить хранилище
112 | Выберите трекер
113 | У вас нет внешних серчеров. Пожалуйста, сначало установите их.
114 |
115 |
--------------------------------------------------------------------------------
/AceStream.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | '''
3 | Torrenter v2 plugin for XBMC/Kodi
4 | Copyright (C) 2012-2015 Vadim Skorba v1 - DiMartino v2
5 | https://forums.tvaddons.ag/addon-releases/29224-torrenter-v2.html
6 |
7 | This program is free software: you can redistribute it and/or modify
8 | it under the terms of the GNU General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 |
12 | This program is distributed in the hope that it will be useful,
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | GNU General Public License for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | along with this program. If not, see
.
19 | '''
20 |
21 | import os
22 | import urllib2
23 | import urllib
24 | import hashlib
25 | import re
26 | import base64
27 | from StringIO import StringIO
28 | import zlib
29 |
30 | from functions import file_decode, file_encode
31 | from functions import magnet_alert, log, loadsw_onstop
32 | import xbmcvfs
33 |
34 |
35 | class AceStream:
36 | try:
37 | fpath = os.path.expanduser("~")
38 | pfile = os.path.join(fpath, 'AppData\Roaming\ACEStream\engine', 'acestream.port')
39 | gf = open(pfile, 'r')
40 | aceport = int(gf.read())
41 | gf.close()
42 | log('[AceStream]: aceport - '+str(aceport))
43 | except:
44 | aceport = 62062
45 |
46 | torrentFile = None
47 | magnetLink = None
48 | storageDirectory = ''
49 | torrentFilesDirectory = 'torrents'
50 | startPart = 0
51 | endPart = 0
52 | partOffset = 0
53 | torrentHandle = None
54 | session = None
55 | downloadThread = None
56 | threadComplete = False
57 | lt = None
58 |
59 | def __init__(self, storageDirectory='', torrentFile='', torrentFilesDirectory='torrents'):
60 | try:
61 | from ASCore import TSengine as tsengine
62 |
63 | log('Imported TSengine from ASCore')
64 | except Exception, e:
65 | log('Error importing TSengine from ASCore. Exception: ' + str(e))
66 | return
67 |
68 | self.TSplayer = tsengine()
69 | del tsengine
70 | self.torrentFilesDirectory = torrentFilesDirectory
71 | self.storageDirectory = storageDirectory
72 | _path = os.path.join(self.storageDirectory, self.torrentFilesDirectory) + os.sep
73 | if re.match("^magnet\:.+$", torrentFile):
74 | torrentFile = self.magnetToTorrent(torrentFile)
75 | if not xbmcvfs.exists(_path):
76 | xbmcvfs.mkdirs(_path)
77 | if xbmcvfs.exists(torrentFile):
78 | self.torrentFile = torrentFile
79 | content = xbmcvfs.File(torrentFile, "rb").read()
80 | self.torrentFileInfo = self.TSplayer.load_torrent(base64.b64encode(content), 'RAW')
81 |
82 | def __exit__(self):
83 | self.TSplayer.end()
84 | loadsw_onstop() # Reload Search Window
85 |
86 | def play_url_ind(self, ind, label, icon):
87 | self.TSplayer.play_url_ind(int(ind), label, str(icon), '')
88 |
89 | def saveTorrent(self, torrentUrl):
90 | if re.match("^magnet\:.+$", torrentUrl):
91 | torrentFile = self.magnetToTorrent(torrentUrl)
92 | content = xbmcvfs.File(file_decode(torrentFile), "rb").read()
93 | else:
94 | torrentFile = self.storageDirectory + os.sep + self.torrentFilesDirectory + os.sep + self.md5(
95 | torrentUrl) + '.torrent'
96 | try:
97 | if not re.match("^[htps]+?://.+$|^://.+$", torrentUrl):
98 | log('xbmcvfs.File for %s' % torrentUrl)
99 | content = xbmcvfs.File(torrentUrl, "rb").read()
100 | else:
101 | log('request for %s' % torrentUrl)
102 | content = self.makeRequest(torrentUrl)
103 |
104 | localFile = xbmcvfs.File(torrentFile, "w+b")
105 | localFile.write(content)
106 | localFile.close()
107 | except Exception, e:
108 | log('Unable to save torrent file from "' + torrentUrl + '" to "' + torrentFile +
109 | '" in Torrent::saveTorrent' + '. Exception: ' + str(e))
110 | return
111 | if xbmcvfs.exists(torrentFile):
112 | self.torrentFile = torrentFile
113 | self.torrentFileInfo = self.TSplayer.load_torrent(base64.b64encode(content), 'RAW')
114 | return self.torrentFile
115 |
116 | def makeRequest(self, torrentUrl):
117 | torrentUrl = re.sub('^://', 'http://', torrentUrl)
118 | x = re.search("://(.+?)/|://(.+?)$", torrentUrl)
119 | if x:
120 | baseurl = x.group(1) if x.group(1) else x.group(2)
121 | else:
122 | baseurl =''
123 |
124 | headers = [('User-Agent',
125 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 YaBrowser/14.10.2062.12061 Safari/537.36'),
126 | ('Referer', 'http://%s/' % baseurl), ('Accept-encoding', 'gzip'), ]
127 |
128 | opener = urllib2.build_opener()
129 | opener.addheaders = headers
130 | result = opener.open(torrentUrl)
131 | if result.info().get('Content-Encoding') == 'gzip':
132 | buf = StringIO(result.read())
133 | decomp = zlib.decompressobj(16 + zlib.MAX_WBITS)
134 | content = decomp.decompress(buf.getvalue())
135 | else:
136 | content = result.read()
137 | return content
138 |
139 | def magnetToTorrent(self, magnet):
140 | try:
141 | from SkorbaLoader import SkorbaLoader
142 | torrent = SkorbaLoader(self.storageDirectory, magnet)
143 | torrent.magnetToTorrent(magnet)
144 | self.torrentFile = torrent.torrentFile
145 | log('[AceStream][magnetToTorrent]: self.torrentFile '+str(self.torrentFile))
146 | return self.torrentFile
147 | except:
148 | magnet_alert()
149 | exit()
150 |
151 | def getFilePath(self, contentId=0):
152 | fileList = self.getContentList()
153 | for i in fileList:
154 | if i['ind'] == contentId:
155 | return os.path.join(file_encode(self.storageDirectory), i['title'])
156 |
157 | def getContentList(self):
158 | filelist = []
159 | for k, v in self.TSplayer.files.iteritems():
160 | stringdata = {"title": urllib.unquote_plus(k), "ind": int(v)}
161 | filelist.append(stringdata)
162 | return filelist
163 |
164 | def md5(self, string):
165 | hasher = hashlib.md5()
166 | try:
167 | hasher.update(string)
168 | except:
169 | hasher.update(string.encode('utf-8', 'ignore'))
170 | return hasher.hexdigest()
171 |
--------------------------------------------------------------------------------
/resources/contenters/unused/CXZ.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | '''
3 | Torrenter plugin for XBMC
4 | Copyright (C) 2012 Vadim Skorba
5 | vadim.skorba@gmail.com
6 |
7 | This program is free software: you can redistribute it and/or modify
8 | it under the terms of the GNU General Public License as published by
9 | the Free Software Foundation, either version 3 of the License, or
10 | (at your option) any later version.
11 |
12 | This program is distributed in the hope that it will be useful,
13 | but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | GNU General Public License for more details.
16 |
17 | You should have received a copy of the GNU General Public License
18 | along with this program. If not, see
.
19 | '''
20 |
21 | import re
22 |
23 | import Content
24 | from BeautifulSoup import BeautifulSoup
25 | from datetime import date
26 |
27 | def make_category_dict():
28 | category_dict = {
29 | 'movies': ('Movies', '/films/fl_foreign_hight/?'),
30 | 'rus_movies': ('Russian Movies', '/films/fl_our_hight/?'),
31 | 'tvshows': ('TV Shows', '/serials/fl_hight/?'),
32 | 'cartoons': ('Cartoons', '/cartoons/fl_hight/?'),
33 | 'anime': ('Anime', '/cartoons/cartoon_genre/anime/?'),
34 | 'hot': ('Most Recent', '/films/fl_hight/?'),
35 | 'top': ('Top 250 Movies', '/films/fl_hight/?sort=popularity&'),
36 | 'genre': {'genre': 'by Genre',
37 | 'action': ('Action', '/films/film_genre/bojevik/?'),
38 | 'adventure': ('Adventure', '/films/film_genre/priklucheniya/?'),
39 | 'biography': ('Biography', '/films/film_genre/biografiya/?'),
40 | 'comedy': ('Comedy', '/films/film_genre/komediya/?'),
41 | 'crime': ('Crime', '/films/film_genre/detektiv/?'),
42 | 'documentary': ('Documentary', '/films/film_genre/dokumentalnyj/?'),
43 | 'drama': ('Drama', '/films/film_genre/drama/?'),
44 | 'erotika': ('Adult', '/films/film_genre/erotika/?'),
45 | 'family': ('Family', '/films/film_genre/semejnyj/?'),
46 | 'fantasy': ('Fantasy', '/films/film_genre/fentezi/?'),
47 | 'film_noir': ('Film-Noir', '/films/film_genre/nuar/?'),
48 | 'history': ('History', '/films/film_genre/istoriya/?'),
49 | 'horror': ('Horror', '/films/film_genre/uzhasy/?'),
50 | 'kids': ('For Kids', '/films/film_genre/detskij/?'),
51 | 'musical': ('Musical', '/films/film_genre/muzikl/?'),
52 | 'mystery': ('Mystery', '/films/film_genre/mistika/?'),
53 | 'romance': ('Romance', '/films/film_genre/melodrama/?'),
54 | 'sci_fi': ('Sci-Fi', '/films/film_genre/fantastika/?'),
55 | 'short': ('Short', '/films/film_genre/korotkometrazhka/?'),
56 | 'thriller': ('Thriller', '/films/film_genre/triller/?'),
57 | 'war': ('War', '/films/film_genre/vojennyj/?'),
58 | 'western': ('Western', '/films/film_genre/vestern/?'),
59 | }
60 | }
61 |
62 | for category in category_dict.keys():
63 | if isinstance(category_dict.get(category), dict):
64 | for subcategory in category_dict.get(category).keys():
65 | if subcategory != category:
66 | x = category_dict[category][subcategory]
67 | category_dict[category][subcategory] = (
68 | x[0], x[1] + 'view=list', {'page': x[1] + 'view=list&page=%d', 'increase': 1, 'second_page': 1})
69 | if not isinstance(category_dict.get(category), dict):
70 | x = category_dict[category]
71 | category_dict[category] = (
72 | x[0], x[1] + 'view=list', {'page': x[1] + 'view=list&page=%d', 'increase': 1, 'second_page': 1})
73 |
74 | category_dict['year'] = {'year': 'by Year', }
75 | for y in range(date.today().year, 1970, -1):
76 | category_dict['year'][str(y)] = (str(y), '/films/year/%s/' % str(y),
77 | {'page': '/films/year/%s/' % str(y) + '?view=list&page=%d', 'increase': 1,
78 | 'second_page': 1})
79 |
80 | return category_dict
81 |
82 |
83 | class CXZ(Content.Content):
84 | category_dict = make_category_dict()
85 |
86 | regex_list = []
87 |
88 | baseurl = "http://cxz.to"
89 | headers = [('User-Agent',
90 | 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124' + \
91 | ' YaBrowser/14.10.2062.12061 Safari/537.36'),
92 | ('Referer', baseurl), ('Accept-Encoding', 'gzip'), ('Accept-Language', 'ru,en;q=0.8')]
93 | '''
94 | Weight of source with this searcher provided.
95 | Will be multiplied on default weight.
96 | Default weight is seeds number
97 | '''
98 | sourceWeight = 2
99 |
100 |
101 | def isTracker(self):
102 | return False
103 |
104 | def isSearcher(self):
105 | return False
106 |
107 | def isInfoLink(self):
108 | return False
109 |
110 | def isPages(self):
111 | return True
112 |
113 | def isSearchOption(self):
114 | return True
115 |
116 | def isScrappable(self):
117 | return True
118 |
119 | def get_contentList(self, category, subcategory=None, apps_property=None):
120 | contentList = []
121 | url = self.get_url(category, subcategory, apps_property)
122 |
123 | response = self.makeRequest(url, headers=self.headers)
124 |
125 | if None != response and 0 < len(response):
126 | #print response
127 | if category:
128 | contentList = self.mode(response)
129 | #print str(contentList)
130 | return contentList
131 |
132 | def mode(self, response):
133 | contentList = []
134 | Soup = BeautifulSoup(response)
135 | result = Soup.findAll('div', {'class': 'b-poster-tile '})
136 | num = 0
137 | for tr in result:
138 | #main
139 | info = {}
140 | year = 0
141 | num = num + 1
142 | title = tr.find('span', 'b-poster-tile__title-full').text.strip()
143 | originaltitle = None
144 | year = re.compile('(\d\d\d\d)').findall(tr.find('span', 'b-poster-tile__title-info-items').text)[0]
145 | link = tr.find('a', 'b-poster-tile__link').get('href')
146 | for i in ['/serials/', '/cartoonserials/', '/tvshow/']:
147 | if i in link:
148 | info['tvshowtitle'] = title
149 | break
150 |
151 | img = tr.find('img').get('src')
152 | img = img if img else ''
153 |
154 | #info
155 |
156 | contentList.append((
157 | int(int(self.sourceWeight) * (251 - int(num))),
158 | originaltitle, title, int(year), img, info,
159 | ))
160 | #print result
161 | return contentList
--------------------------------------------------------------------------------
/resources/utorrent/dopal/utils.py:
--------------------------------------------------------------------------------
1 | # File: utils.py
2 | # Library: DOPAL - DO Python Azureus Library
3 | #
4 | # This program is free software; you can redistribute it and/or modify
5 | # it under the terms of the GNU General Public License as published by
6 | # the Free Software Foundation; version 2 of the License.
7 | #
8 | # This program is distributed in the hope that it will be useful,
9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 | # GNU General Public License for more details ( see the COPYING file ).
12 | #
13 | # You should have received a copy of the GNU General Public License
14 | # along with this program; if not, write to the Free Software
15 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 |
17 | '''
18 | General utility functions.
19 | '''
20 |
21 | __pychecker__ = 'unusednames=metaclass_,attr_,no-unreachable'
22 |
23 | # Converts a binary string character into a hexidecimal equivalent.
24 | #
25 | # >>> byte_to_hex_form('N')
26 | # u'4E'
27 | def byte_to_hex_form(byte):
28 | # ord is used to convert the byte into it's numeric equivalent.
29 | # hex will turn the number into a string like "0x5" or "0xa3"
30 | # the slicing will chop off the "0x" prefix.
31 | # zfill will change any single digit values to have a leading 0 character.
32 | # upper will force all the A-F characters to be uppercase.
33 | return unicode(hex(ord(byte))[2:].zfill(2).upper())
34 |
35 |
36 | # Converts a binary string into a hexidecimal equivalent.
37 | #
38 | # >>> string_to_hex_form('xK3-')
39 | # u'784B332D'
40 | def string_to_hex_form(chars):
41 | if not chars:
42 | raise ValueError, "cannot convert empty string"
43 | return ''.join([byte_to_hex_form(char) for char in chars])
44 |
45 |
46 | # Converts a 2 character hexidecimal string into a binary character.
47 | #
48 | # >>> hex_form_to_byte(u'4E')
49 | # 'L'
50 | def hex_pair_to_byte(hex_pair):
51 | return chr(int(hex_pair.encode('utf-8'), 16))
52 |
53 |
54 | # Converts a hexidecimal string (a string containing only characters valid
55 | # in use for displaying a hexidecimal number, i.e. 0123456789ABCDEF) into
56 | # a binary string.
57 | #
58 | # >>> hex_string_to_binary(u'784B332D')
59 | # 'xK3-'
60 | def hex_string_to_binary(hex_string):
61 | if len(hex_string) % 2:
62 | raise ValueError, "string given has odd-number of characters, must be even"
63 | if not hex_string:
64 | raise ValueError, "cannot convert empty string"
65 | return ''.join([hex_pair_to_byte(hex_string[i:i + 2]) for i in range(0, len(hex_string), 2)])
66 |
67 |
68 | def make_short_object_id(object_id):
69 | # Due to the way Azureus generates object ID's (it tends to be a
70 | # long integer which is then incremented for each object it
71 | # generates), we don't bother rendering the entire object ID
72 | # as it would be too long (it can be as long as 20 characters).
73 | #
74 | # So instead, we only use the last 6 digits - turning the ID into
75 | # hexidecimal. This gives us a range of 16**6 = 16.7 million. So
76 | # Azureus would have to generate more than 16 million objects
77 | # before it generates an ID which has the same short ID form as
78 | # another object.
79 | #
80 | # We show the short ID as a way of easily seeing whether two
81 | # objects represent the same remote object or not.
82 | hex_id = hex(object_id)
83 |
84 | if hex_id[-1] == 'L':
85 | hex_short_id = hex_id[-7:-1]
86 | else:
87 | hex_short_id = hex_id[-6:]
88 |
89 | return hex_short_id
90 |
91 |
92 | def parse_azureus_version_string(ver_string):
93 | ver_bits = ver_string.split('_', 2)
94 | if len(ver_bits) == 1:
95 | major_ver, minor_ver = ver_string, None
96 | else:
97 | major_ver, minor_ver = ver_bits
98 |
99 | ver_segments = [int(bit) for bit in major_ver.split('.')]
100 | if minor_ver:
101 | if minor_ver[0].lower() == 'b':
102 | ver_segments.append('b')
103 | try:
104 | beta_ver = int(minor_ver[1:])
105 | except ValueError:
106 | pass
107 | else:
108 | ver_segments.append(beta_ver)
109 |
110 | return tuple(ver_segments)
111 |
112 | #
113 | # I love this code. :)
114 | #
115 | # I might turn it into something more generic, and use it elsewhere..
116 | #
117 | # Would be nicer if there was a better API for doing this, but given the amount
118 | # of hackery that I'm doing right now, I won't complain. :)
119 | #
120 | # What a lot of effort just to act as if these methods were defined in the
121 | # class itself.
122 | import new
123 |
124 |
125 | class MethodFactory(object):
126 | def __init__(self, method_object):
127 | _codeobj = method_object.func_code
128 | code_arguments = [
129 | _codeobj.co_argcount, _codeobj.co_nlocals, _codeobj.co_stacksize,
130 | _codeobj.co_flags, _codeobj.co_code, _codeobj.co_consts,
131 | _codeobj.co_names, _codeobj.co_varnames, _codeobj.co_filename,
132 | _codeobj.co_name, _codeobj.co_firstlineno, _codeobj.co_lnotab,
133 | ]
134 | self.code_arguments = code_arguments
135 |
136 | def _build_function(self, name):
137 | code_args = self.code_arguments[:]
138 | code_args[9] = name
139 | # code_args[8] =
140 | codeobj = new.code(*code_args)
141 | return new.function(codeobj, {'__funcname__': name, '__builtins__': __builtins__}, name)
142 |
143 | def make_instance_method(self, name, instanceobj):
144 | method = self._build_function(name)
145 | return new.instancemethod(method, instanceobj, type(instanceobj))
146 |
147 | def make_class_method(self, name, classobj):
148 | method = self._build_function(name)
149 | return new.instancemethod(method, None, classobj)
150 |
151 |
152 | def _not_implemented(self, *args, **kwargs):
153 | class_name = self.__class__.__name__
154 | funcname = __funcname__
155 | raise NotImplementedError, "%(class_name)s.%(funcname)s" % locals()
156 |
157 |
158 | not_implemented_factory = MethodFactory(_not_implemented)
159 | make_not_implemented_class_method = not_implemented_factory.make_class_method
160 | del _not_implemented, not_implemented_factory
161 |
162 |
163 | def handle_kwargs(kwargs, *required, **optional):
164 | result = {}
165 | result.update(optional)
166 |
167 | required_args = dict([(x, None) for x in required])
168 |
169 | for kwarg_key, kwarg_value in kwargs.iteritems():
170 | if optional.has_key(kwarg_key):
171 | pass
172 | else:
173 | try:
174 | required_args.pop(kwarg_key)
175 | except KeyError:
176 | raise TypeError, "unexpected keyword argument: %r" % kwarg_key
177 |
178 | result[kwarg_key] = kwarg_value
179 |
180 | if required_args:
181 | missing_key = required_args.popitem()[0]
182 | raise TypeError, "missing keyword argument: %r" % missing_key
183 |
184 | return result
185 |
186 |
187 | class Sentinel(object):
188 | def __init__(self, value):
189 | self.value = value
190 |
191 | def __str__(self):
192 | return str(self.value)
193 |
194 | def __repr__(self):
195 | return '' % (self.value, id(self))
196 |
--------------------------------------------------------------------------------
/resources/scrapers/kinopoisk/pageparser.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #
3 | # Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
4 | # Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
5 | # Copyright (C) 2013 Yevgeny Nyden
6 | #
7 | # This program is free software; you can redistribute it and/or
8 | # modify it under the terms of the GNU General Public License
9 | # as published by the Free Software Foundation; either version 2
10 | # of the License, or (at your option) any later version.
11 | #
12 | # This program is distributed in the hope that it will be useful,
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | # GNU General Public License for more details.
16 | #
17 | # You should have received a copy of the GNU General Public License
18 | # along with this program; if not, write to the Free Software
19 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 | # 02110-1301, USA.
21 | #
22 | # @author zhenya (Yevgeny Nyden)
23 | # @version 1.52
24 | # @revision 148
25 |
26 | import re
27 | import urllib
28 | import operator
29 |
30 | import common
31 | import pluginsettings as S
32 | import translit
33 |
34 |
35 |
36 |
37 |
38 | # MATCHER_MOVIE_DURATION = re.compile('\s*(\d+).*?', re.UNICODE | re.DOTALL)
39 | # MATCHER_IMDB_RATING = re.compile('IMDb:\s*(\d+\.?\d*)\s*\(\s*([\s\d]+)\s*\)', re.UNICODE | re.DOTALL)
40 | # MATCHER_IMDB_RATING = re.compile('IMDb:\s*(\d+\.?\d*)\s?\((.*)\)', re.UNICODE)
41 |
42 | USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22'
43 | '''
44 | MOVIE_THUMBNAIL_SMALL_WIDTH = 130
45 | MOVIE_THUMBNAIL_SMALL_HEIGHT = 168
46 | MOVIE_THUMBNAIL_BIG_WIDTH = 780
47 | MOVIE_THUMBNAIL_BIG_HEIGHT = 1024
48 |
49 | # Compiled regex matchers.
50 | MATCHER_WIDTH_FROM_STYLE = re.compile('.*width\s*:\s*(\d+)px.*', re.UNICODE)
51 | MATCHER_HEIGHT_FROM_STYLE = re.compile('.*height\s*:\s*(\d+)px.*', re.UNICODE)
52 | MATCHER_LEADING_NONALPHA = re.compile('^[\s\d\.\(\)]*', re.UNICODE | re.MULTILINE)
53 |
54 |
55 | # Русские месяца, пригодятся для определения дат.
56 | RU_MONTH = {
57 | u'января': '01',
58 | u'февраля': '02',
59 | u'марта': '03',
60 | u'апреля': '04',
61 | u'мая': '05',
62 | u'июня': '06',
63 | u'июля': '07',
64 | u'августа': '08',
65 | u'сентября': '09',
66 | u'октября': '10',
67 | u'ноября': '11',
68 | u'декабря': '12'
69 | }
70 | '''
71 |
72 |
73 | class PageParser:
74 | def __init__(self, logger, httpUtils=common.HttpUtils(S.ENCODING_KINOPOISK_PAGE, USER_AGENT), isDebug=False):
75 | self.log = logger
76 | self.isDebug = isDebug
77 | self.httpUtils = httpUtils
78 |
79 | def fetchAndParseSearchResults(self, mediaName, mediaYear, mediaAltName=None):
80 | """ Searches for movie titles on KinoPoisk.
81 | @param mediaName Movie title parsed from a filename.
82 | @param mediaName Movie year parsed from a filename.
83 | @return Array of tuples: [kinoPoiskId, title, year, score]
84 | """
85 | self.log.Info('Quering kinopoisk...')
86 | results = self.queryKinoPoisk(mediaName, mediaYear)
87 |
88 | # Check media name is all ASCII characters, and if it is,
89 | # issue another query to KinoPoisk using a translified media name;
90 | # lastly, merge the scored results.
91 | if common.isAsciiString(mediaName):
92 | translifiedMediaName = translit.detranslify(mediaName)
93 | moreResults = self.queryKinoPoisk(translifiedMediaName, mediaYear)
94 | resultsMap = dict()
95 | for result in results:
96 | resultsMap[result[0]] = result
97 | results = [] # Recreate and repopulate the results array removing duplicates.
98 | for result in moreResults:
99 | currId = result[0]
100 | if currId in resultsMap.keys():
101 | origResult = resultsMap[currId]
102 | del resultsMap[currId]
103 | if result[3] >= origResult[3]:
104 | results.append(result)
105 | else:
106 | results.append(origResult)
107 | else:
108 | results.append(result)
109 | results.extend(resultsMap.values())
110 |
111 | if mediaAltName:
112 | moreResults = self.queryKinoPoisk(mediaAltName, mediaYear)
113 | resultsMap = dict()
114 | for result in results:
115 | resultsMap[result[0]] = result
116 | results = [] # Recreate and repopulate the results array removing duplicates.
117 | for result in moreResults:
118 | currId = result[0]
119 | if currId in resultsMap.keys():
120 | origResult = resultsMap[currId]
121 | del resultsMap[currId]
122 | if result[3] >= origResult[3]:
123 | results.append(result)
124 | else:
125 | results.append(origResult)
126 | else:
127 | results.append(result)
128 | results.extend(resultsMap.values())
129 |
130 | # Sort all results based on their score.
131 | results.sort(key=operator.itemgetter(3))
132 | results.reverse()
133 | if self.isDebug:
134 | self.log.Debug('Search produced %d results:' % len(results))
135 | index = -1
136 | for result in results:
137 | index += 1
138 | self.log.Debug(' ... %d: id="%s", name="%s", year="%s", score="%d".' %
139 | (index, result[0], result[1], str(result[2]), result[3]))
140 | return results
141 |
142 | def queryKinoPoisk(self, mediaName, mediaYear):
143 | """ Ищет фильм на кинопоиске.
144 | Returns title results as they are returned (no sorting is done here!).
145 | """
146 | results = []
147 | try:
148 | encodedName = urllib.quote(mediaName.encode(S.ENCODING_KINOPOISK_PAGE))
149 | except:
150 | encodedName = urllib.quote(mediaName.encode('utf-8'))
151 | page = self.httpUtils.requestAndParseHtmlPage(S.KINOPOISK_SEARCH_SIMPLE % encodedName)
152 | if page is None:
153 | self.log.Warn(' ### nothing was found on kinopoisk for media name "%s"' % mediaName)
154 | return results
155 |
156 | # Страница получена, берем с нее перечень всех названий фильмов.
157 | self.log.Debug('**** Finding %s (%s) got a KinoPoisk...' % (mediaName, mediaYear))
158 |
159 | reobj = re.compile(r'(.+?)
(.+?)')
160 | result = reobj.findall(page)
161 |
162 | # Inspect query results titles and score them.
163 | self.log.Debug('found %d results (div info tags)' % len(result))
164 | itemIndex = -1
165 | for itemKinoPoiskId, itemTitleitemYear, itemAltTitle in result:
166 | itemIndex = itemIndex + 1
167 | itemAltTitle = itemAltTitle.replace(' ', '')
168 | try:
169 | itemTitle, itemYear = re.compile('^(.+?), (\d\d\d\d)$').findall(itemTitleitemYear.strip())[0]
170 | except:
171 | itemYear = None
172 | itemTitle = itemTitleitemYear
173 | itemScore = common.scoreMediaTitleMatch(mediaName, mediaYear, itemTitle, itemAltTitle, itemYear, itemIndex)
174 | results.append([itemKinoPoiskId, itemTitle, itemYear, itemScore])
175 |
176 | return results
177 |
--------------------------------------------------------------------------------
/resources/scrapers/kinopoisk/HTTP.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | #
3 | # Russian metadata plugin for Plex, which uses http://www.kinopoisk.ru/ to get the tag data.
4 | # Плагин для обновления информации о фильмах использующий КиноПоиск (http://www.kinopoisk.ru/).
5 | # Copyright (C) 2012 Yevgeny Nyden
6 | #
7 | # This program is free software; you can redistribute it and/or
8 | # modify it under the terms of the GNU General Public License
9 | # as published by the Free Software Foundation; either version 2
10 | # of the License, or (at your option) any later version.
11 | #
12 | # This program is distributed in the hope that it will be useful,
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 | # GNU General Public License for more details.
16 | #
17 | # You should have received a copy of the GNU General Public License
18 | # along with this program; if not, write to the Free Software
19 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
20 | # 02110-1301, USA.
21 | #
22 | # @author zhenya (Yevgeny Nyden)
23 | # @revision 148
24 |
25 | import urllib
26 | import urllib2
27 | import base64
28 | import time
29 |
30 | import LOGGER as Log
31 |
32 |
33 | class HTTP:
34 | def __init__(self):
35 | pass
36 |
37 |
38 | def fetch(self, request, **kwargs):
39 | self.con, self.fd, self.progress, self.cookies, self.request = None, None, None, None, request
40 |
41 | if not isinstance(self.request, HTTPRequest):
42 | self.request = HTTPRequest(url=self.request, **kwargs)
43 |
44 | self.response = HTTPResponse(self.request)
45 |
46 | Log.Debug('XBMCup: HTTP: request: ' + str(self.request))
47 | try:
48 | self._opener()
49 | self._fetch()
50 | except Exception, e:
51 | Log.Debug('XBMCup: HTTP: ' + str(e))
52 | if isinstance(e, urllib2.HTTPError):
53 | self.response.code = e.code
54 | self.response.error = e
55 | else:
56 | self.response.code = 200
57 |
58 | if self.fd:
59 | self.fd.close()
60 | self.fd = None
61 |
62 | if self.con:
63 | self.con.close()
64 | self.con = None
65 |
66 | if self.progress:
67 | self.progress.close()
68 | self.progress = None
69 |
70 | self.response.time = time.time() - self.response.time
71 |
72 | return self.response
73 |
74 |
75 | def _opener(self):
76 |
77 | build = [urllib2.HTTPHandler()]
78 |
79 | if self.request.redirect:
80 | build.append(urllib2.HTTPRedirectHandler())
81 |
82 | if self.request.proxy_host and self.request.proxy_port:
83 | build.append(urllib2.ProxyHandler(
84 | {self.request.proxy_protocol: self.request.proxy_host + ':' + str(self.request.proxy_port)}))
85 |
86 | if self.request.proxy_username:
87 | proxy_auth_handler = urllib2.ProxyBasicAuthHandler()
88 | proxy_auth_handler.add_password('realm', 'uri', self.request.proxy_username,
89 | self.request.proxy_password)
90 | build.append(proxy_auth_handler)
91 |
92 | urllib2.install_opener(urllib2.build_opener(*build))
93 |
94 |
95 | def _fetch(self):
96 | params = {} if self.request.params is None else self.request.params
97 |
98 | if self.request.method == 'POST':
99 | if isinstance(params, dict) or isinstance(params, list):
100 | params = urllib.urlencode(params)
101 | req = urllib2.Request(self.request.url, params)
102 | else:
103 | req = urllib2.Request(self.request.url)
104 |
105 | for key, value in self.request.headers.iteritems():
106 | req.add_header(key, value)
107 |
108 | if self.request.auth_username and self.request.auth_password:
109 | req.add_header('Authorization', 'Basic %s' % base64.encodestring(
110 | ':'.join([self.request.auth_username, self.request.auth_password])).strip())
111 |
112 | # self.con = urllib2.urlopen(req, timeout=self.request.timeout)
113 | self.con = urllib2.urlopen(req)
114 | self.response.headers = self._headers(self.con.info())
115 |
116 | self.response.body = self.con.read()
117 |
118 | if self.request.cookies:
119 | self.cookies.save(self.request.cookies)
120 |
121 |
122 | def _headers(self, raw):
123 | headers = {}
124 | for line in raw.headers:
125 | pair = line.split(':', 1)
126 | if len(pair) == 2:
127 | tag = pair[0].lower().strip()
128 | value = pair[1].strip()
129 | if tag and value:
130 | headers[tag] = value
131 | return headers
132 |
133 |
134 | def _progress(self, read, size, name):
135 | res = []
136 | if size < 0:
137 | res.append(1)
138 | else:
139 | res.append(int(float(read) / (float(size) / 100.0)))
140 | if name:
141 | res.append(u'File: ' + name)
142 | if size != -1:
143 | res.append(u'Size: ' + self._human(size))
144 | res.append(u'Load: ' + self._human(read))
145 | return res
146 |
147 | def _human(self, size):
148 | human = None
149 | for h, f in (('KB', 1024), ('MB', 1024 * 1024), ('GB', 1024 * 1024 * 1024), ('TB', 1024 * 1024 * 1024 * 1024)):
150 | if size / f > 0:
151 | human = h
152 | factor = f
153 | else:
154 | break
155 | if human is None:
156 | return (u'%10.1f %s' % (size, u'byte')).replace(u'.0', u'')
157 | else:
158 | return u'%10.2f %s' % (float(size) / float(factor), human)
159 |
160 |
161 | class HTTPRequest:
162 | def __init__(self, url, method='GET', headers=None, cookies=None, params=None, upload=None, download=None,
163 | progress=False, auth_username=None, auth_password=None, proxy_protocol='http', proxy_host=None,
164 | proxy_port=None, proxy_username=None, proxy_password='', timeout=20.0, redirect=True, gzip=False):
165 | if headers is None:
166 | headers = {}
167 |
168 | self.url = url
169 | self.method = method
170 | self.headers = headers
171 |
172 | self.cookies = cookies
173 |
174 | self.params = params
175 |
176 | self.upload = upload
177 | self.download = download
178 | self.progress = progress
179 |
180 | self.auth_username = auth_username
181 | self.auth_password = auth_password
182 |
183 | self.proxy_protocol = proxy_protocol
184 | self.proxy_host = proxy_host
185 | self.proxy_port = proxy_port
186 | self.proxy_username = proxy_username
187 | self.proxy_password = proxy_password
188 |
189 | self.timeout = timeout
190 |
191 | self.redirect = redirect
192 |
193 | self.gzip = gzip
194 |
195 | def __repr__(self):
196 | return '%s(%s)' % (self.__class__.__name__, ','.join('%s=%r' % i for i in self.__dict__.iteritems()))
197 |
198 |
199 | class HTTPResponse:
200 | def __init__(self, request):
201 | self.request = request
202 | self.code = None
203 | self.headers = {}
204 | self.error = None
205 | self.body = None
206 | self.filename = None
207 | self.time = time.time()
208 |
209 | def __repr__(self):
210 | args = ','.join('%s=%r' % i for i in self.__dict__.iteritems() if i[0] != 'body')
211 | if self.body:
212 | args += ',body='
213 | else:
214 | args += ',body=None'
215 | return '%s(%s)' % (self.__class__.__name__, args)
216 |
--------------------------------------------------------------------------------
/resources/utorrent/dopal/xmlutils.py:
--------------------------------------------------------------------------------
1 | # File: xmlutils.py
2 | # Library: DOPAL - DO Python Azureus Library
3 | #
4 | # This program is free software; you can redistribute it and/or modify
5 | # it under the terms of the GNU General Public License as published by
6 | # the Free Software Foundation; version 2 of the License.
7 | #
8 | # This program is distributed in the hope that it will be useful,
9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 | # GNU General Public License for more details ( see the COPYING file ).
12 | #
13 | # You should have received a copy of the GNU General Public License
14 | # along with this program; if not, write to the Free Software
15 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 |
17 | '''
18 | XML utility functions.
19 | '''
20 |
21 | # Given an object which has the same interface as xml.dom.Node:
22 | # a) Join all concurrent text nodes together.
23 | # b) Strip all trailing and leading whitespace from each text node.
24 | #
25 | # This function will recursively process the tree structure given in the node
26 | # object. No value will be returned by this function, instead the given object
27 | # will be modified.
28 | def normalise_xml_structure(xml_node):
29 | # Concurrent text nodes should be joined together.
30 | xml_node.normalize()
31 |
32 | # Strip all text nodes which are empty of content (whitespace is not
33 | # content).
34 | from xml.dom import Node
35 |
36 | nodes_to_delete = []
37 |
38 | for node in xml_node.childNodes:
39 | if node.nodeType == Node.TEXT_NODE:
40 | stripped_text = node.nodeValue.strip()
41 | if stripped_text:
42 | node.nodeValue = stripped_text
43 | else:
44 | nodes_to_delete.append(node)
45 | else:
46 | normalise_xml_structure(node)
47 |
48 | for node in nodes_to_delete:
49 | xml_node.removeChild(node)
50 | node.unlink()
51 |
52 |
53 | def get_text_content(node):
54 | from xml.dom import Node
55 |
56 | # Text content is stored directly in this node.
57 | if node.nodeType == Node.TEXT_NODE:
58 | return node.nodeValue
59 |
60 | # Otherwise, must be in a child node.
61 | #elif len(node.childNodes) == 1 and \
62 | # node.firstChild.nodeType == Node.TEXT_NODE:
63 | # return node.firstChild.nodeValue
64 |
65 | # Sometimes happens for attributes with no real value.
66 | elif len(node.childNodes) == 0:
67 | return ''
68 |
69 | text_node = None
70 | err_text = None
71 | for child in node.childNodes:
72 | if child.nodeType == Node.TEXT_NODE:
73 | if text_node is None:
74 | text_node = child
75 | else:
76 | err_text = "contained multiple text nodes"
77 | break
78 | else:
79 | if text_node is None:
80 | if len(node.childNodes) != 1:
81 | err_text = "contained multiple nodes, but none were text"
82 | else:
83 | err_text = "did not contain a character string as its value"
84 | else:
85 | return text_node.nodeValue
86 |
87 | raise ValueError, ("the node %s " % node.nodeName) + err_text
88 |
89 |
90 | from xml.sax.saxutils import quoteattr, escape
91 |
92 | # This base class will be removed when XMLObject is removed.
93 | class _XMLObjectBase(object):
94 | def __init__(self, tag_name):
95 | self.tag_name = tag_name
96 | self.attributes = {}
97 | self.contents = []
98 |
99 | def add_attribute(self, attribute_name, attribute_value):
100 | self.attributes[attribute_name] = attribute_value
101 |
102 | def add_content(self, content):
103 | self.contents.append(content)
104 |
105 | def to_string(self, out=None, indent=0):
106 | if out is None:
107 | # We use StringIO instead of cStringIO not to lose unicode strings.
108 | import StringIO
109 |
110 | out = StringIO.StringIO()
111 | return_as_string = True
112 | else:
113 | return_as_string = False
114 |
115 | indent_string = ' ' * indent
116 | out.write(indent_string)
117 | out.write('<')
118 | out.write(self.tag_name)
119 | for attr_name, attr_value in self.attributes.items():
120 | out.write(' ')
121 | out.write(attr_name)
122 | out.write('=')
123 | out.write(quoteattr(attr_value))
124 |
125 | # If we have no contents, we'll close the tag here.
126 | if not self.contents:
127 | out.write(' />\n')
128 |
129 | else:
130 | out.write('>')
131 |
132 | # If we have one piece of content, which is just a string, then
133 | # we'll put it on the same line as the opening tag is on.
134 | if len(self.contents) == 1 and not hasattr(self.contents[0], 'to_string'):
135 | out.write(escape(self.contents[0]))
136 |
137 | # Otherwise, we assume we have some more XML blocks to write out,
138 | # so we'll indent them and put them on newlines.
139 | elif self.contents:
140 | out.write('\n')
141 | for content in self.contents:
142 | content.to_string(out, indent + 2)
143 | out.write(indent_string)
144 |
145 | # Write out the closing tag (if we haven't written it already).
146 | if self.contents:
147 | out.write('\n')
150 |
151 | # If the invocation of this method was not passed a buffer to write
152 | # into, then we return the string representation.
153 | if return_as_string:
154 | return out.getvalue()
155 |
156 | return None
157 |
158 |
159 | class XMLObject(_XMLObjectBase):
160 | '''
161 | B{Deprecated:} An object representing a block of XML.
162 |
163 | @attention: B{Deprecated:} This class does not provide any guarantees in
164 | the way that byte strings are handled. Use L{UXMLObject} instead.
165 | '''
166 |
167 | def __init__(self, tag_name):
168 | from dopal.errors import DopalPendingDeprecationWarning
169 |
170 | import warnings
171 |
172 | warnings.warn("XMLObject is deprecated - use UXMLObject instead", DopalPendingDeprecationWarning)
173 |
174 | _XMLObjectBase.__init__(self, tag_name)
175 |
176 |
177 | class UXMLObject(_XMLObjectBase):
178 | '''
179 | An object representing a block of XML.
180 |
181 | Any string which is added to this block (either through the L{add_content}
182 | or L{add_attribute} methods should be a unicode string, rather than a byte
183 | string. If it is a byte string, then it must be a string which contains
184 | text in the system's default encoding - attempting to add text encoding in
185 | other formats is not allowed.
186 | '''
187 |
188 | def to_string(self, out=None, indent=0):
189 | result = _XMLObjectBase.to_string(self, out, indent)
190 | if result is None:
191 | return None
192 | return unicode(result)
193 |
194 | def encode(self, encoding='UTF-8'):
195 | return (('\n' % encoding) + self.to_string()).encode(encoding)
196 |
197 | def __unicode__(self):
198 | return self.to_string()
199 |
200 |
201 | def make_xml_ref_for_az_object(object_id):
202 | '''
203 | Creates an XML block which represents a remote object in Azureus with the given object ID.
204 |
205 | @param object_id: The object ID to reference.
206 | @type object_id: int / long
207 | @return: A L{UXMLObject} instance.
208 | '''
209 | object_id_block = UXMLObject('_object_id')
210 | object_id_block.add_content(str(object_id))
211 |
212 | object_block = UXMLObject('OBJECT')
213 | object_block.add_content(object_id_block)
214 | return object_block
215 |
--------------------------------------------------------------------------------
/resources/settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
43 |
44 |
45 |
46 |
48 |
49 |
50 |
51 |
52 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
--------------------------------------------------------------------------------
/resources/scrapers/kinopoisk/translit.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # -*- test-case-name: pytils.test.test_translit -*-
3 | # pytils - simple processing for russian strings
4 | # Copyright (C) 2006-2007 Yury Yurevich
5 | #
6 | # http://www.pyobject.ru/projects/pytils/
7 | #
8 | # This program is free software; you can redistribute it and/or
9 | # modify it under the terms of the GNU General Public License
10 | # as published by the Free Software Foundation, version 2
11 | # of the License.
12 | #
13 | # This program is distributed in the hope that it will be useful,
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 | # GNU General Public License for more details.
17 | """
18 | Simple transliteration
19 | """
20 |
21 | # __id__ = __revision__ = "$Id: translit.py 102 2007-07-12 12:33:36Z the.pythy $"
22 | # __url__ = "$URL: https://pythy.googlecode.com/svn/tags/pytils/0_2_2/pytils/translit.py $"
23 |
24 | import re
25 |
26 | TRANSTABLE = (
27 | (u"'", u"'"),
28 | (u'"', u'"'),
29 | (u"‘", u"'"),
30 | (u"’", u"'"),
31 | (u"«", u'"'),
32 | (u"»", u'"'),
33 | (u"–", u"-"),
34 | (u"…", u"..."),
35 | (u"№", u"#"),
36 | ## верхний регистр
37 | # трехбуквенные замены
38 | (u"Щ", u"Sch"),
39 | # при замене русский->английский будет первая замена,
40 | # т.е. Sch
41 | # а вот если английский->русский, то вариант SCH и Sch --
42 | # оба пройдут
43 | (u"Щ", u"SCH"),
44 | # двухбуквенные замены
45 | (u"Ё", u"Yo"),
46 | (u"Ё", u"YO"),
47 | (u"Ж", u"Zh"),
48 | (u"Ж", u"ZH"),
49 | (u"Ц", u"Ts"),
50 | (u"Ц", u"TS"),
51 | (u"Ч", u"Ch"),
52 | (u"Ч", u"CH"),
53 | (u"Ш", u"Sh"),
54 | (u"Ш", u"SH"),
55 | (u"Ы", u"Yi"),
56 | (u"Ы", u"YI"),
57 | (u"Ю", u"Yu"),
58 | (u"Ю", u"YU"),
59 | (u"Я", u"Ya"),
60 | (u"Я", u"YA"),
61 | (u"ИЙ", u"IY"),
62 | # однобуквенные замены
63 | (u"А", u"A"),
64 | (u"Б", u"B"),
65 | (u"В", u"V"),
66 | (u"Г", u"G"),
67 | (u"Д", u"D"),
68 | (u"Е", u"E"),
69 | (u"З", u"Z"),
70 | (u"И", u"I"),
71 | (u"Й", u"J"),
72 | (u"К", u"K"),
73 | (u"Л", u"L"),
74 | (u"М", u"M"),
75 | (u"Н", u"N"),
76 | (u"О", u"O"),
77 | (u"П", u"P"),
78 | (u"Р", u"R"),
79 | (u"С", u"S"),
80 | (u"Т", u"T"),
81 | (u"У", u"U"),
82 | (u"Ф", u"F"),
83 | (u"Х", u"H"),
84 | (u"Э", u"E"),
85 | (u"Ъ", u"`"),
86 | (u"Ы", u"Y"),
87 | (u"Ь", u"'"),
88 | ## нижний регистр
89 | # трехбуквенные замены
90 | (u"щ", u"sch"),
91 | # двухбуквенные замены
92 | (u"ё", u"yo"),
93 | (u"ж", u"zh"),
94 | (u"ц", u"ts"),
95 | (u"ч", u"ch"),
96 | (u"ш", u"sh"),
97 | (u"ы", u"yi"),
98 | (u"ю", u"yu"),
99 | (u"я", u"ya"),
100 | (u"я", u"ja"),
101 | (u"ий", u"iy"),
102 | # однобуквенные замены
103 | (u"а", u"a"),
104 | (u"б", u"b"),
105 | (u"в", u"v"),
106 | (u"г", u"g"),
107 | (u"д", u"d"),
108 | (u"е", u"e"),
109 | (u"з", u"z"),
110 | (u"и", u"i"),
111 | (u"й", u"j"),
112 | (u"к", u"k"),
113 | (u"л", u"l"),
114 | (u"м", u"m"),
115 | (u"н", u"n"),
116 | (u"о", u"o"),
117 | (u"п", u"p"),
118 | (u"р", u"r"),
119 | (u"с", u"s"),
120 | (u"т", u"t"),
121 | (u"у", u"u"),
122 | (u"ф", u"f"),
123 | (u"х", u"h"),
124 | (u"э", u"e"),
125 | (u"ъ", u"`"),
126 | (u"ь", u"'"),
127 | # для полноты английского алфавит (в slugify)
128 | # дополняем английскими буквами, которых
129 | # не в парах
130 | (u"c", u"c"),
131 | (u"q", u"q"),
132 | (u"y", u"y"),
133 | (u"x", u"x"),
134 | (u"w", u"w"),
135 | (u"1", u"1"),
136 | (u"2", u"2"),
137 | (u"3", u"3"),
138 | (u"4", u"4"),
139 | (u"5", u"5"),
140 | (u"6", u"6"),
141 | (u"7", u"7"),
142 | (u"8", u"8"),
143 | (u"9", u"9"),
144 | (u"0", u"0"),
145 | ) #: Translation table
146 |
147 | RU_ALPHABET = [x[0] for x in TRANSTABLE] #: Russian alphabet that we can translate
148 | EN_ALPHABET = [x[1] for x in TRANSTABLE] #: English alphabet that we can detransliterate
149 | ALPHABET = RU_ALPHABET + EN_ALPHABET #: Alphabet that we can (de)transliterate
150 |
151 |
152 | def translify(in_string):
153 | """
154 | Translify russian text
155 |
156 | @param in_string: input string
157 | @type in_string: C{unicode}
158 |
159 | @return: transliterated string
160 | @rtype: C{str}
161 |
162 | @raise TypeError: when in_string is not C{unicode}
163 | @raise ValueError: when string doesn't transliterate completely
164 | """
165 | if not isinstance(in_string, unicode):
166 | raise TypeError("Argument must be unicode, not %s" % type(in_string))
167 |
168 | translit = in_string
169 | for symb_in, symb_out in TRANSTABLE:
170 | translit = translit.replace(symb_in, symb_out)
171 |
172 | try:
173 | translit = str(translit)
174 | except UnicodeEncodeError:
175 | raise ValueError("Unicode string doesn't transliterate completely, " + \
176 | "is it russian?")
177 |
178 | return translit
179 |
180 |
181 | def detranslify(in_string):
182 | """
183 | Detranslify
184 |
185 | @param in_string: input string
186 | @type in_string: C{basestring}
187 |
188 | @return: detransliterated string
189 | @rtype: C{str}
190 |
191 | @raise TypeError: when in_string neither C{str}, no C{unicode}
192 | @raise ValueError: if in_string is C{str}, but it isn't ascii
193 | """
194 | if not isinstance(in_string, basestring):
195 | raise TypeError("Argument must be basestring, not %s" % type(in_string))
196 |
197 | # в unicode
198 | try:
199 | russian = unicode(in_string)
200 | except UnicodeDecodeError:
201 | raise ValueError("We expects when in_string is str type," + \
202 | "it is an ascii, but now it isn't. Use unicode " + \
203 | "in this case.")
204 |
205 | for symb_out, symb_in in TRANSTABLE:
206 | russian = russian.replace(symb_in, symb_out)
207 |
208 | return russian
209 |
210 |
211 | def slugify(in_string):
212 | """
213 | Prepare string for slug (i.e. URL or file/dir name)
214 |
215 | @param in_string: input string
216 | @type in_string: C{basestring}
217 |
218 | @return: slug-string
219 | @rtype: C{str}
220 |
221 | @raise TypeError: when in_string isn't C{unicode} or C{str}
222 | @raise ValueError: if in_string is C{str}, but it isn't ascii
223 | """
224 | if not isinstance(in_string, basestring):
225 | raise TypeError("Argument must be basestring, not %s" % type(in_string))
226 | try:
227 | u_in_string = unicode(in_string).lower()
228 | except UnicodeDecodeError:
229 | raise ValueError("We expects when in_string is str type," + \
230 | "it is an ascii, but now it isn't. Use unicode " + \
231 | "in this case.")
232 | # convert & to "and"
233 | u_in_string = re.sub('\&\;|\&', ' and ', u_in_string)
234 | # replace spaces by hyphen
235 | u_in_string = re.sub('[-\s]+', '-', u_in_string)
236 | # remove symbols that not in alphabet
237 | u_in_string = u''.join([symb for symb in u_in_string if symb in ALPHABET])
238 | # translify it
239 | out_string = translify(u_in_string)
240 | # remove non-alpha
241 | return re.sub('[^\w\s-]', '', out_string).strip().lower()
242 |
243 |
244 | def dirify(in_string):
245 | """
246 | Alias for L{slugify}
247 | """
248 | slugify(in_string)
249 |
250 |
251 | def provide_unicode(stext, encoding, default=u"неизвестно"):
252 | """
253 | Provide Unicode from text
254 |
255 | @param stext: text
256 | @type stext: C{str}
257 |
258 | @param encoding: encoding if input text
259 | @type encoding: C{str}
260 |
261 | @return: C{unicode}
262 | """
263 | try:
264 | utext = str(stext).decode(encoding)
265 | except UnicodeDecodeError, err:
266 | utext = default % {'error': err, 'value': u""}
267 | return utext
268 |
269 |
270 | def provide_str(utext, encoding, default="unknown"):
271 | """
272 | Provide text from Unicode
273 |
274 | @param utext: unicode text
275 | @type utext: C{unicode}
276 |
277 | @param encoding: encoding of output text
278 | @type encoding: C{str}
279 |
280 | @return: C{str}
281 | """
282 | try:
283 | stext = unicode(utext).encode(encoding)
284 | except UnicodeEncodeError, err:
285 | stext = default % {'error': err, 'value': ""}
286 | return stext
287 |
--------------------------------------------------------------------------------
/resources/scrapers/fuzzywuzzy/fuzz.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # encoding: utf-8
3 | """
4 | fuzz.py
5 |
6 | Copyright (c) 2011 Adam Cohen
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining
9 | a copy of this software and associated documentation files (the
10 | "Software"), to deal in the Software without restriction, including
11 | without limitation the rights to use, copy, modify, merge, publish,
12 | distribute, sublicense, and/or sell copies of the Software, and to
13 | permit persons to whom the Software is furnished to do so, subject to
14 | the following conditions:
15 |
16 | The above copyright notice and this permission notice shall be
17 | included in all copies or substantial portions of the Software.
18 |
19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
23 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
25 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 | """
27 | from __future__ import unicode_literals
28 |
29 | try:
30 | from .StringMatcher import StringMatcher as SequenceMatcher
31 | except ImportError:
32 | from difflib import SequenceMatcher
33 |
34 | from . import utils
35 |
36 |
37 | ###########################
38 | # Basic Scoring Functions #
39 | ###########################
40 |
41 |
42 | def ratio(s1, s2):
43 |
44 | if s1 is None:
45 | raise TypeError("s1 is None")
46 | if s2 is None:
47 | raise TypeError("s2 is None")
48 | s1, s2 = utils.make_type_consistent(s1, s2)
49 | if len(s1) == 0 or len(s2) == 0:
50 | return 0
51 |
52 | m = SequenceMatcher(None, s1, s2)
53 | return utils.intr(100 * m.ratio())
54 |
55 |
56 | # todo: skip duplicate indexes for a little more speed
57 | def partial_ratio(s1, s2):
58 |
59 | if s1 is None:
60 | raise TypeError("s1 is None")
61 | if s2 is None:
62 | raise TypeError("s2 is None")
63 | s1, s2 = utils.make_type_consistent(s1, s2)
64 | if len(s1) == 0 or len(s2) == 0:
65 | return 0
66 |
67 | if len(s1) <= len(s2):
68 | shorter = s1
69 | longer = s2
70 | else:
71 | shorter = s2
72 | longer = s1
73 |
74 | m = SequenceMatcher(None, shorter, longer)
75 | blocks = m.get_matching_blocks()
76 |
77 | # each block represents a sequence of matching characters in a string
78 | # of the form (idx_1, idx_2, len)
79 | # the best partial match will block align with at least one of those blocks
80 | # e.g. shorter = "abcd", longer = XXXbcdeEEE
81 | # block = (1,3,3)
82 | # best score === ratio("abcd", "Xbcd")
83 | scores = []
84 | for block in blocks:
85 | long_start = block[1] - block[0] if (block[1] - block[0]) > 0 else 0
86 | long_end = long_start + len(shorter)
87 | long_substr = longer[long_start:long_end]
88 |
89 | m2 = SequenceMatcher(None, shorter, long_substr)
90 | r = m2.ratio()
91 | if r > .995:
92 | return 100
93 | else:
94 | scores.append(r)
95 |
96 | return int(100 * max(scores))
97 |
98 |
99 | ##############################
100 | # Advanced Scoring Functions #
101 | ##############################
102 |
103 | # Sorted Token
104 | # find all alphanumeric tokens in the string
105 | # sort those tokens and take ratio of resulting joined strings
106 | # controls for unordered string elements
107 | def _token_sort(s1, s2, partial=True, force_ascii=True):
108 |
109 | if s1 is None:
110 | raise TypeError("s1 is None")
111 | if s2 is None:
112 | raise TypeError("s2 is None")
113 |
114 | # pull tokens
115 | tokens1 = utils.full_process(s1, force_ascii=force_ascii).split()
116 | tokens2 = utils.full_process(s2, force_ascii=force_ascii).split()
117 |
118 | # sort tokens and join
119 | sorted1 = " ".join(sorted(tokens1))
120 | sorted2 = " ".join(sorted(tokens2))
121 |
122 | sorted1 = sorted1.strip()
123 | sorted2 = sorted2.strip()
124 |
125 | if partial:
126 | return partial_ratio(sorted1, sorted2)
127 | else:
128 | return ratio(sorted1, sorted2)
129 |
130 |
131 | def token_sort_ratio(s1, s2, force_ascii=True):
132 | return _token_sort(s1, s2, partial=False, force_ascii=force_ascii)
133 |
134 |
135 | def partial_token_sort_ratio(s1, s2, force_ascii=True):
136 | return _token_sort(s1, s2, partial=True, force_ascii=force_ascii)
137 |
138 |
139 | # Token Set
140 | # find all alphanumeric tokens in each string...treat them as a set
141 | # construct two strings of the form
142 | #
143 | # take ratios of those two strings
144 | # controls for unordered partial matches
145 | def _token_set(s1, s2, partial=True, force_ascii=True):
146 |
147 | if s1 is None:
148 | raise TypeError("s1 is None")
149 | if s2 is None:
150 | raise TypeError("s2 is None")
151 |
152 | p1 = utils.full_process(s1, force_ascii=force_ascii)
153 | p2 = utils.full_process(s2, force_ascii=force_ascii)
154 |
155 | if not utils.validate_string(p1):
156 | return 0
157 | if not utils.validate_string(p2):
158 | return 0
159 |
160 | # pull tokens
161 | tokens1 = set(utils.full_process(p1).split())
162 | tokens2 = set(utils.full_process(p2).split())
163 |
164 | intersection = tokens1.intersection(tokens2)
165 | diff1to2 = tokens1.difference(tokens2)
166 | diff2to1 = tokens2.difference(tokens1)
167 |
168 | sorted_sect = " ".join(sorted(intersection))
169 | sorted_1to2 = " ".join(sorted(diff1to2))
170 | sorted_2to1 = " ".join(sorted(diff2to1))
171 |
172 | combined_1to2 = sorted_sect + " " + sorted_1to2
173 | combined_2to1 = sorted_sect + " " + sorted_2to1
174 |
175 | # strip
176 | sorted_sect = sorted_sect.strip()
177 | combined_1to2 = combined_1to2.strip()
178 | combined_2to1 = combined_2to1.strip()
179 |
180 | if partial:
181 | ratio_func = partial_ratio
182 | else:
183 | ratio_func = ratio
184 |
185 | pairwise = [
186 | ratio_func(sorted_sect, combined_1to2),
187 | ratio_func(sorted_sect, combined_2to1),
188 | ratio_func(combined_1to2, combined_2to1)
189 | ]
190 | return max(pairwise)
191 |
192 |
193 | def token_set_ratio(s1, s2, force_ascii=True):
194 | return _token_set(s1, s2, partial=False, force_ascii=force_ascii)
195 |
196 |
197 | def partial_token_set_ratio(s1, s2, force_ascii=True):
198 | return _token_set(s1, s2, partial=True, force_ascii=force_ascii)
199 |
200 |
201 | # TODO: numerics
202 |
203 | ###################
204 | # Combination API #
205 | ###################
206 |
207 | # q is for quick
208 | def QRatio(s1, s2, force_ascii=True):
209 |
210 | p1 = utils.full_process(s1, force_ascii=force_ascii)
211 | p2 = utils.full_process(s2, force_ascii=force_ascii)
212 |
213 | if not utils.validate_string(p1):
214 | return 0
215 | if not utils.validate_string(p2):
216 | return 0
217 |
218 | return ratio(p1, p2)
219 |
220 |
221 | def UQRatio(s1, s2):
222 | return QRatio(s1, s2, force_ascii=False)
223 |
224 |
225 | # w is for weighted
226 | def WRatio(s1, s2, force_ascii=True):
227 |
228 | p1 = utils.full_process(s1, force_ascii=force_ascii)
229 | p2 = utils.full_process(s2, force_ascii=force_ascii)
230 |
231 | if not utils.validate_string(p1):
232 | return 0
233 | if not utils.validate_string(p2):
234 | return 0
235 |
236 | # should we look at partials?
237 | try_partial = True
238 | unbase_scale = .95
239 | partial_scale = .90
240 |
241 | base = ratio(p1, p2)
242 | len_ratio = float(max(len(p1), len(p2))) / min(len(p1), len(p2))
243 |
244 | # if strings are similar length, don't use partials
245 | if len_ratio < 1.5:
246 | try_partial = False
247 |
248 | # if one string is much much shorter than the other
249 | if len_ratio > 8:
250 | partial_scale = .6
251 |
252 | if try_partial:
253 | partial = partial_ratio(p1, p2) * partial_scale
254 | ptsor = partial_token_sort_ratio(p1, p2, force_ascii=force_ascii) \
255 | * unbase_scale * partial_scale
256 | ptser = partial_token_set_ratio(p1, p2, force_ascii=force_ascii) \
257 | * unbase_scale * partial_scale
258 |
259 | return int(max(base, partial, ptsor, ptser))
260 | else:
261 | tsor = token_sort_ratio(p1, p2, force_ascii=force_ascii) * unbase_scale
262 | tser = token_set_ratio(p1, p2, force_ascii=force_ascii) * unbase_scale
263 |
264 | return int(max(base, tsor, tser))
265 |
266 |
267 | def UWRatio(s1, s2):
268 | return WRatio(s1, s2, force_ascii=False)
269 |
--------------------------------------------------------------------------------