“今天，对凯塔感兴趣的俱乐部都知道，要想拉齐奥放走他，你必须拿出一大笔的资金，”经纪人萨维尼(Ulisse Savini)告诉TuttoMercatoWeb.com。“没有人打电话给我，但我们都很清楚：对凯塔感兴趣的俱乐部很多，这一点也不意外。除了利物浦经常在关注他之外，还有曼联。”
最后，经纪人解释道，这名19岁的前巴塞罗那球员需要拿到西班牙的护照才能转投英国踢球，尽管这问题不大。
20 |
21 | ####TODO：
22 |
23 | * 自定义样式
24 | * 进一步改进提取正确率
--------------------------------------------------------------------------------
/extract_utils.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 |
4 | __author__ = 'liuzhijun'
5 | import re
6 | import urllib2
7 | import gzip
8 |
9 | try:
10 | from cStringIO import StringIO # py2
11 | except ImportError:
12 | from io import StringIO # py3
13 |
14 | #正则表达式元字符
15 | meta_chars = [
16 | '+', '*', '?', '[', ']', '.', '{', '}', '(', ')'
17 | ]
18 | meta_regex = '([' + '\\'.join(meta_chars) + '])'
19 |
20 |
21 | def escape_regex_meta(text):
22 | """
23 | text中正则表达式元字符替换成普通成字符
24 | """
25 | return re.sub(meta_regex, lambda matchobj: '\\' + matchobj.group(), text)
26 |
27 |
28 | def url_validate(url):
29 | regex = re.compile(
30 | r'^(?:http|ftp)s?://' # http:// or https://
31 | r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
32 | r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
33 | r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
34 | r'(?::\d+)?' # optional port
35 | r'(?:/?|[/?]\S+)$', re.IGNORECASE)
36 | return regex.match(url) is not None
37 |
38 |
39 | def html_escape(text):
40 | """
41 | html转义
42 | """
43 | text = (text.replace(""", "\"").replace("“", "“").replace("”", "”")
44 | .replace("·", "·").replace("’", "’").replace("“", "“")
45 | .replace("”", "\”").replace("—", "——").replace("…", "…")
46 | .replace("•", "·").replace("(", "(").replace(")", ")")
47 | .replace("·", "·").replace("&", "&").replace("•", "·")
48 | .replace("<", "<").replace("<", "<").replace(">", ">")
49 | .replace(">", ">").replace(" ", " ").replace(" ", " ")
50 | .replace("˜", "~").replace("—", "—").replace("©", "@")
51 | .replace("©", "@").replace("♂", "").replace("\r\n|\r", "\n"))
52 | return text
53 |
54 |
55 | def get_html(url):
56 | assert url_validate(url), "invalid url"
57 | request = urllib2.Request(url)
58 | request.add_header("Accept-encoding", 'gzip')
59 | request.add_header("User-Agent", 'Mozilla/5.0 (Windows NT 6.2; WOW64) '
60 | 'AppleWebKit/537.36 '
61 | '(KHTML, like Gecko) Chrome/34.0.1847.131')
62 | response = urllib2.urlopen(request)
63 | html = response.read()
64 |
65 | def encode(html):
66 | try:
67 | html = unicode(html, 'utf-8').encode('utf-8')
68 | except UnicodeDecodeError:
69 | html = unicode(html, 'gbk').encode('utf-8')
70 | return html
71 |
72 | if response.info().get("Content-Encoding") == 'gzip':
73 | buf = StringIO(html)
74 | f = gzip.GzipFile(fileobj=buf)
75 | html = f.read()
76 | f.close()
77 | buf.close()
78 | html = encode(html)
79 | return html
--------------------------------------------------------------------------------
/htm_body_extractor.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- coding: utf-8 -*-
3 |
4 | __author__ = 'liuzhijun'
5 | import re
6 | import os
7 | from urlparse import urlparse
8 | import extract_utils
9 |
10 |
11 | class BodyExtractor(object):
12 | """
13 | url:链接地址
14 | body:正文内容
15 | depth:行块深度
16 | """
17 |
18 | def __init__(self, url):
19 | self.url = url
20 | self.domain = ''
21 | self.body = '' #正文内容
22 | self.depth = 3 #行块的深度
23 | self.html = ''
24 | self.plain_text = ''
25 | self.html_text = ''
26 | self.margin = 35 #从text的margin长度开始去匹配text_a_p，数值越大匹配越精确，效率越差
27 |
28 | def execute(self):
29 | self._pre_process()
30 | self._extract()
31 | self._post_process()
32 |
33 | def _pre_process(self):
34 | html = extract_utils.get_html(self.url)
35 | self.html = html
36 | parsed_uri = urlparse(self.url)
37 | self.domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
38 |
39 | plain_text, html_text = clean_html(self.html)
40 | self.html_text = html_text
41 | self.plain_text = plain_text
42 |
43 | def _post_process(self):
44 | """
45 | 把资源链接的相对路径改为完整路径
46 | 清空标签的无用属性，比如class, style
47 | """
48 | #TODO: 清空标签无用属性，比如class
49 |
50 | def repl(match):
51 | s = match.group()
52 | return s.replace('="', '="' + self.domain)
53 | self.body = re.sub(r'(?:href=["\']/(.*?)["\'])|(?:src=["\']/(.*?)["\'])', repl, self.body)
54 |
55 | def _extract(self):
56 | lines = tuple(self.plain_text.split('\n'))
57 | #lines对应每行的长度
58 | len_per_lines = [len(re.sub(r'\s+', '', line)) for line in lines]
59 |
60 | #每个块对应的长度
61 | len_per_blocks = []
62 | for i in range(len(len_per_lines) - self.depth + 1):
63 | word_len = sum([len_per_lines[j] for j in range(i, i + self.depth)])
64 | len_per_blocks.append(word_len)
65 |
66 | text_list = []
67 | text_begin_list = []
68 | text_end_list = []
69 |
70 | for i, value in enumerate(len_per_blocks):
71 | if value > 0:
72 | text_begin_list.append(i)
73 | tmp = lines[i]
74 | while i < len(len_per_blocks) and len_per_blocks[i] > 0:
75 | i += 1
76 | tmp += lines[i] + "\n"
77 | text_end_list.append(i)
78 | text_list.append(tmp)
79 |
80 | result = reduce(lambda str1, str2: str1 if len(str1) > len(str2) else str2, text_list)
81 | result = result.strip()
82 | i_start = self._start(result)
83 | i_end = self._end(result)
84 | if i_start == 0 or i_end == 0 or i_start > i_end:
85 | i_start = self._start(result, position=30) - 47
86 | if i_start < i_end:
87 | self.body = self.html_text[i_start:i_end]
88 | else:
89 | self.body = []
90 | self.body = ''.join(self.body.splitlines())
91 | return self.body
92 |
93 | def _start(self, result, position=0):
94 | i_start = 0
95 | for i in range(self.margin)[::-1]:
96 | start = result[position:i + position]
97 | start = extract_utils.escape_regex_meta(start)
98 | p = re.compile(start, re.IGNORECASE)
99 | match = p.search(self.html_text)
100 | if match:
101 | s = match.group()
102 | i_start = self.html_text.index(s)
103 | break
104 | return i_start
105 |
106 | def _end(self, result):
107 | i_end = 0
108 | for i in range(1, self.margin)[::-1]:
109 | end = result[-i:]
110 | end = extract_utils.escape_regex_meta(end)
111 | p = re.compile(end, re.IGNORECASE)
112 | match = p.search(self.html_text)
113 | if match:
114 | s = match.group()
115 | i_end = self.html_text.index(s) + len(s)
116 | break
117 | return i_end
118 |
119 |
120 | def clean_html(html):
121 | """
122 | 清洗html文本，去掉无用标签
123 | 1. "script","style",注释标签整行用空格代替
124 | 2. 特殊字符转义
125 | return:(pure_text,html_text):纯文本和包含标签的html文本
126 | """
127 | regex = re.compile(
128 | r'(?:)|' #doctype
129 | r'(?: