├── .gitignore
├── README.rst
├── Vagrantfile
├── config.yaml
├── exitwp.py
├── html2text.py
├── pip_requirements.txt
└── wordpress-xml
    └── .gitignore


/.gitignore:
--------------------------------------------------------------------------------
 1 | build/
 2 | *.pyc
 3 | .ropeproject/
 4 | .Python
 5 | lib/
 6 | bin/
 7 | build/
 8 | include/
 9 | .vagrant/
10 | .DS_Store
11 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | ######
 2 | Exitwp
 3 | ######
 4 | 
 5 | Exitwp is tool for making migration from one or more wordpress blogs to the `jekyll blog engine <https://github.com/mojombo/jekyll/>`_ as easy as possible.
 6 | 
 7 | By default it will try to convert as much information as possible from wordpress but can also be told to filter the amount of data it converts.
 8 | 
 9 | The latest version of these docs should always be available at https://github.com/thomasf/exitwp
10 | 
11 | Getting started
12 | ===============
13 |  * `Download <https://github.com/thomasf/exitwp/zipball/master>`_ or clone using ``git clone https://github.com/thomasf/exitwp.git``
14 |  * Export one or more wordpress blogs using the wordpress exporter under tools/export in wordpress admin.
15 |  * Put all wordpress xml files in the ``wordpress-xml`` directory
16 |  * Special note for Wordpress 3.1, you need to add a missing namespace in rss tag : ``xmlns:atom="http://www.w3.org/2005/Atom"``
17 |  * Run xmllint on your export file and fix errors if there are.
18 |  * Run the converter by typing ``python exitwp.py`` in the console from the directory of the unzipped archive
19 |  * You should now have all the blogs converted into separate directories under the ``build`` directory
20 | 
21 | Runtime dependencies
22 | ====================
23 |  * `Python <http://python.org/>`_ 2.6, 2.7, ???
24 |  * `html2text <http://www.aaronsw.com/2002/html2text/>`_ :  converts HTML to markdown (python)
25 |  * `PyYAML <http://pyyaml.org/wiki/PyYAML>`_ : Reading configuration files and writing YAML headers (python)
26 |  * `Beautiful soup <http://www.crummy.com/software/BeautifulSoup/>`_ : Parsing and downloading of post images/attachments (python)
27 | 
28 | 
29 | Installing dependencies in ubuntu/debian
30 | ----------------------------------------
31 | 
32 |    ``sudo apt-get install python-yaml python-bs4 python-html2text``
33 | 
34 | Installing Python dependencies using python package installer (pip)
35 | -------------------------------------------------------------------
36 | 
37 | From the checked out root for this project, type:
38 | 
39 |    ``sudo pip install --upgrade  -r pip_requirements.txt``
40 | 
41 | Note that PyYAML will require other packages to compile correctly under ubuntu/debian, these are installed by typing:
42 | 
43 |    ``sudo apt-get install libyaml-dev python-dev build-essential``
44 | 
45 | Using Vagrant for dependency management
46 | ---------------------------------------
47 | 
48 | In the event your local system is incompatible with the dependencies listed (or you'd rather not install them), you can use the included Vagrantfile to start a VM with all necessary dependencies installed.
49 | 
50 | 1. Lint and place all wordpress xml files in the ``wordpress-xml`` directory as mentioned above
51 | 2. In the directory of the unzipped archive, run ``vagrant up``.
52 | 3. SSH to your Vagrant VM using ``vagrant ssh``
53 | 4. Run ``cd /vagrant`` to open the VM's shared folder
54 | 5. Run the converter from the VM by typing ``python exitwp.py``
55 | 6. After the converter completes, exit the SSH session using ``exit``
56 | 7. You should now have all the blogs converted into separate directories under the ``build`` directory
57 | 8. **Important:** Once satisfied with the results, run ``vagrant destroy -f`` to shut down the VM and remove the virtual drive from your local machine
58 | 
59 | Configuration/Customization
60 | ===========================
61 | 
62 | See the `configuration file <https://github.com/thomasf/exitwp/blob/master/config.yaml>`_ for all configurable options.
63 | 
64 | Some things like custom handling of non standard post types is not fully configurable through the config file. You might have to modify the `source code <https://github.com/thomasf/exitwp/blob/master/exitwp.py>`_ to add custom parsing behaviour.
65 | 
66 | Known issues
67 | ============
68 |  * Target file names are some times less than optimal.
69 |  * Rewriting of image/attachment links if they are downloaded would be a good feature
70 |  * There will probably be issues when migrating non utf-8 encoded wordpress dump files (if they exist).
71 | 
72 | Other Tools
73 | ===========
74 |  * A Gist to convert WP-Footnotes style footnotes to PHP Markdown Extra style footnotes: https://gist.github.com/1246047
75 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | $script = <<SCRIPT
 5 | echo 'Provisioning Exitwp Library Dependencies...'
 6 | apt-get update --fix-missing
 7 | apt-get install -q -y git python-pip
 8 | apt-get install -q -y python-yaml python-bs4 python-html2text libyaml-dev python-dev
 9 | cd /vagrant
10 | echo 'Provisioning Exitwp Python Requirements...'
11 | pip install --upgrade -r pip_requirements.txt
12 | echo 'Provisioning complete!'
13 | SCRIPT
14 | 
15 | Vagrant.configure("2") do |config|
16 |   config.vm.box = "Puppetlabs Ubuntu 12.04.2 x86_64, VBox 4.2.10, No Puppet or Chef"
17 |   config.vm.box_url = "http://puppet-vagrant-boxes.puppetlabs.com/ubuntu-server-12042-x64-vbox4210-nocm.box"
18 |   config.vm.provision :shell, :inline => $script
19 | end


--------------------------------------------------------------------------------
/config.yaml:
--------------------------------------------------------------------------------
 1 | # The directory where exitwp looks for wordpress export xml files.
 2 | wp_exports: wordpress-xml
 3 | 
 4 | # The target directory where all output is saved.
 5 | build_dir: build
 6 | 
 7 | # Output format: primary choices are html or markdown.
 8 | target_format: markdown
 9 | 
10 | # The date format of the wikipedia export file.
11 | # I'm not sure if this ever differs depending on wordpress localization.
12 | # Wordpress is often so full of strange quirks so I wouldnt rule it out.
13 | date_format: '%Y-%m-%d %H:%M:%S'
14 | 
15 | # Try to download and reloacate all images locally to the blog.
16 | download_images: False
17 | 
18 | # Item types we don't want to import.
19 | item_type_filter: {attachment, nav_menu_item}
20 | 
21 | # filter by any field type on the post.
22 | # By default, we're filtering based on field "status" set to "draft"
23 | item_field_filter: {status: draft}
24 | 
25 | taxonomies:
26 |   # Filter taxonomies.
27 |   filter: {}
28 |   # Filter taxonomies entries.
29 |   entry_filter: {category: Uncategorized}
30 |   # Rename taxonomies when writing jekyll output format.
31 |   name_mapping: {category: categories, post_tag: tags}
32 | 
33 | # Replace certain patterns in body
34 | # Simply replace the key with its value
35 | body_replace: {
36 |   # '<pre.*?lang="(.*?)".*?>': '\n{% codeblock \1 lang:\1 %}\n',
37 |   # '<pre.*?>': '\n{% codeblock %}\n',
38 |   # '</pre>': '\n{% endcodeblock %}\n',
39 | 
40 | #    '[python]': '{% codeblock lang:python %}',
41 | #    '[/python]': '{% endcodeblock %}',
42 | }
43 | 


--------------------------------------------------------------------------------
/exitwp.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | 
  3 | import codecs
  4 | import os
  5 | import re
  6 | import sys
  7 | from datetime import datetime, timedelta, tzinfo
  8 | from glob import glob
  9 | from urllib import urlretrieve
 10 | from urlparse import urljoin, urlparse
 11 | from xml.etree.ElementTree import ElementTree, XMLTreeBuilder
 12 | 
 13 | import yaml
 14 | from bs4 import BeautifulSoup
 15 | 
 16 | from html2text import html2text_file
 17 | 
 18 | '''
 19 | exitwp - Wordpress xml exports to Jekykll blog format conversion
 20 | 
 21 | Tested with Wordpress 3.3.1 and jekyll 0.11.2
 22 | 
 23 | '''
 24 | ######################################################
 25 | # Configration
 26 | ######################################################
 27 | config = yaml.load(file('config.yaml', 'r'))
 28 | wp_exports = config['wp_exports']
 29 | build_dir = config['build_dir']
 30 | download_images = config['download_images']
 31 | target_format = config['target_format']
 32 | taxonomy_filter = set(config['taxonomies']['filter'])
 33 | taxonomy_entry_filter = config['taxonomies']['entry_filter']
 34 | taxonomy_name_mapping = config['taxonomies']['name_mapping']
 35 | item_type_filter = set(config['item_type_filter'])
 36 | item_field_filter = config['item_field_filter']
 37 | date_fmt = config['date_format']
 38 | body_replace = config['body_replace']
 39 | 
 40 | 
 41 | # Time definitions
 42 | ZERO = timedelta(0)
 43 | HOUR = timedelta(hours=1)
 44 | 
 45 | 
 46 | # UTC support
 47 | class UTC(tzinfo):
 48 |     """UTC."""
 49 | 
 50 |     def utcoffset(self, dt):
 51 |         return ZERO
 52 | 
 53 |     def tzname(self, dt):
 54 |         return 'UTC'
 55 | 
 56 |     def dst(self, dt):
 57 |         return ZERO
 58 | 
 59 | 
 60 | class ns_tracker_tree_builder(XMLTreeBuilder):
 61 | 
 62 |     def __init__(self):
 63 |         XMLTreeBuilder.__init__(self)
 64 |         self._parser.StartNamespaceDeclHandler = self._start_ns
 65 |         self.namespaces = {}
 66 | 
 67 |     def _start_ns(self, prefix, ns):
 68 |         self.namespaces[prefix] = '{' + ns + '}'
 69 | 
 70 | 
 71 | def html2fmt(html, target_format):
 72 |     #   html = html.replace("\n\n", '<br/><br/>')
 73 |     #   html = html.replace('<pre lang="xml">', '<pre lang="xml"><![CDATA[')
 74 |     #   html = html.replace('</pre>', ']]></pre>')
 75 |     if target_format == 'html':
 76 |         return html
 77 |     else:
 78 |         return html2text_file(html, None)
 79 | 
 80 | 
 81 | def parse_wp_xml(file):
 82 |     parser = ns_tracker_tree_builder()
 83 |     tree = ElementTree()
 84 |     print 'reading: ' + wpe
 85 |     root = tree.parse(file, parser)
 86 |     ns = parser.namespaces
 87 |     ns[''] = ''
 88 | 
 89 |     c = root.find('channel')
 90 | 
 91 |     def parse_header():
 92 |         return {
 93 |             'title': unicode(c.find('title').text),
 94 |             'link': unicode(c.find('link').text),
 95 |             'description': unicode(c.find('description').text)
 96 |         }
 97 | 
 98 |     def parse_items():
 99 |         export_items = []
100 |         xml_items = c.findall('item')
101 |         for i in xml_items:
102 |             taxanomies = i.findall('category')
103 |             export_taxanomies = {}
104 |             for tax in taxanomies:
105 |                 if 'domain' not in tax.attrib:
106 |                     continue
107 |                 t_domain = unicode(tax.attrib['domain'])
108 |                 t_entry = unicode(tax.text)
109 |                 if (not (t_domain in taxonomy_filter) and
110 |                     not (t_domain
111 |                          in taxonomy_entry_filter and
112 |                          taxonomy_entry_filter[t_domain] == t_entry)):
113 |                     if t_domain not in export_taxanomies:
114 |                         export_taxanomies[t_domain] = []
115 |                     export_taxanomies[t_domain].append(t_entry)
116 | 
117 |             def gi(q, unicode_wrap=True, empty=False):
118 |                 namespace = ''
119 |                 tag = ''
120 |                 if q.find(':') > 0:
121 |                     namespace, tag = q.split(':', 1)
122 |                 else:
123 |                     tag = q
124 |                 try:
125 |                     result = (i.find(q, ns) or i.find(tag) or i.find(ns[namespace] + tag)).text.strip()
126 |                     print result.encode('utf-8')
127 |                 except AttributeError:
128 |                     result = 'No Content Found'
129 |                     if empty:
130 |                         result = ''
131 |                 if unicode_wrap:
132 |                     result = unicode(result)
133 |                 return result
134 | 
135 |             body = gi('content:encoded')
136 |             for key in body_replace:
137 |                 # body = body.replace(key, body_replace[key])
138 |                 body = re.sub(key, body_replace[key], body)
139 | 
140 |             img_srcs = []
141 |             if body is not None:
142 |                 try:
143 |                     soup = BeautifulSoup(body)
144 |                     img_tags = soup.find_all('img')
145 |                     for img in img_tags:
146 |                         img_srcs.append(img['src'])
147 |                 except:
148 |                     print 'could not parse html: ' + body
149 |             # print img_srcs
150 | 
151 |             excerpt = gi('excerpt:encoded', empty=True)
152 | 
153 |             export_item = {
154 |                 'title': gi('title'),
155 |                 'link': gi('link'),
156 |                 'author': gi('dc:creator'),
157 |                 'date': gi('wp:post_date_gmt'),
158 |                 'slug': gi('wp:post_name'),
159 |                 'status': gi('wp:status'),
160 |                 'type': gi('wp:post_type'),
161 |                 'wp_id': gi('wp:post_id'),
162 |                 'parent': gi('wp:post_parent'),
163 |                 'comments': gi('wp:comment_status') == u'open',
164 |                 'taxanomies': export_taxanomies,
165 |                 'body': body,
166 |                 'excerpt': excerpt,
167 |                 'img_srcs': img_srcs
168 |             }
169 | 
170 |             export_items.append(export_item)
171 | 
172 |         return export_items
173 | 
174 |     return {
175 |         'header': parse_header(),
176 |         'items': parse_items(),
177 |     }
178 | 
179 | 
180 | def write_jekyll(data, target_format):
181 | 
182 |     sys.stdout.write('writing')
183 |     item_uids = {}
184 |     attachments = {}
185 | 
186 |     def get_blog_path(data, path_infix='jekyll'):
187 |         name = data['header']['link']
188 |         name = re.sub('^https?', '', name)
189 |         name = re.sub('[^A-Za-z0-9_.-]', '', name)
190 |         return os.path.normpath(build_dir + '/' + path_infix + '/' + name)
191 | 
192 |     blog_dir = get_blog_path(data)
193 | 
194 |     def get_full_dir(dir):
195 |         full_dir = os.path.normpath(blog_dir + '/' + dir)
196 |         if (not os.path.exists(full_dir)):
197 |             os.makedirs(full_dir)
198 |         return full_dir
199 | 
200 |     def open_file(file):
201 |         f = codecs.open(file, 'w', encoding='utf-8')
202 |         return f
203 | 
204 |     def get_item_uid(item, date_prefix=False, namespace=''):
205 |         result = None
206 |         if namespace not in item_uids:
207 |             item_uids[namespace] = {}
208 | 
209 |         if item['wp_id'] in item_uids[namespace]:
210 |             result = item_uids[namespace][item['wp_id']]
211 |         else:
212 |             uid = []
213 |             if (date_prefix):
214 |                 try:
215 |                     dt = datetime.strptime(item['date'], date_fmt)
216 |                 except:
217 |                     dt = datetime.today()
218 |                     print 'Wrong date in', item['title']
219 |                 uid.append(dt.strftime('%Y-%m-%d'))
220 |                 uid.append('-')
221 |             s_title = item['slug']
222 |             if s_title is None or s_title == '':
223 |                 s_title = item['title']
224 |             if s_title is None or s_title == '':
225 |                 s_title = 'untitled'
226 |             s_title = s_title.replace(' ', '_')
227 |             s_title = re.sub('[^a-zA-Z0-9_-]', '', s_title)
228 |             uid.append(s_title)
229 |             fn = ''.join(uid)
230 |             n = 1
231 |             while fn in item_uids[namespace]:
232 |                 n = n + 1
233 |                 fn = ''.join(uid) + '_' + str(n)
234 |                 item_uids[namespace][i['wp_id']] = fn
235 |             result = fn
236 |         return result
237 | 
238 |     def get_item_path(item, dir=''):
239 |         full_dir = get_full_dir(dir)
240 |         filename_parts = [full_dir, '/']
241 |         filename_parts.append(item['uid'])
242 |         if item['type'] == 'page':
243 |             if (not os.path.exists(''.join(filename_parts))):
244 |                 os.makedirs(''.join(filename_parts))
245 |             filename_parts.append('/index')
246 |         filename_parts.append('.')
247 |         filename_parts.append(target_format)
248 |         return ''.join(filename_parts)
249 | 
250 |     def get_attachment_path(src, dir, dir_prefix='images'):
251 |         try:
252 |             files = attachments[dir]
253 |         except KeyError:
254 |             attachments[dir] = files = {}
255 | 
256 |         try:
257 |             filename = files[src]
258 |         except KeyError:
259 |             file_root, file_ext = os.path.splitext(os.path.basename(
260 |                 urlparse(src)[2]))
261 |             file_infix = 1
262 |             if file_root == '':
263 |                 file_root = '1'
264 |             current_files = files.values()
265 |             maybe_filename = file_root + file_ext
266 |             while maybe_filename in current_files:
267 |                 maybe_filename = file_root + '-' + str(file_infix) + file_ext
268 |                 file_infix = file_infix + 1
269 |             files[src] = filename = maybe_filename
270 | 
271 |         target_dir = os.path.normpath(blog_dir + '/' + dir_prefix + '/' + dir)
272 |         target_file = os.path.normpath(target_dir + '/' + filename)
273 | 
274 |         if (not os.path.exists(target_dir)):
275 |             os.makedirs(target_dir)
276 | 
277 |         # if src not in attachments[dir]:
278 |         #     print target_name
279 |         return target_file
280 | 
281 |     for i in data['items']:
282 |         skip_item = False
283 | 
284 |         for field, value in item_field_filter.iteritems():
285 |             if(i[field] == value):
286 |                 skip_item = True
287 |                 break
288 | 
289 |         if(skip_item):
290 |             continue
291 | 
292 |         sys.stdout.write('.')
293 |         sys.stdout.flush()
294 |         out = None
295 |         try:
296 |             date = datetime.strptime(i['date'], '%Y-%m-%d %H:%M:%S').replace(tzinfo=UTC())
297 |         except:
298 |             date = datetime.today()
299 |             print 'Wrong date in', i['title']
300 |         yaml_header = {
301 |             'title': i['title'],
302 |             'link': i['link'],
303 |             'author': i['author'],
304 |             'date': date,
305 |             'slug': i['slug'],
306 |             'wordpress_id': int(i['wp_id']),
307 |             'comments': i['comments'],
308 |         }
309 |         if len(i['excerpt']) > 0:
310 |             yaml_header['excerpt'] = i['excerpt']
311 |         if i['status'] != u'publish':
312 |             yaml_header['published'] = False
313 | 
314 |         if i['type'] == 'post':
315 |             i['uid'] = get_item_uid(i, date_prefix=True)
316 |             fn = get_item_path(i, dir='_posts')
317 |             out = open_file(fn)
318 |             yaml_header['layout'] = 'post'
319 |         elif i['type'] == 'page':
320 |             i['uid'] = get_item_uid(i)
321 |             # Chase down parent path, if any
322 |             parentpath = ''
323 |             item = i
324 |             while item['parent'] != '0':
325 |                 item = next((parent for parent in data['items']
326 |                              if parent['wp_id'] == item['parent']), None)
327 |                 if item:
328 |                     parentpath = get_item_uid(item) + '/' + parentpath
329 |                 else:
330 |                     break
331 |             fn = get_item_path(i, parentpath)
332 |             out = open_file(fn)
333 |             yaml_header['layout'] = 'page'
334 |         elif i['type'] in item_type_filter:
335 |             pass
336 |         else:
337 |             print 'Unknown item type :: ' + i['type']
338 | 
339 |         if download_images:
340 |             for img in i['img_srcs']:
341 |                 try:
342 |                     urlretrieve(urljoin(data['header']['link'],
343 |                                         img.encode('utf-8')),
344 |                                 get_attachment_path(img, i['uid']))
345 |                 except:
346 |                     print '\n unable to download ' + urljoin(
347 |                         data['header']['link'], img.encode('utf-8'))
348 | 
349 |         if out is not None:
350 |             def toyaml(data):
351 |                 return yaml.safe_dump(data, allow_unicode=True,
352 |                                       default_flow_style=False).decode('utf-8')
353 | 
354 |             tax_out = {}
355 |             for taxonomy in i['taxanomies']:
356 |                 for tvalue in i['taxanomies'][taxonomy]:
357 |                     t_name = taxonomy_name_mapping.get(taxonomy, taxonomy)
358 |                     if t_name not in tax_out:
359 |                         tax_out[t_name] = []
360 |                     if tvalue in tax_out[t_name]:
361 |                         continue
362 |                     tax_out[t_name].append(tvalue)
363 | 
364 |             out.write('---\n')
365 |             if len(yaml_header) > 0:
366 |                 out.write(toyaml(yaml_header))
367 |             if len(tax_out) > 0:
368 |                 out.write(toyaml(tax_out))
369 | 
370 |             out.write('---\n\n')
371 |             try:
372 |                 out.write(html2fmt(i['body'], target_format))
373 |             except:
374 |                 print '\n Parse error on: ' + i['title']
375 | 
376 |             out.close()
377 |     print '\n'
378 | 
379 | wp_exports = glob(wp_exports + '/*.xml')
380 | for wpe in wp_exports:
381 |     data = parse_wp_xml(wpe)
382 |     write_jekyll(data, target_format)
383 | 
384 | print 'done'
385 | 


--------------------------------------------------------------------------------
/html2text.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | """html2text: Turn HTML into equivalent Markdown-structured text."""
  3 | __version__ = "3.1"
  4 | __author__ = "Aaron Swartz (me@aaronsw.com)"
  5 | __copyright__ = "(C) 2004-2008 Aaron Swartz. GNU GPL 3."
  6 | __contributors__ = ["Martin 'Joey' Schulze", "Ricardo Reyes", "Kevin Jay North"]
  7 | 
  8 | # TODO:
  9 | #   Support decoded entities with unifiable.
 10 | 
 11 | try:
 12 |     True
 13 | except NameError:
 14 |     setattr(__builtins__, 'True', 1)
 15 |     setattr(__builtins__, 'False', 0)
 16 | 
 17 | def has_key(x, y):
 18 |     if hasattr(x, 'has_key'): return x.has_key(y)
 19 |     else: return y in x
 20 | 
 21 | try:
 22 |     import htmlentitydefs
 23 |     import urlparse
 24 |     import HTMLParser
 25 | except ImportError: #Python3
 26 |     import html.entities as htmlentitydefs
 27 |     import urllib.parse as urlparse
 28 |     import html.parser as HTMLParser
 29 | try: #Python3
 30 |     import urllib.request as urllib
 31 | except:
 32 |     import urllib
 33 | import optparse, re, sys, codecs, types
 34 | 
 35 | try: from textwrap import wrap
 36 | except: pass
 37 | 
 38 | # Use Unicode characters instead of their ascii psuedo-replacements
 39 | UNICODE_SNOB = 0
 40 | 
 41 | # Put the links after each paragraph instead of at the end.
 42 | LINKS_EACH_PARAGRAPH = 0
 43 | 
 44 | # Wrap long lines at position. 0 for no wrapping. (Requires Python 2.3.)
 45 | BODY_WIDTH = 78
 46 | 
 47 | # Don't show internal links (href="#local-anchor") -- corresponding link targets
 48 | # won't be visible in the plain text file anyway.
 49 | SKIP_INTERNAL_LINKS = True
 50 | 
 51 | # Use inline, rather than reference, formatting for images and links
 52 | INLINE_LINKS = True
 53 | 
 54 | # Number of pixels Google indents nested lists
 55 | GOOGLE_LIST_INDENT = 36
 56 | 
 57 | IGNORE_ANCHORS = False
 58 | IGNORE_IMAGES = False
 59 | 
 60 | ### Entity Nonsense ###
 61 | 
 62 | def name2cp(k):
 63 |     if k == 'apos': return ord("'")
 64 |     if hasattr(htmlentitydefs, "name2codepoint"): # requires Python 2.3
 65 |         return htmlentitydefs.name2codepoint[k]
 66 |     else:
 67 |         k = htmlentitydefs.entitydefs[k]
 68 |         if k.startswith("&#") and k.endswith(";"): return int(k[2:-1]) # not in latin-1
 69 |         return ord(codecs.latin_1_decode(k)[0])
 70 | 
 71 | unifiable = {'rsquo':"'", 'lsquo':"'", 'rdquo':'"', 'ldquo':'"',
 72 | 'copy':'(C)', 'mdash':'--', 'nbsp':' ', 'rarr':'->', 'larr':'<-', 'middot':'*',
 73 | 'ndash':'-', 'oelig':'oe', 'aelig':'ae',
 74 | 'agrave':'a', 'aacute':'a', 'acirc':'a', 'atilde':'a', 'auml':'a', 'aring':'a',
 75 | 'egrave':'e', 'eacute':'e', 'ecirc':'e', 'euml':'e',
 76 | 'igrave':'i', 'iacute':'i', 'icirc':'i', 'iuml':'i',
 77 | 'ograve':'o', 'oacute':'o', 'ocirc':'o', 'otilde':'o', 'ouml':'o',
 78 | 'ugrave':'u', 'uacute':'u', 'ucirc':'u', 'uuml':'u',
 79 | 'lrm':'', 'rlm':''}
 80 | 
 81 | unifiable_n = {}
 82 | 
 83 | for k in unifiable.keys():
 84 |     unifiable_n[name2cp(k)] = unifiable[k]
 85 | 
 86 | def charref(name):
 87 |     if name[0] in ['x','X']:
 88 |         c = int(name[1:], 16)
 89 |     else:
 90 |         c = int(name)
 91 | 
 92 |     if not UNICODE_SNOB and c in unifiable_n.keys():
 93 |         return unifiable_n[c]
 94 |     else:
 95 |         try:
 96 |             return unichr(c)
 97 |         except NameError: #Python3
 98 |             return chr(c)
 99 | 
100 | def entityref(c):
101 |     if not UNICODE_SNOB and c in unifiable.keys():
102 |         return unifiable[c]
103 |     else:
104 |         try: name2cp(c)
105 |         except KeyError: return "&" + c + ';'
106 |         else:
107 |             try:
108 |                 return unichr(name2cp(c))
109 |             except NameError: #Python3
110 |                 return chr(name2cp(c))
111 | 
112 | def replaceEntities(s):
113 |     s = s.group(1)
114 |     if s[0] == "#":
115 |         return charref(s[1:])
116 |     else: return entityref(s)
117 | 
118 | r_unescape = re.compile(r"&(#?[xX]?(?:[0-9a-fA-F]+|\w{1,8}));")
119 | def unescape(s):
120 |     return r_unescape.sub(replaceEntities, s)
121 | 
122 | ### End Entity Nonsense ###
123 | 
124 | def onlywhite(line):
125 |     """Return true if the line does only consist of whitespace characters."""
126 |     for c in line:
127 |         if c is not ' ' and c is not '  ':
128 |             return c is ' '
129 |     return line
130 | 
131 | def optwrap(text):
132 |     """Wrap all paragraphs in the provided text."""
133 |     if not BODY_WIDTH:
134 |         return text
135 | 
136 |     assert wrap, "Requires Python 2.3."
137 |     result = ''
138 |     newlines = 0
139 |     for para in text.split("\n"):
140 |         if len(para) > 0:
141 |             if para[0] != ' ' and para[0] != '-' and para[0] != '*':
142 |                 for line in wrap(para, BODY_WIDTH):
143 |                     result += line + "\n"
144 |                 result += "\n"
145 |                 newlines = 2
146 |             else:
147 |                 if not onlywhite(para):
148 |                     result += para + "\n"
149 |                     newlines = 1
150 |         else:
151 |             if newlines < 2:
152 |                 result += "\n"
153 |                 newlines += 1
154 |     return result
155 | 
156 | def hn(tag):
157 |     if tag[0] == 'h' and len(tag) == 2:
158 |         try:
159 |             n = int(tag[1])
160 |             if n in range(1, 10): return n
161 |         except ValueError: return 0
162 | 
163 | def dumb_property_dict(style):
164 |     """returns a hash of css attributes"""
165 |     return dict([(x.strip(), y.strip()) for x, y in [z.split(':', 1) for z in style.split(';') if ':' in z]]);
166 | 
167 | def dumb_css_parser(data):
168 |     """returns a hash of css selectors, each of which contains a hash of css attributes"""
169 |     # remove @import sentences
170 |     importIndex = data.find('@import')
171 |     while importIndex != -1:
172 |         data = data[0:importIndex] + data[data.find(';', importIndex) + 1:]
173 |         importIndex = data.find('@import')
174 | 
175 |     # parse the css. reverted from dictionary compehension in order to support older pythons
176 |     elements =  [x.split('{') for x in data.split('}') if '{' in x.strip()]
177 |     elements = dict([(a.strip(), dumb_property_dict(b)) for a, b in elements])
178 | 
179 |     return elements
180 | 
181 | def element_style(attrs, style_def, parent_style):
182 |     """returns a hash of the 'final' style attributes of the element"""
183 |     style = parent_style.copy()
184 |     if 'class' in attrs:
185 |         for css_class in attrs['class'].split():
186 |             css_style = style_def['.' + css_class]
187 |             style.update(css_style)
188 |     if 'style' in attrs:
189 |         immediate_style = dumb_property_dict(attrs['style'])
190 |         style.update(immediate_style)
191 |     return style
192 | 
193 | def google_list_style(style):
194 |     """finds out whether this is an ordered or unordered list"""
195 |     if 'list-style-type' in style:
196 |         list_style = style['list-style-type']
197 |         if list_style in ['disc', 'circle', 'square', 'none']:
198 |             return 'ul'
199 |     return 'ol'
200 | 
201 | def google_nest_count(style):
202 |     """calculate the nesting count of google doc lists"""
203 |     nest_count = 0
204 |     if 'margin-left' in style:
205 |         nest_count = int(style['margin-left'][:-2]) / GOOGLE_LIST_INDENT
206 |     return nest_count
207 | 
208 | def google_has_height(style):
209 |     """check if the style of the element has the 'height' attribute explicitly defined"""
210 |     if 'height' in style:
211 |         return True
212 |     return False
213 | 
214 | def google_text_emphasis(style):
215 |     """return a list of all emphasis modifiers of the element"""
216 |     emphasis = []
217 |     if 'text-decoration' in style:
218 |         emphasis.append(style['text-decoration'])
219 |     if 'font-style' in style:
220 |         emphasis.append(style['font-style'])
221 |     if 'font-weight' in style:
222 |         emphasis.append(style['font-weight'])
223 |     return emphasis
224 | 
225 | def google_fixed_width_font(style):
226 |     """check if the css of the current element defines a fixed width font"""
227 |     font_family = ''
228 |     if 'font-family' in style:
229 |         font_family = style['font-family']
230 |     if 'Courier New' == font_family or 'Consolas' == font_family:
231 |         return True
232 |     return False
233 | 
234 | def list_numbering_start(attrs):
235 |     """extract numbering from list element attributes"""
236 |     if 'start' in attrs:
237 |         return int(attrs['start']) - 1
238 |     else:
239 |         return 0
240 | 
241 | class _html2text(HTMLParser.HTMLParser):
242 |     def __init__(self, out=None, baseurl=''):
243 |         HTMLParser.HTMLParser.__init__(self)
244 | 
245 |         if out is None: self.out = self.outtextf
246 |         else: self.out = out
247 |         self.outtextlist = [] # empty list to store output characters before they are  "joined"
248 |         try:
249 |             self.outtext = unicode()
250 |         except NameError: # Python3
251 |             self.outtext = str()
252 |         self.quiet = 0
253 |         self.p_p = 0 # number of newline character to print before next output
254 |         self.outcount = 0
255 |         self.start = 1
256 |         self.space = 0
257 |         self.a = []
258 |         self.astack = []
259 |         self.acount = 0
260 |         self.list = []
261 |         self.blockquote = 0
262 |         self.pre = 0
263 |         self.startpre = 0
264 |         self.code = False
265 |         self.br_toggle = ''
266 |         self.lastWasNL = 0
267 |         self.lastWasList = False
268 |         self.style = 0
269 |         self.style_def = {}
270 |         self.tag_stack = []
271 |         self.emphasis = 0
272 |         self.drop_white_space = 0
273 |         self.inheader = False
274 |         self.abbr_title = None # current abbreviation definition
275 |         self.abbr_data = None # last inner HTML (for abbr being defined)
276 |         self.abbr_list = {} # stack of abbreviations to write later
277 |         self.baseurl = baseurl
278 | 
279 |         if options.google_doc:
280 |             del unifiable_n[name2cp('nbsp')]
281 |             unifiable['nbsp'] = '&nbsp_place_holder;'
282 | 
283 |     def feed(self, data):
284 |         data = data.replace("</' + 'script>", "</ignore>")
285 |         HTMLParser.HTMLParser.feed(self, data)
286 | 
287 |     def outtextf(self, s):
288 |         self.outtextlist.append(s)
289 |         if s: self.lastWasNL = s[-1] == '\n'
290 | 
291 |     def close(self):
292 |         HTMLParser.HTMLParser.close(self)
293 | 
294 |         self.pbr()
295 |         self.o('', 0, 'end')
296 | 
297 |         self.outtext = self.outtext.join(self.outtextlist)
298 | 
299 |         if options.google_doc:
300 |             self.outtext = self.outtext.replace('&nbsp_place_holder;', ' ');
301 | 
302 |         return self.outtext
303 | 
304 |     def handle_charref(self, c):
305 |         self.o(charref(c), 1)
306 | 
307 |     def handle_entityref(self, c):
308 |         self.o(entityref(c), 1)
309 | 
310 |     def handle_starttag(self, tag, attrs):
311 |         self.handle_tag(tag, attrs, 1)
312 | 
313 |     def handle_endtag(self, tag):
314 |         self.handle_tag(tag, None, 0)
315 | 
316 |     def previousIndex(self, attrs):
317 |         """ returns the index of certain set of attributes (of a link) in the
318 |             self.a list
319 | 
320 |             If the set of attributes is not found, returns None
321 |         """
322 |         if not has_key(attrs, 'href'): return None
323 | 
324 |         i = -1
325 |         for a in self.a:
326 |             i += 1
327 |             match = 0
328 | 
329 |             if has_key(a, 'href') and a['href'] == attrs['href']:
330 |                 if has_key(a, 'title') or has_key(attrs, 'title'):
331 |                         if (has_key(a, 'title') and has_key(attrs, 'title') and
332 |                             a['title'] == attrs['title']):
333 |                             match = True
334 |                 else:
335 |                     match = True
336 | 
337 |             if match: return i
338 | 
339 |     def drop_last(self, nLetters):
340 |         if not self.quiet:
341 |             self.outtext = self.outtext[:-nLetters]
342 | 
343 |     def handle_comment(self, data):
344 |         if data == "more":
345 |             self.o("<!-- more -->")
346 | 
347 |     def handle_emphasis(self, start, tag_style, parent_style):
348 |         """handles various text emphases"""
349 |         tag_emphasis = google_text_emphasis(tag_style)
350 |         parent_emphasis = google_text_emphasis(parent_style)
351 | 
352 |         # handle Google's text emphasis
353 |         strikethrough =  'line-through' in tag_emphasis and options.hide_strikethrough
354 |         bold = 'bold' in tag_emphasis and not 'bold' in parent_emphasis
355 |         italic = 'italic' in tag_emphasis and not 'italic' in parent_emphasis
356 |         fixed = google_fixed_width_font(tag_style) and not \
357 |                 google_fixed_width_font(parent_style) and not self.pre
358 | 
359 |         if start:
360 |             # crossed-out text must be handled before other attributes
361 |             # in order not to output qualifiers unnecessarily
362 |             if bold or italic or fixed:
363 |                 self.emphasis += 1
364 |             if strikethrough:
365 |                 self.quiet += 1
366 |             if italic:
367 |                 self.o("_")
368 |                 self.drop_white_space += 1
369 |             if bold:
370 |                 self.o("**")
371 |                 self.drop_white_space += 1
372 |             if fixed:
373 |                 self.o('`')
374 |                 self.drop_white_space += 1
375 |                 self.code = True
376 |         else:
377 |             if bold or italic or fixed:
378 |                 # there must not be whitespace before closing emphasis mark
379 |                 self.emphasis -= 1
380 |                 self.space = 0
381 |                 self.outtext = self.outtext.rstrip()
382 |             if fixed:
383 |                 if self.drop_white_space:
384 |                     # empty emphasis, drop it
385 |                     self.drop_last(1)
386 |                     self.drop_white_space -= 1
387 |                 else:
388 |                     self.o('`')
389 |                 self.code = False
390 |             if bold:
391 |                 if self.drop_white_space:
392 |                     # empty emphasis, drop it
393 |                     self.drop_last(2)
394 |                     self.drop_white_space -= 1
395 |                 else:
396 |                     self.o("**")
397 |             if italic:
398 |                 if self.drop_white_space:
399 |                     # empty emphasis, drop it
400 |                     self.drop_last(1)
401 |                     self.drop_white_space -= 1
402 |                 else:
403 |                     self.o("_")
404 |             # space is only allowed after *all* emphasis marks
405 |             if (bold or italic) and not self.emphasis:
406 |                     self.o(" ")
407 |             if strikethrough:
408 |                 self.quiet -= 1
409 | 
410 |     def handle_tag(self, tag, attrs, start):
411 |         if attrs is None:
412 |             attrs = {}
413 |         else:
414 |             attrs = dict(attrs)
415 | 
416 |         if self.pre and not tag == "pre":
417 |             if start:
418 |                 attrs_string = ""
419 |                 for k, v in attrs.items():
420 |                     attrs_string += " " + k + "=\"" + v + "\""
421 |                 self.o("<"+tag+attrs_string+">")
422 |             else:
423 |                 self.o("</"+tag+">")
424 |             return None
425 | 
426 |         if options.google_doc:
427 |             # the attrs parameter is empty for a closing tag. in addition, we
428 |             # need the attributes of the parent nodes in order to get a
429 |             # complete style description for the current element. we assume
430 |             # that google docs export well formed html.
431 |             parent_style = {}
432 |             if start:
433 |                 if self.tag_stack:
434 |                   parent_style = self.tag_stack[-1][2]
435 |                 tag_style = element_style(attrs, self.style_def, parent_style)
436 |                 self.tag_stack.append((tag, attrs, tag_style))
437 |             else:
438 |                 dummy, attrs, tag_style = self.tag_stack.pop()
439 |                 if self.tag_stack:
440 |                     parent_style = self.tag_stack[-1][2]
441 | 
442 |         if hn(tag):
443 |             self.p()
444 |             if start:
445 |                 self.inheader = True
446 |                 self.o(hn(tag)*"#" + ' ')
447 |             else:
448 |                 self.inheader = False
449 |                 return # prevent redundant emphasis marks on headers
450 | 
451 |         if tag in ['p', 'div']:
452 |             if options.google_doc:
453 |                 if start and google_has_height(tag_style):
454 |                     self.p()
455 |                 else:
456 |                     self.soft_br()
457 |             else:
458 |                 self.p()
459 | 
460 |         if tag == "br" and start: self.o("  \n")
461 | 
462 |         if tag == "hr" and start:
463 |             self.p()
464 |             self.o("* * *")
465 |             self.p()
466 | 
467 |         if tag in ["head", "style", 'script']:
468 |             if start: self.quiet += 1
469 |             else: self.quiet -= 1
470 | 
471 |         if tag == "style":
472 |             if start: self.style += 1
473 |             else: self.style -= 1
474 | 
475 |         if tag in ["body"]:
476 |             self.quiet = 0 # sites like 9rules.com never close <head>
477 | 
478 |         if tag == "blockquote":
479 |             if start:
480 |                 self.p(); self.o('<blockquote>', 0, 1); self.start = 1
481 |                 # self.p(); self.o('> ', 0, 1); self.start = 1
482 |                 self.blockquote += 1
483 |             else:
484 |                 self.o('</blockquote>', 0, 1)
485 |                 self.blockquote -= 1
486 |                 self.p()
487 | 
488 |         if tag in ['em', 'i', 'u']: self.o("_")
489 |         if tag in ['strong', 'b']: self.o("**")
490 |         if tag in ['del', 'strike']:
491 |             if start:
492 |                 self.o("<"+tag+">")
493 |             else:
494 |                 self.o("</"+tag+">")
495 | 
496 |         if options.google_doc:
497 |             if not self.inheader:
498 |                 # handle some font attributes, but leave headers clean
499 |                 self.handle_emphasis(start, tag_style, parent_style)
500 | 
501 |         if tag == "code" and not self.pre: self.o('`') #TODO: `` `this` ``
502 |         if tag == "abbr":
503 |             if start:
504 |                 self.abbr_title = None
505 |                 self.abbr_data = ''
506 |                 if has_key(attrs, 'title'):
507 |                     self.abbr_title = attrs['title']
508 |             else:
509 |                 if self.abbr_title != None:
510 |                     self.abbr_list[self.abbr_data] = self.abbr_title
511 |                     self.abbr_title = None
512 |                 self.abbr_data = ''
513 | 
514 |         if tag == "a" and not IGNORE_ANCHORS:
515 |             if start:
516 |                 if has_key(attrs, 'href') and not (SKIP_INTERNAL_LINKS and attrs['href'].startswith('#')):
517 |                     self.astack.append(attrs)
518 |                     self.o("[")
519 |                 else:
520 |                     self.astack.append(None)
521 |             else:
522 |                 if self.astack:
523 |                     a = self.astack.pop()
524 |                     if a:
525 |                         if INLINE_LINKS:
526 |                             self.o("](" + a['href'] + ")")
527 |                         else:
528 |                             i = self.previousIndex(a)
529 |                             if i is not None:
530 |                                 a = self.a[i]
531 |                             else:
532 |                                 self.acount += 1
533 |                                 a['count'] = self.acount
534 |                                 a['outcount'] = self.outcount
535 |                                 self.a.append(a)
536 |                             self.o("][" + str(a['count']) + "]")
537 | 
538 |         if tag == "img" and start and not IGNORE_IMAGES:
539 |             if has_key(attrs, 'src'):
540 |                 attrs['href'] = attrs['src']
541 |                 alt = attrs.get('alt', '')
542 |                 if INLINE_LINKS:
543 |                     self.o("![")
544 |                     self.o(alt)
545 |                     self.o("]("+ attrs['href'] +")")
546 |                 else:
547 |                     i = self.previousIndex(attrs)
548 |                     if i is not None:
549 |                         attrs = self.a[i]
550 |                     else:
551 |                         self.acount += 1
552 |                         attrs['count'] = self.acount
553 |                         attrs['outcount'] = self.outcount
554 |                         self.a.append(attrs)
555 |                     self.o("![")
556 |                     self.o(alt)
557 |                     self.o("]["+ str(attrs['count']) +"]")
558 | 
559 |         if tag == 'dl' and start: self.p()
560 |         if tag == 'dt' and not start: self.pbr()
561 |         if tag == 'dd' and start: self.o('    ')
562 |         if tag == 'dd' and not start: self.pbr()
563 | 
564 |         if tag in ["ol", "ul"]:
565 |             # Google Docs create sub lists as top level lists
566 |             if (not self.list) and (not self.lastWasList):
567 |                 self.p()
568 |             if start:
569 |                 if options.google_doc:
570 |                     list_style = google_list_style(tag_style)
571 |                 else:
572 |                     list_style = tag
573 |                 numbering_start = list_numbering_start(attrs)
574 |                 self.list.append({'name':list_style, 'num':numbering_start})
575 |             else:
576 |                 if self.list: self.list.pop()
577 |             self.lastWasList = True
578 |         else:
579 |             self.lastWasList = False
580 | 
581 |         if tag == 'li':
582 |             self.pbr()
583 |             if start:
584 |                 if self.list: li = self.list[-1]
585 |                 else: li = {'name':'ul', 'num':0}
586 |                 if options.google_doc:
587 |                     nest_count = google_nest_count(tag_style)
588 |                 else:
589 |                     nest_count = len(self.list)
590 |                 self.o("  " * nest_count) #TODO: line up <ol><li>s > 9 correctly.
591 |                 if li['name'] == "ul": self.o(options.ul_item_mark + " ")
592 |                 elif li['name'] == "ol":
593 |                     li['num'] += 1
594 |                     self.o(str(li['num'])+". ")
595 |                 self.start = 1
596 | 
597 |         # if tag in ["table", "tr"] and start: self.p()
598 |         if tag == 'td': self.pbr()
599 |         if tag in ["table","tbody","td","tr"]:
600 |             temp = ""
601 |             if attrs:
602 |                 for (k,v) in  attrs.items():
603 |                     temp += k+"="+"\""+v+"\""+" "
604 |             if start:
605 |                 self.o("<"+tag+" "+temp+">", 0, 1)
606 |             else:
607 |                 self.o("</"+tag+">",0,1)
608 | 
609 |         if tag == "pre":
610 |             if start:
611 |                 self.startpre = 1
612 |                 self.pre = 1
613 |             else:
614 |                 self.pre = 0
615 |             self.p()
616 | 
617 |     def pbr(self):
618 |         if self.p_p == 0: self.p_p = 1
619 | 
620 |     def p(self): self.p_p = 2
621 | 
622 |     def soft_br(self):
623 |         self.pbr()
624 |         self.br_toggle = '  '
625 | 
626 |     def o(self, data, puredata=0, force=0):
627 |         if self.abbr_data is not None: self.abbr_data += data
628 | 
629 |         if not self.quiet:
630 |             if options.google_doc:
631 |                 # prevent white space immediately after 'begin emphasis' marks ('**' and '_')
632 |                 lstripped_data = data.lstrip()
633 |                 if self.drop_white_space and not (self.pre or self.code):
634 |                     data = lstripped_data
635 |                 if lstripped_data != '':
636 |                     self.drop_white_space = 0
637 | 
638 |             if puredata and not self.pre:
639 | #                data = re.sub('\s+', ' ', data)
640 |                 if data and data[0] == ' ':
641 |                     self.space = 1
642 |                     data = data[1:]
643 |             if not data and not force: return
644 | 
645 |             if self.startpre:
646 |                 #self.out(" :") #TODO: not output when already one there
647 |                 self.startpre = 0
648 | 
649 |             bq = (">" * self.blockquote)
650 |             if not (force and data and data[0] == ">") and self.blockquote: bq += " "
651 | 
652 |             if self.pre:
653 |                 bq += "    "
654 |                 data = data.replace("\n", "\n"+bq)
655 | 
656 |             if self.start:
657 |                 self.space = 0
658 |                 self.p_p = 0
659 |                 self.start = 0
660 | 
661 |             if force == 'end':
662 |                 # It's the end.
663 |                 self.p_p = 0
664 |                 self.out("\n")
665 |                 self.space = 0
666 | 
667 |             if self.p_p:
668 |                 self.out((self.br_toggle+'\n'+bq)*self.p_p)
669 |                 self.space = 0
670 |                 self.br_toggle = ''
671 | 
672 |             if self.space:
673 |                 if not self.lastWasNL: self.out(' ')
674 |                 self.space = 0
675 | 
676 |             if self.a and ((self.p_p == 2 and LINKS_EACH_PARAGRAPH) or force == "end"):
677 |                 if force == "end": self.out("\n")
678 | 
679 |                 newa = []
680 |                 for link in self.a:
681 |                     if self.outcount > link['outcount']:
682 |                         self.out("   ["+ str(link['count']) +"]: " + urlparse.urljoin(self.baseurl, link['href']))
683 |                         if has_key(link, 'title'): self.out(" ("+link['title']+")")
684 |                         self.out("\n")
685 |                     else:
686 |                         newa.append(link)
687 | 
688 |                 if self.a != newa: self.out("\n") # Don't need an extra line when nothing was done.
689 | 
690 |                 self.a = newa
691 | 
692 |             if self.abbr_list and force == "end":
693 |                 for abbr, definition in self.abbr_list.items():
694 |                     self.out("  *[" + abbr + "]: " + definition + "\n")
695 | 
696 |             self.p_p = 0
697 |             self.out(data)
698 |             self.outcount += 1
699 | 
700 |     def handle_data(self, data):
701 |         if r'\/script>' in data: self.quiet -= 1
702 | 
703 |         if self.style:
704 |             self.style_def.update(dumb_css_parser(data))
705 | 
706 |         self.o(data, 1)
707 | 
708 |     def unknown_decl(self, data): pass
709 | 
710 | def wrapwrite(text):
711 |     text = text.encode('utf-8')
712 |     try: #Python3
713 |         sys.stdout.buffer.write(text)
714 |     except AttributeError:
715 |         sys.stdout.write(text)
716 | 
717 | def html2text_file(html, out=wrapwrite, baseurl=''):
718 |     h = _html2text(out, baseurl)
719 |     h.feed(html)
720 |     h.feed("")
721 |     return h.close()
722 | 
723 | def html2text(html, baseurl=''):
724 |     return optwrap(html2text_file(html, None, baseurl))
725 | 
726 | class Storage: pass
727 | options = Storage()
728 | options.google_doc = False
729 | options.ul_item_mark = '*'
730 | 
731 | if __name__ == "__main__":
732 |     baseurl = ''
733 | 
734 |     p = optparse.OptionParser('%prog [(filename|url) [encoding]]',
735 |                               version='%prog ' + __version__)
736 |     p.add_option("-g", "--google-doc", action="store_true", dest="google_doc",
737 |         default=False, help="convert an html-exported Google Document")
738 |     p.add_option("-d", "--dash-unordered-list", action="store_true", dest="ul_style_dash",
739 |         default=False, help="use a dash rather than a star for unordered list items")
740 |     p.add_option("-b", "--body-width", dest="body_width", action="store", type="int",
741 |         default=78, help="number of characters per output line, 0 for no wrap")
742 |     p.add_option("-i", "--google-list-indent", dest="list_indent", action="store", type="int",
743 |         default=GOOGLE_LIST_INDENT, help="number of pixels Google indents nested lists")
744 |     p.add_option("-s", "--hide-strikethrough", action="store_true", dest="hide_strikethrough",
745 |         default=False, help="hide strike-through text. only relevent when -g is specified as well")
746 |     (options, args) = p.parse_args()
747 | 
748 |     # handle options
749 |     if options.ul_style_dash:
750 |         options.ul_item_mark = '-'
751 |     else:
752 |         options.ul_item_mark = '*'
753 | 
754 |     BODY_WIDTH = options.body_width
755 |     GOOGLE_LIST_INDENT = options.list_indent
756 | 
757 |     # process input
758 |     if len(args) > 0:
759 |         file_ = args[0]
760 |         encoding = None
761 |         if len(args) == 2:
762 |             encoding = args[1]
763 |         if len(args) > 2:
764 |             p.error('Too many arguments')
765 | 
766 |         if file_.startswith('http://') or file_.startswith('https://'):
767 |             baseurl = file_
768 |             j = urllib.urlopen(baseurl)
769 |             text = j.read()
770 |             if encoding is None:
771 |                 try:
772 |                     from feedparser import _getCharacterEncoding as enc
773 |                 except ImportError:
774 |                     enc = lambda x, y: ('utf-8', 1)
775 |                 encoding = enc(j.headers, text)[0]
776 |                 if encoding == 'us-ascii':
777 |                     encoding = 'utf-8'
778 |             data = text.decode(encoding)
779 | 
780 |         else:
781 |             data = open(file_, 'rb').read()
782 |             if encoding is None:
783 |                 try:
784 |                     from chardet import detect
785 |                 except ImportError:
786 |                     detect = lambda x: {'encoding': 'utf-8'}
787 |                 encoding = detect(data)['encoding']
788 |             data = data.decode(encoding)
789 |     else:
790 |         data = sys.stdin.read()
791 |     wrapwrite(html2text(data, baseurl))
792 | 
793 | 


--------------------------------------------------------------------------------
/pip_requirements.txt:
--------------------------------------------------------------------------------
1 | html5lib==1.0b1
2 | beautifulsoup4==4.2.0
3 | PyYAML==3.10
4 | html2text==3.200.3
5 | 


--------------------------------------------------------------------------------
/wordpress-xml/.gitignore:
--------------------------------------------------------------------------------
1 | *.xml


--------------------------------------------------------------------------------