]*>([^<]+)', str(chapter_select))
63 | for option in options:
64 | story.add(Chapter(title=option[1], contents=self._chapter(base_url + option[0] + suffix), date=False))
65 |
66 | # fix up the dates
67 | story[-1].date = updated
68 | story[0].date = published
69 | else:
70 | story.add(Chapter(title=story.title, contents=self._chapter(url), date=published))
71 |
72 | self._finalize(story)
73 |
74 | return story
75 |
76 | def _chapter(self, url):
77 | logger.info("Fetching chapter @ %s", url)
78 | soup, base = self._soup(url)
79 |
80 | content = soup.find(id="content_wrapper_inner")
81 | if not content:
82 | raise SiteException("No chapter content")
83 |
84 | text = content.find(id="storytext")
85 | if not text:
86 | raise SiteException("No chapter content")
87 |
88 | # clean up some invalid xhtml attributes
89 | # TODO: be more selective about this somehow
90 | try:
91 | for tag in text.find_all(True):
92 | tag.attrs.clear()
93 | except Exception:
94 | logger.exception("Trouble cleaning attributes")
95 |
96 | self._clean(text, base)
97 |
98 | return text.prettify()
99 |
100 | def _soup(self, url, *args, **kwargs):
101 | if self._cloudflared:
102 | fallback = f"https://archive.org/wayback/available?url={urllib.parse.quote(url)}"
103 | try:
104 | response = self.session.get(fallback)
105 | wayback = response.json()
106 | closest = wayback['archived_snapshots']['closest']['url']
107 | return super()._soup(closest, *args, delay=1, **kwargs)
108 | except Exception:
109 | self.session.cache.delete_url(fallback)
110 | raise CloudflareException("Couldn't fetch, presumably because of Cloudflare protection, and falling back to archive.org failed; if some chapters were succeeding, try again?", url, fallback)
111 | try:
112 | return super()._soup(self, url, *args, **kwargs)
113 | except CloudflareException:
114 | self._cloudflared = True
115 | return self._soup(url, *args, **kwargs)
116 |
117 |
118 | @register
119 | class FictionPress(FanFictionNet):
120 | @staticmethod
121 | def matches(url):
122 | # e.g. https://www.fictionpress.com/s/2961893/1/Mother-of-Learning
123 | match = re.match(r'^https?://(?:www|m)\.fictionpress\.com/s/(\d+)/?.*', url)
124 | if match:
125 | return 'https://www.fictionpress.com/s/' + match.group(1) + '/'
126 |
--------------------------------------------------------------------------------
/sites/fictionlive.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import logging
4 | import itertools
5 | import datetime
6 | import re
7 | from . import register, Site, Section, Chapter
8 |
9 | logger = logging.getLogger(__name__)
10 |
11 |
12 | @register
13 | class FictionLive(Site):
14 | """fiction.live: it's... mostly smut, I think? Terrible smut. But, hey, I had a rec to follow."""
15 | @staticmethod
16 | def matches(url):
17 | # e.g. https://fiction.live/stories/Descendant-of-a-Demon-Lord/SBBA49fQavNQMWxFT
18 | match = re.match(r'^(https?://fiction\.live/(?:stories|Sci-fi)/[^\/]+/[0-9a-zA-Z\-]+)/?.*', url)
19 | if match:
20 | return match.group(1)
21 |
22 | def extract(self, url):
23 | workid = re.match(r'^https?://fiction\.live/(?:stories|Sci-fi)/[^\/]+/([0-9a-zA-Z\-]+)/?.*', url).group(1)
24 |
25 | response = self.session.get(f'https://fiction.live/api/node/{workid}').json()
26 |
27 | story = Section(
28 | title=response['t'],
29 | author=response['u'][0]['n'],
30 | # Could normalize the URL here from the returns, but I'd have to
31 | # go look up how they handle special characters in titles...
32 | url=url
33 | )
34 | # There's a summary (or similar) in `d` and `b`, if I want to use that later.
35 |
36 | # TODO: extract these #special ones and send them off to an endnotes section?
37 | chapters = ({'ct': 0},) + tuple(c for c in response['bm'] if not c['title'].startswith('#special')) + ({'ct': 9999999999999999},)
38 |
39 | for prevc, currc, nextc in contextiterate(chapters):
40 | # `id`, `title`, `ct`, `isFirst`
41 | # https://fiction.live/api/anonkun/chapters/SBBA49fQavNQMWxFT/0/1448245168594
42 | # https://fiction.live/api/anonkun/chapters/SBBA49fQavNQMWxFT/1449266444062/1449615394752
43 | # https://fiction.live/api/anonkun/chapters/SBBA49fQavNQMWxFT/1502823848216/9999999999999998
44 | # i.e. format is [current timestamp] / [next timestamp - 1]
45 | chapter_url = f'https://fiction.live/api/anonkun/chapters/{workid}/{currc["ct"]}/{nextc["ct"] - 1}'
46 | logger.info("Extracting chapter \"%s\" @ %s", currc['title'], chapter_url)
47 | data = self.session.get(chapter_url).json()
48 | html = []
49 |
50 | updated = currc['ct']
51 | for segment in (d for d in data if not d.get('t', '').startswith('#special')):
52 | updated = max(updated, segment['ct'])
53 | # TODO: work out if this is actually enough types handled
54 | # There's at least also a reader post type, which mostly seems to be used for die rolls.
55 | try:
56 | if segment['nt'] == 'chapter':
57 | html.extend(('', segment['b'].replace('
', '
'), '
'))
58 | elif segment['nt'] == 'choice':
59 | if 'votes' not in segment:
60 | # Somehow, sometime, we end up with a choice without votes (or choices)
61 | continue
62 | votes = {}
63 | for vote in segment['votes']:
64 | votechoices = segment['votes'][vote]
65 | if isinstance(votechoices, str):
66 | # This caused issue #30, where for some reason one
67 | # choice on a story was a string rather than an
68 | # index into the choices array.
69 | continue
70 | if isinstance(votechoices, int):
71 | votechoices = (votechoices,)
72 | for choice in votechoices:
73 | if int(choice) < len(segment['choices']):
74 | # sometimes someone has voted for a presumably-deleted choice
75 | choice = segment['choices'][int(choice)]
76 | votes[choice] = votes.get(choice, 0) + 1
77 | choices = [(votes[v], v) for v in votes]
78 | choices.sort(reverse=True)
79 | html.append('
')
80 | for votecount, choice in choices:
81 | html.append(f'- {choice}: {votecount}
')
82 | html.append('
')
83 | elif segment['nt'] == 'readerPost':
84 | pass
85 | else:
86 | logger.info("Skipped chapter-segment of unhandled type: %s", segment['nt'])
87 | except Exception as e:
88 | logger.error("Skipped chapter-segment due to parsing error", exc_info=e)
89 |
90 | story.add(Chapter(
91 | title=currc['title'],
92 | contents='\n'.join(html),
93 | date=datetime.datetime.fromtimestamp(updated / 1000.0)
94 | ))
95 |
96 | self._finalize(story)
97 |
98 | return story
99 |
100 |
101 | # Stolen from the itertools docs
102 | def contextiterate(iterable):
103 | "s -> (s0,s1), (s1,s2), (s2, s3), ..."
104 | a, b, c = itertools.tee(iterable, 3)
105 | next(b, None)
106 | next(c, None)
107 | next(c, None)
108 | return zip(a, b, c)
109 |
--------------------------------------------------------------------------------
/sites/royalroad.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import http.client
4 | import logging
5 | import datetime
6 | import re
7 | from . import register, Site, Section, Chapter, SiteSpecificOption
8 |
9 | logger = logging.getLogger(__name__)
10 |
11 |
12 | @register
13 | class RoyalRoad(Site):
14 | domain = r'royalroad'
15 |
16 | @staticmethod
17 | def get_site_specific_option_defs():
18 | return Site.get_site_specific_option_defs() + [
19 | SiteSpecificOption(
20 | 'offset',
21 | '--offset',
22 | type=int,
23 | help="The chapter index to start in the chapter marks."
24 | ),
25 | SiteSpecificOption(
26 | 'limit',
27 | '--limit',
28 | type=int,
29 | help="The chapter to end at at in the chapter marks."
30 | ),
31 | ]
32 |
33 | """Royal Road: a place where people write novels, mostly seeming to be light-novel in tone."""
34 | @classmethod
35 | def matches(cls, url):
36 | # e.g. https://royalroad.com/fiction/6752/lament-of-the-fallen
37 | match = re.match(r'^(https?://(?:www\.)?%s\.com/fiction/\d+)/?.*' % cls.domain, url)
38 | if match:
39 | return match.group(1) + '/'
40 |
41 | def extract(self, url):
42 | workid = re.match(r'^https?://(?:www\.)?%s\.com/fiction/(\d+)/?.*' % self.domain, url).group(1)
43 | soup, base = self._soup(f'https://www.{self.domain}.com/fiction/{workid}')
44 | # should have gotten redirected, for a valid title
45 |
46 | original_maxheaders = http.client._MAXHEADERS
47 | http.client._MAXHEADERS = 1000
48 |
49 | story = Section(
50 | title=soup.find('h1').string.strip(),
51 | author=soup.find('meta', property='books:author').get('content').strip(),
52 | url=soup.find('meta', property='og:url').get('content').strip(),
53 | cover_url=self._join_url(base, soup.find('img', class_='thumbnail')['src']),
54 | summary=str(soup.find('div', class_='description')).strip(),
55 | tags=[tag.get_text().strip() for tag in soup.select('span.tags a.fiction-tag')]
56 | )
57 |
58 | for index, chapter in enumerate(soup.select('#chapters tbody tr[data-url]')):
59 | if self.options['offset'] and index < self.options['offset']:
60 | continue
61 | if self.options['limit'] and index >= self.options['limit']:
62 | continue
63 | chapter_url = str(self._join_url(story.url, str(chapter.get('data-url'))))
64 |
65 | contents, updated = self._chapter(chapter_url, len(story) + 1)
66 |
67 | story.add(Chapter(title=chapter.find('a', href=True).string.strip(), contents=contents, date=updated))
68 |
69 | http.client._MAXHEADERS = original_maxheaders
70 |
71 | self._finalize(story)
72 |
73 | return story
74 |
75 | def _chapter(self, url, chapterid):
76 | logger.info("Extracting chapter @ %s", url)
77 | soup, base = self._soup(url)
78 | content = soup.find('div', class_='chapter-content')
79 |
80 | self._clean(content, full_page=soup, base=base)
81 | self._clean_spoilers(content, chapterid)
82 |
83 | content = str(content)
84 |
85 | author_note = soup.find_all('div', class_='author-note-portlet')
86 |
87 | if len(author_note) == 1:
88 | # Find the parent of chapter-content and check if the author's note is the first child div
89 | if 'author-note-portlet' in soup.find('div', class_='chapter-content').parent.find('div')['class']:
90 | content = str(author_note[0]) + '
' + content
91 | else: # The author note must be after the chapter content
92 | content = content + '
' + str(author_note[0])
93 | elif len(author_note) == 2:
94 | content = str(author_note[0]) + '
' + content + '
' + str(author_note[1])
95 |
96 | updated = datetime.datetime.fromtimestamp(
97 | int(soup.find(class_="profile-info").find('time').get('unixtime'))
98 | )
99 |
100 | return content, updated
101 |
102 | def _clean(self, contents, full_page, base=False):
103 | contents = super()._clean(contents, base=base)
104 |
105 | # Royalroad has started inserting "this was stolen" notices into its
106 | # HTML, and hiding them with CSS. Currently the CSS is very easy to
107 | # find, so do so and filter them out.
108 | for style in full_page.find_all('style'):
109 | if m := re.match(r'\s*\.(\w+)\s*{[^}]*display:\s*none;[^}]*}', style.string):
110 | for warning in contents.find_all(class_=m.group(1)):
111 | warning.decompose()
112 |
113 | return contents
114 |
115 | def _clean_spoilers(self, content, chapterid):
116 | # Spoilers to footnotes
117 | for spoiler in content.find_all(class_=('spoiler-new')):
118 | spoiler_title = spoiler.get('data-caption')
119 | new_spoiler = self._new_tag('div', class_="leech-spoiler")
120 | if self.options['spoilers'] == 'skip':
121 | new_spoiler.append(spoiler_title and f'[SPOILER: {spoiler_title}]' or '[SPOILER]')
122 | elif self.options['spoilers'] == 'inline':
123 | if spoiler_title:
124 | new_spoiler.append(f"{spoiler_title}: ")
125 | new_spoiler.append(spoiler)
126 | else:
127 | link = self._footnote(spoiler, chapterid)
128 | if spoiler_title:
129 | link.string = spoiler_title
130 | new_spoiler.append(link)
131 | spoiler.replace_with(new_spoiler)
132 |
133 |
134 | @register
135 | class RoyalRoadL(RoyalRoad):
136 | domain = 'royalroadl'
137 |
--------------------------------------------------------------------------------
/sites/stash.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import logging
4 | import datetime
5 | import re
6 | from . import register, Site, SiteException, Section, Chapter
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | @register
12 | class Stash(Site):
13 | @staticmethod
14 | def matches(url):
15 | # Need a stack page
16 | match = re.match(r'^(https?://sta\.sh/2.+)/?.*', url)
17 | if match:
18 | return match.group(1) + '/'
19 |
20 | def extract(self, url):
21 | soup, base = self._soup(url)
22 | content = soup.find(id="stash-body")
23 | if not content:
24 | return
25 |
26 | # metadata = content.find(id='profile_top')
27 | story = Section(
28 | title=str(soup.find(class_="stash-folder-name").h2.string),
29 | author=str(soup.find('span', class_="oh-stashlogo-name").string).rstrip("'s"),
30 | url=url
31 | )
32 |
33 | thumbs = content.select(".stash-folder-stream .thumb")
34 | if not thumbs:
35 | return
36 | for thumb in thumbs:
37 | try:
38 | if thumb['href'] != '#':
39 | story.add(self._chapter(thumb['href']))
40 | except Exception:
41 | logger.exception("Couldn't extract chapters from thumbs")
42 |
43 | self._finalize(story)
44 |
45 | return story
46 |
47 | def _chapter(self, url):
48 | logger.info("Fetching chapter @ %s", url)
49 | soup, base = self._soup(url)
50 |
51 | content = soup.find(class_="journal-wrapper")
52 | if not content:
53 | raise SiteException("No content")
54 |
55 | title = str(content.find(class_="gr-top").find(class_='metadata').h2.a.string)
56 |
57 | text = content.find(class_="text")
58 |
59 | # clean up some invalid xhtml attributes
60 | # TODO: be more selective about this somehow
61 | try:
62 | for tag in text.find_all(True):
63 | tag.attrs = None
64 | except Exception as e:
65 | raise SiteException("Trouble cleaning attributes", e)
66 |
67 | self._clean(text, base)
68 |
69 | return Chapter(title=title, contents=text.prettify(), date=self._date(soup))
70 |
71 | def _date(self, soup):
72 | maybe_date = soup.find('div', class_="dev-metainfo-details").find('span', ts=True)
73 | return datetime.datetime.fromtimestamp(int(maybe_date['ts']))
74 |
--------------------------------------------------------------------------------
/sites/wattpad.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import logging
4 | import datetime
5 | import re
6 | from . import register, Site, Section, Chapter
7 |
8 | logger = logging.getLogger(__name__)
9 |
10 |
11 | @register
12 | class Wattpad(Site):
13 | """Wattpad"""
14 | @classmethod
15 | def matches(cls, url):
16 | # e.g. https://www.wattpad.com/story/208753031-summoned-to-have-tea-with-the-demon-lord-i-guess
17 | # chapter URLs are e.g. https://www.wattpad.com/818687865-summoned-to-have-tea-with-the-demon-lord-i-guess
18 | match = re.match(r'^(https?://(?:www\.)?wattpad\.com/story/\d+)?.*', url)
19 | if match:
20 | # the story-title part is unnecessary
21 | return match.group(1)
22 |
23 | def extract(self, url):
24 | workid = re.match(r'^https?://(?:www\.)?wattpad\.com/story/(\d+)?.*', url).group(1)
25 | info = self.session.get(f"https://www.wattpad.com/api/v3/stories/{workid}").json()
26 |
27 | story = Section(
28 | title=info['title'],
29 | author=info['user']['name'],
30 | url=url,
31 | cover_url=info['cover']
32 | )
33 |
34 | for chapter in info['parts']:
35 | story.add(Chapter(
36 | title=chapter['title'],
37 | contents=self._chapter(chapter['id']),
38 | # "2020-05-03T22:14:29Z"
39 | date=datetime.datetime.fromisoformat(chapter['createDate'].rstrip('Z')) # modifyDate also?
40 | ))
41 |
42 | self._finalize(story)
43 |
44 | return story
45 |
46 | def _chapter(self, chapterid):
47 | logger.info(f"Extracting chapter @ {chapterid}")
48 | api = self.session.get(f"https://www.wattpad.com/apiv2/storytext?id={chapterid}")
49 | return '' + api.text + '
'
50 |
--------------------------------------------------------------------------------
/sites/xenforo.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import datetime
4 | import re
5 | import logging
6 | import requests_cache
7 |
8 | from . import Site, SiteException, SiteSpecificOption, Section, Chapter
9 | import mintotp
10 |
11 | logger = logging.getLogger(__name__)
12 |
13 |
14 | class XenForo(Site):
15 | """XenForo is forum software that powers a number of fiction-related forums."""
16 |
17 | domain = False
18 | index_urls = False
19 |
20 | @staticmethod
21 | def get_site_specific_option_defs():
22 | return Site.get_site_specific_option_defs() + [
23 | SiteSpecificOption(
24 | 'include_index',
25 | '--include-index/--no-include-index',
26 | default=False,
27 | help="If true, the post marked as an index will be included as a chapter."
28 | ),
29 | SiteSpecificOption(
30 | 'offset',
31 | '--offset',
32 | type=int,
33 | help="The chapter index to start in the chapter marks."
34 | ),
35 | SiteSpecificOption(
36 | 'limit',
37 | '--limit',
38 | type=int,
39 | help="The chapter to end at at in the chapter marks."
40 | ),
41 | ]
42 |
43 | @classmethod
44 | def matches(cls, url):
45 | match = re.match(r'^(https?://%s/(?:index\.php\?)?threads/[^/]*\d+/(?:\d+/)?reader)/?.*' % cls.domain, url)
46 | if match:
47 | return match.group(1)
48 | match = re.match(r'^(https?://%s/(?:index\.php\?)?threads/[^/]*\d+)/?.*' % cls.domain, url)
49 | if match:
50 | return match.group(1) + '/'
51 |
52 | def siteurl(self, path):
53 | if self.index_urls:
54 | return f'https://{self.domain}/index.php?{path}'
55 | return f'https://{self.domain}/{path}'
56 |
57 | def login(self, login_details):
58 | with requests_cache.disabled():
59 | # Can't just pass this url to _soup because I need the cookies later
60 | login = self.session.get(self.siteurl('login/'))
61 | soup, nobase = self._soup(login.text)
62 | post, action, method = self._form_data(soup.find(class_='p-body-content'))
63 | post['login'] = login_details[0]
64 | post['password'] = login_details[1]
65 | # I feel the session *should* handle this cookies bit for me. But
66 | # it doesn't. And I don't know why.
67 | result = self.session.post(
68 | self._join_url(login.url, action),
69 | data=post, cookies=login.cookies
70 | )
71 | if not result.ok:
72 | return logger.error("Failed to log in as %s", login_details[0])
73 | soup, nobase = self._soup(result.text)
74 | if twofactor := soup.find('form', action="/login/two-step"):
75 | if len(login_details) < 3:
76 | return logger.error("Failed to log in as %s; login requires 2FA secret", login_details[0])
77 | post, action, method = self._form_data(twofactor)
78 | post['code'] = mintotp.totp(login_details[2])
79 | result = self.session.post(
80 | self._join_url(login.url, action),
81 | data=post, cookies=login.cookies
82 | )
83 | if not result.ok:
84 | return logger.error("Failed to log in as %s; 2FA failed", login_details[0])
85 | logger.info("Logged in as %s", login_details[0])
86 |
87 | def extract(self, url):
88 | soup, base = self._soup(url)
89 |
90 | story = self._base_story(soup)
91 |
92 | threadmark_categories = {}
93 | # Note to self: in the source this is data-categoryId, but the parser
94 | # in bs4 lowercases tags and attributes...
95 | for cat in soup.find_all('a', attrs={'data-categoryid': True}):
96 | threadmark_categories[int(cat['data-categoryid'])] = cat['title']
97 |
98 | if url.endswith('/reader'):
99 | reader_url = url
100 | elif soup.find('a', class_='readerToggle'):
101 | reader_url = soup.find('a', class_='readerToggle').get('href')
102 | elif soup.find('div', class_='threadmarks-reader'):
103 | # Technically this is the xenforo2 bit, but :shrug:
104 | reader_url = soup.find('div', class_='threadmarks-reader').find('a').get('href')
105 | else:
106 | reader_url = False
107 |
108 | if reader_url:
109 | match = re.search(r'\d+/(\d+)/reader', reader_url)
110 | if match:
111 | cat = int(match.group(1))
112 | if cat != 1 and cat in threadmark_categories:
113 | story.title = f'{story.title} ({threadmark_categories[cat]})'
114 | idx = 0
115 | while reader_url:
116 | reader_url = self._join_url(base, reader_url)
117 | logger.info("Fetching chapters @ %s", reader_url)
118 | reader_soup, reader_base = self._soup(reader_url)
119 | posts = self._posts_from_page(reader_soup)
120 |
121 | for post in posts:
122 | idx = idx + 1
123 | if self.options['offset'] and idx < self.options['offset']:
124 | continue
125 | if self.options['limit'] and idx >= self.options['limit']:
126 | continue
127 | title = self._threadmark_title(post)
128 | logger.info("Extracting chapter \"%s\"", title)
129 |
130 | story.add(Chapter(
131 | title=title,
132 | contents=self._clean_chapter(post, len(story) + 1, base),
133 | date=self._post_date(post)
134 | ))
135 |
136 | reader_url = False
137 | if reader_soup.find('link', rel='next'):
138 | reader_url = reader_soup.find('link', rel='next').get('href')
139 | else:
140 | # TODO: Research whether reader mode is guaranteed to be enabled
141 | # when threadmarks are; if so, can delete this branch.
142 | marks = [
143 | mark for mark in self._chapter_list(url)
144 | if '/members' not in mark.get('href') and '/threadmarks' not in mark.get('href')
145 | ]
146 | marks = marks[self.options['offset']:self.options['limit']]
147 |
148 | for idx, mark in enumerate(marks, 1):
149 | href = self._join_url(base, mark.get('href'))
150 | title = str(mark.string).strip()
151 | logger.info("Fetching chapter \"%s\" @ %s", title, href)
152 | contents, post_date = self._chapter(href, idx)
153 | chapter = Chapter(title=title, contents=contents, date=post_date)
154 | story.add(chapter)
155 |
156 | self._finalize(story)
157 |
158 | return story
159 |
160 | def _base_story(self, soup):
161 | url = soup.find('meta', property='og:url').get('content')
162 | title = soup.select('div.titleBar > h1')[0]
163 | # clean out informational bits from the title
164 | for tag in title.find_all(class_='prefix'):
165 | tag.decompose()
166 | tags = [tag.get_text().strip() for tag in soup.select('div.tagBlock a.tag')]
167 | return Section(
168 | title=title.get_text().strip(),
169 | author=soup.find('p', id='pageDescription').find('a', class_='username').get_text(),
170 | url=url,
171 | tags=tags
172 | )
173 |
174 | def _posts_from_page(self, soup, postid=False):
175 | if postid:
176 | return soup.find('li', id='post-' + postid)
177 | return soup.select('#messageList > li.hasThreadmark')
178 |
179 | def _threadmark_title(self, post):
180 | # Get the title, removing "Threadmark:" which precedes it
181 | return ''.join(post.select('div.threadmarker > span.label')[0].findAll(text=True, recursive=False)).strip()
182 |
183 | def _chapter_list(self, url):
184 | try:
185 | return self._chapter_list_threadmarks(url)
186 | except SiteException as e:
187 | logger.debug("Tried threadmarks (%r)", e.args)
188 | return self._chapter_list_index(url)
189 |
190 | def _chapter_list_threadmarks(self, url):
191 | soup, base = self._soup(url)
192 |
193 | threadmarks_link = soup.find(class_="threadmarksTrigger", href=True)
194 | if not threadmarks_link:
195 | try:
196 | threadmarks_link = soup.select('.threadmarkMenus a.OverlayTrigger')[0]
197 | except IndexError:
198 | pass
199 |
200 | if not threadmarks_link:
201 | raise SiteException("No threadmarks")
202 |
203 | href = threadmarks_link.get('href')
204 | soup, base = self._soup(self._join_url(base, href))
205 |
206 | fetcher = soup.find(class_='ThreadmarkFetcher')
207 | while fetcher:
208 | # ThreadmarksPro, hiding some threadmarks. Means the API is available to do this.
209 | # Note: the fetched threadmarks can contain more placeholder elements to fetch. Ergo, loop.
210 | # Good test case: https://forums.sufficientvelocity.com/threads/ignition-mtg-multicross-planeswalker-pc.26099/threadmarks
211 | # e.g.:
212 | response = self.session.post(self.siteurl('threads/threadmarks/load-range'), data={
213 | # I did try a fetch on min/data-min+data-max, but there seems
214 | # to be an absolute limit which the API fetch won't override
215 | 'min': fetcher.get('data-range-min'),
216 | 'max': fetcher.get('data-range-max'),
217 | 'thread_id': fetcher.get('data-thread-id'),
218 | 'category_id': fetcher.get('data-category-id'),
219 | '_xfResponseType': 'json',
220 | }).json()
221 | responseSoup, nobase = self._soup(response['templateHtml'])
222 | fetcher.replace_with(responseSoup)
223 | fetcher = soup.find(class_='ThreadmarkFetcher')
224 |
225 | marks = soup.find(class_='threadmarks').select('li.primaryContent.threadmarkListItem a, li.primaryContent.threadmarkItem a')
226 | if not marks:
227 | raise SiteException("No marks on threadmarks page")
228 |
229 | return marks
230 |
231 | def _chapter_list_index(self, url):
232 | post = self._post_from_url(url)
233 | if not post:
234 | raise SiteException("Unparseable post URL", url)
235 |
236 | links = post.find('blockquote', class_='messageText').find_all('a', class_='internalLink')
237 | if not links:
238 | raise SiteException("No links in index?")
239 |
240 | if self.options['include_index']:
241 | fake_link = self._new_tag('a', href=url)
242 | fake_link.string = "Index"
243 | links.insert(0, fake_link)
244 |
245 | return links
246 |
247 | def _chapter(self, url, chapterid):
248 | post, base = self._post_from_url(url)
249 |
250 | return self._clean_chapter(post, chapterid, base), self._post_date(post)
251 |
252 | def _post_from_url(self, url):
253 | # URLs refer to specific posts, so get just that one
254 | # if no specific post referred to, get the first one
255 | match = re.search(r'posts/(\d+)/?', url)
256 | if not match:
257 | match = re.match(r'.+#post-(\d+)$', url)
258 | # could still be nothing here
259 | postid = match and match.group(1)
260 | if postid:
261 | # create a proper post-url, because threadmarks can sometimes
262 | # mess up page-wise with anchors
263 | url = self.siteurl(f'posts/{postid}/')
264 | soup, base = self._soup(url, 'lxml')
265 |
266 | if postid:
267 | return self._posts_from_page(soup, postid), base
268 |
269 | # just the first one in the thread, then
270 | return soup.find('li', class_='message'), base
271 |
272 | def _chapter_contents(self, post):
273 | return post.find('blockquote', class_='messageText')
274 |
275 | def _clean_chapter(self, post, chapterid, base):
276 | post = self._chapter_contents(post)
277 | post.name = 'div'
278 | # mostly, we want to remove colors because the Kindle is terrible at them
279 | # TODO: find a way to denote colors, because it can be relevant
280 | # TODO: at least invisitext, because outside of silly DC Lantern stuff, it's the most common
281 | for tag in post.find_all(style=True):
282 | if tag['style'] == 'color: transparent' and tag.text == 'TAB':
283 | # Some stories fake paragraph indents like this. The output
284 | # stylesheet will handle this just fine.
285 | tag.decompose()
286 | else:
287 | # There's a few things which xenforo does as styles, despite there being perfectly good tags
288 | # TODO: more robust CSS parsing? This is very whitespace dependent, if nothing else.
289 | if "font-family: 'Courier New'" in tag['style']:
290 | tag.wrap(self._new_tag('code'))
291 | if "text-decoration: strikethrough" in tag['style']:
292 | tag.wrap(self._new_tag('strike'))
293 | if "margin-left" in tag['style']:
294 | continue
295 | del tag['style']
296 | for tag in post.select('.quoteExpand, .bbCodeBlock-expandLink, .bbCodeBlock-shrinkLink'):
297 | tag.decompose()
298 | for tag in post.find_all('noscript'):
299 | # TODO: strip the noscript from these?
300 | # mostly this will be the lazyload images
301 | tag.decompose()
302 | for tag in post.select('img.lazyload[data-src]'):
303 | tag['src'] = tag['data-url']
304 | if tag['src'].startswith('proxy.php'):
305 | tag['src'] = f"{self.domain}/{tag['src']}"
306 | self._clean(post, base)
307 | self._clean_spoilers(post, chapterid)
308 | return post.prettify()
309 |
310 | def _clean_spoilers(self, post, chapterid):
311 | # spoilers don't work well, so turn them into epub footnotes
312 | for spoiler in post.find_all(class_='ToggleTriggerAnchor'):
313 | spoiler_title = spoiler.find(class_='SpoilerTitle')
314 | if self.options['skip_spoilers']:
315 | link = self._footnote(spoiler.find(class_='SpoilerTarget').extract(), chapterid)
316 | if spoiler_title:
317 | link.string = spoiler_title.get_text()
318 | else:
319 | if spoiler_title:
320 | link = f'[SPOILER: {spoiler_title.get_text()}]'
321 | else:
322 | link = '[SPOILER]'
323 | new_spoiler = self._new_tag('div', class_="leech-spoiler")
324 | new_spoiler.append(link)
325 | spoiler.replace_with(new_spoiler)
326 |
327 | def _post_date(self, post):
328 | maybe_date = post.find(class_='DateTime')
329 | if 'data-time' in maybe_date.attrs:
330 | return datetime.datetime.fromtimestamp(int(maybe_date['data-time']))
331 | if 'title' in maybe_date.attrs:
332 | # title="Feb 24, 2015 at 1:17 PM"
333 | return datetime.datetime.strptime(maybe_date['title'], "%b %d, %Y at %I:%M %p")
334 | raise SiteException("No date", maybe_date)
335 |
336 |
337 | class XenForoIndex(XenForo):
338 | @classmethod
339 | def matches(cls, url):
340 | match = re.match(r'^(https?://%s/posts/\d+)/?.*' % cls.domain, url)
341 | if match:
342 | return match.group(1) + '/'
343 |
344 | def _chapter_list(self, url):
345 | return self._chapter_list_index(url)
346 |
--------------------------------------------------------------------------------
/sites/xenforo2.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 |
3 | import datetime
4 | import logging
5 |
6 | from . import register, Section, SiteException
7 | from .xenforo import XenForo, XenForoIndex
8 |
9 | logger = logging.getLogger(__name__)
10 |
11 |
12 | class XenForo2(XenForo):
13 | def _base_story(self, soup):
14 | url = soup.find('meta', property='og:url').get('content')
15 | title = soup.select('h1.p-title-value')[0]
16 | # clean out informational bits from the title
17 | for tag in title.select('.labelLink,.label-append'):
18 | tag.decompose()
19 | tags = [tag.get_text().strip() for tag in soup.select('.tagList a.tagItem')]
20 | return Section(
21 | title=title.get_text().strip(),
22 | author=soup.find('div', class_='p-description').find('a', class_='username').get_text(),
23 | url=url,
24 | tags=tags
25 | )
26 |
27 | def _posts_from_page(self, soup, postid=False):
28 | if postid:
29 | return soup.find('article', id='js-post-' + postid)
30 | return soup.select('article.message--post')
31 |
32 | def _threadmark_title(self, post):
33 | # Get the title, removing "Threadmark:" which precedes it
34 | return post.find('span', class_='threadmarkLabel').get_text()
35 |
36 | def _chapter_contents(self, post):
37 | return post.find('div', class_='message-userContent')
38 |
39 | def _clean_spoilers(self, post, chapterid):
40 | # spoilers don't work well, so turn them into epub footnotes
41 | for spoiler in post.find_all(class_='bbCodeSpoiler'):
42 | spoiler_title = spoiler.find(class_='bbCodeSpoiler-button-title')
43 | spoiler_contents = spoiler.find(class_='bbCodeBlock-content').extract()
44 | new_spoiler = self._new_tag('div', class_="leech-spoiler")
45 | if self.options['spoilers'] == 'skip':
46 | new_spoiler.append(spoiler_title and f'[SPOILER: {spoiler_title.get_text()}]' or '[SPOILER]')
47 | elif self.options['spoilers'] == 'inline':
48 | if spoiler_title:
49 | new_spoiler.append(f"{spoiler_title.get_text()}: ")
50 | new_spoiler.append(spoiler_contents)
51 | else:
52 | link = self._footnote(spoiler_contents, chapterid)
53 | if spoiler_title:
54 | link.string = spoiler_title.get_text()
55 | new_spoiler.append(link)
56 | spoiler.replace_with(new_spoiler)
57 |
58 | def _post_date(self, post):
59 | if post.find('time'):
60 | return datetime.datetime.fromtimestamp(int(post.find('time').get('data-time')))
61 | raise SiteException("No date")
62 |
63 |
64 | @register
65 | class SpaceBattles(XenForo2):
66 | domain = 'forums.spacebattles.com'
67 |
68 |
69 | @register
70 | class SpaceBattlesIndex(SpaceBattles, XenForoIndex):
71 | _key = "SpaceBattles"
72 |
73 |
74 | @register
75 | class SufficientVelocity(XenForo2):
76 | domain = 'forums.sufficientvelocity.com'
77 |
78 |
79 | @register
80 | class TheSietch(XenForo2):
81 | domain = 'www.the-sietch.com'
82 | index_urls = True
83 |
84 |
85 | @register
86 | class QuestionableQuesting(XenForo2):
87 | domain = 'forum.questionablequesting.com'
88 |
89 |
90 | @register
91 | class QuestionableQuestingIndex(QuestionableQuesting, XenForoIndex):
92 | _key = "QuestionableQuesting"
93 |
94 |
95 | @register
96 | class AlternateHistory(XenForo2):
97 | domain = 'www.alternatehistory.com/forum'
98 |
99 |
100 | @register
101 | class AlternateHistoryIndex(AlternateHistory, XenForoIndex):
102 | _key = "AlternateHistory"
103 |
--------------------------------------------------------------------------------