', '
')[0]
73 | return [
74 | (text.unescape(url), None)
75 | for url in re.findall(r"
]* src=[\"']([^\"']+)", readerarea)
76 | if "/Banner-" not in url and "/WM-Sampingan." not in url
77 | ]
78 |
79 |
80 | class KomikcastMangaExtractor(KomikcastBase, MangaExtractor):
81 | """Extractor for manga from komikcast.com"""
82 | chapterclass = KomikcastChapterExtractor
83 | pattern = (r"(?:https?://)?(?:www\.)?komikcast\.com"
84 | r"(/(?:komik/)?[^/?&#]+)/?$")
85 | test = (
86 | ("https://komikcast.com/komik/090-eko-to-issho/", {
87 | "url": "dc798d107697d1f2309b14ca24ca9dba30c6600f",
88 | "keyword": "837a7e96867344ff59d840771c04c20dc46c0ab1",
89 | }),
90 | ("https://komikcast.com/tonari-no-kashiwagi-san/"),
91 | )
92 |
93 | def chapters(self, page):
94 | results = []
95 | data = self.metadata(page)
96 |
97 | for item in text.extract_iter(
98 | page, 'Chapter ')
100 | self.parse_chapter_string(chapter_string, data)
101 | results.append((url, data.copy()))
102 | return results
103 |
104 | @staticmethod
105 | def metadata(page):
106 | """Return a dict with general metadata"""
107 | manga , pos = text.extract(page, "" , "")
108 | genres, pos = text.extract(page, ">Genres:", "", pos)
109 | author, pos = text.extract(page, ">Author:", "", pos)
110 | mtype , pos = text.extract(page, ">Type:" , "", pos)
111 |
112 | return {
113 | "manga": text.unescape(manga[:-12]),
114 | "author": text.remove_html(author),
115 | "genres": text.split_html(genres)[::2],
116 | "type": text.remove_html(mtype),
117 | }
118 |
--------------------------------------------------------------------------------
/gallery_dl/extractor/plurk.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2019 Mike Fährmann
4 | #
5 | # This program is free software; you can redistribute it and/or modify
6 | # it under the terms of the GNU General Public License version 2 as
7 | # published by the Free Software Foundation.
8 |
9 | """Extractors for https://www.plurk.com/"""
10 |
11 | from .common import Extractor, Message
12 | from .. import text, extractor, exception
13 | import datetime
14 | import json
15 | import re
16 |
17 |
18 | class PlurkExtractor(Extractor):
19 | """Base class for plurk extractors"""
20 | category = "plurk"
21 | root = "https://www.plurk.com"
22 |
23 | def items(self):
24 | urls = self._urls_ex if self.config("comments", False) else self._urls
25 |
26 | yield Message.Version, 1
27 | with extractor.blacklist(("plurk",)):
28 | for plurk in self.plurks():
29 | for url in urls(plurk):
30 | yield Message.Queue, url, plurk
31 |
32 | def plurks(self):
33 | """Return an iterable with all relevant 'plurk' objects"""
34 |
35 | @staticmethod
36 | def _urls(obj):
37 | """Extract URLs from a 'plurk' object"""
38 | return text.extract_iter(obj["content"], ' href="', '"')
39 |
40 | def _urls_ex(self, plurk):
41 | """Extract URLs from a 'plurk' and its comments"""
42 | yield from self._urls(plurk)
43 | for comment in self._comments(plurk):
44 | yield from self._urls(comment)
45 |
46 | def _comments(self, plurk):
47 | """Return an iterable with a 'plurk's comments"""
48 | url = "https://www.plurk.com/Responses/get"
49 | data = {"plurk_id": plurk["id"], "count": "200"}
50 |
51 | while True:
52 | info = self.request(url, "POST", data=data).json()
53 | yield from info["responses"]
54 | if not info["has_newer"]:
55 | return
56 | data["from_response_id"] = info["responses"][-1]["id"]
57 |
58 | @staticmethod
59 | def _load(data):
60 | if not data:
61 | raise exception.NotFoundError("user")
62 | return json.loads(re.sub(r"new Date\(([^)]+)\)", r"\1", data))
63 |
64 |
65 | class PlurkTimelineExtractor(PlurkExtractor):
66 | """Extractor for URLs from all posts in a Plurk timeline"""
67 | subcategory = "timeline"
68 | pattern = r"(?:https?://)?(?:www\.)?plurk\.com/(?!p/)(\w+)/?(?:$|[?&#])"
69 | test = ("https://www.plurk.com/plurkapi", {
70 | "pattern": r"https?://.+",
71 | "count": ">= 23"
72 | })
73 |
74 | def __init__(self, match):
75 | PlurkExtractor.__init__(self, match)
76 | self.user = match.group(1)
77 |
78 | def plurks(self):
79 | url = "{}/{}".format(self.root, self.user)
80 | page = self.request(url).text
81 | user_id, pos = text.extract(page, '"user_id":', ',')
82 | plurks = self._load(text.extract(page, "_PLURKS = ", ";\n", pos)[0])
83 |
84 | url = "https://www.plurk.com/TimeLine/getPlurks"
85 | data = {"user_id": user_id.strip()}
86 | headers = {"Referer": url, "X-Requested-With": "XMLHttpRequest"}
87 |
88 | while plurks:
89 | yield from plurks
90 |
91 | offset = datetime.datetime.strptime(
92 | plurks[-1]["posted"], "%a, %d %b %Y %H:%M:%S %Z")
93 | data["offset"] = offset.strftime("%Y-%m-%dT%H:%M:%S.000Z")
94 | response = self.request(url, "POST", headers=headers, data=data)
95 | plurks = response.json()["plurks"]
96 |
97 |
98 | class PlurkPostExtractor(PlurkExtractor):
99 | """Extractor for URLs from a Plurk post"""
100 | subcategory = "post"
101 | pattern = r"(?:https?://)?(?:www\.)?plurk\.com/p/(\w+)"
102 | test = (
103 | ("https://www.plurk.com/p/i701j1", {
104 | "url": "2115f208564591b8748525c2807a84596aaaaa5f",
105 | "count": 3,
106 | }),
107 | ("https://www.plurk.com/p/i701j1", {
108 | "options": (("comments", True),),
109 | "count": ">= 210",
110 | }),
111 | )
112 |
113 | def __init__(self, match):
114 | PlurkExtractor.__init__(self, match)
115 | self.plurk_id = match.group(1)
116 |
117 | def plurks(self):
118 | url = "{}/p/{}".format(self.root, self.plurk_id)
119 | page = self.request(url).text
120 | user, pos = text.extract(page, " GLOBAL = ", "\n")
121 | data, pos = text.extract(page, "plurk = ", ";\n", pos)
122 |
123 | data = self._load(data)
124 | data["user"] = self._load(user)["page_user"]
125 | return (data,)
126 |
--------------------------------------------------------------------------------
/gallery_dl/extractor/sankakucomplex.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2019 Mike Fährmann
4 | #
5 | # This program is free software; you can redistribute it and/or modify
6 | # it under the terms of the GNU General Public License version 2 as
7 | # published by the Free Software Foundation.
8 |
9 | """Extractors for https://www.sankakucomplex.com/"""
10 |
11 | from .common import Extractor, Message
12 | from .. import text
13 | import re
14 |
15 |
16 | class SankakucomplexExtractor(Extractor):
17 | """Base class for sankakucomplex extractors"""
18 | category = "sankakucomplex"
19 | root = "https://www.sankakucomplex.com"
20 |
21 | def __init__(self, match):
22 | Extractor.__init__(self, match)
23 | self.path = match.group(1)
24 |
25 |
26 | class SankakucomplexArticleExtractor(SankakucomplexExtractor):
27 | """Extractor for articles on www.sankakucomplex.com"""
28 | subcategory = "article"
29 | directory_fmt = ("{category}", "{date:%Y-%m-%d} {title}")
30 | filename_fmt = "{filename}.{extension}"
31 | archive_fmt = "{date:%Y%m%d}_{filename}"
32 | pattern = (r"(?:https?://)?www\.sankakucomplex\.com"
33 | r"/(\d{4}/\d\d/\d\d/[^/?&#]+)")
34 | test = (
35 | ("https://www.sankakucomplex.com/2019/05/11/twitter-cosplayers", {
36 | "url": "4a9ecc5ae917fbce469280da5b6a482510cae84d",
37 | "keyword": "bfe08310e7d9a572f568f6900e0ed0eb295aa2b3",
38 | }),
39 | ("https://www.sankakucomplex.com/2009/12/01/sexy-goddesses-of-2ch", {
40 | "url": "a1e249173fd6c899a8134fcfbd9c925588a63f7c",
41 | "keyword": "e78fcc23c2711befc0969a45ea5082a29efccf68",
42 | }),
43 | )
44 |
45 | def items(self):
46 | url = "{}/{}/?pg=X".format(self.root, self.path)
47 | extr = text.extract_from(self.request(url).text)
48 | data = {
49 | "title" : text.unescape(
50 | extr('property="og:title" content="', '"')),
51 | "description": text.unescape(
52 | extr('property="og:description" content="', '"')),
53 | "date" : text.parse_datetime(
54 | extr('property="article:published_time" content="', '"')),
55 | }
56 | imgs = self.images(extr)
57 | data["count"] = len(imgs)
58 | data["tags"] = text.split_html(extr('="meta-tags">', ''))[::2]
59 |
60 | yield Message.Version, 1
61 | yield Message.Directory, data
62 | for img in imgs:
63 | img.update(data)
64 | yield Message.Url, img["url"], img
65 |
66 | def images(self, extr):
67 | num = 0
68 | imgs = []
69 | urls = set()
70 | orig = re.compile(r"-\d+x\d+\.")
71 |
72 | extr('
', '')
73 | while True:
74 | url = extr('data-lazy-src="', '"')
75 | if not url:
76 | return imgs
77 | if url in urls:
78 | continue
79 | if url[0] == "/":
80 | url = text.urljoin(self.root, url)
81 | url = orig.sub(".", url)
82 | num += 1
83 | imgs.append(text.nameext_from_url(url, {
84 | "url" : url,
85 | "num" : num,
86 | }))
87 | urls.add(url)
88 |
89 |
90 | class SankakucomplexTagExtractor(SankakucomplexExtractor):
91 | """Extractor for sankakucomplex blog articles by tag or author"""
92 | subcategory = "tag"
93 | pattern = (r"(?:https?://)?www\.sankakucomplex\.com"
94 | r"/((?:tag|category|author)/[^/&?#]+)")
95 | test = (
96 | ("https://www.sankakucomplex.com/tag/cosplay/", {
97 | "range": "1-50",
98 | "count": 50,
99 | "pattern": SankakucomplexArticleExtractor.pattern,
100 | }),
101 | ("https://www.sankakucomplex.com/category/anime/"),
102 | ("https://www.sankakucomplex.com/author/rift/page/5/"),
103 | )
104 |
105 | def items(self):
106 | pnum = 1
107 | last = None
108 | data = {"_extractor": SankakucomplexArticleExtractor}
109 |
110 | yield Message.Version, 1
111 | while True:
112 | url = "{}/{}/page/{}/".format(self.root, self.path, pnum)
113 | response = self.request(url, fatal=False)
114 | if response.status_code >= 400:
115 | return
116 | for url in text.extract_iter(response.text, 'data-direct="', '"'):
117 | if url != last:
118 | last = url
119 | yield Message.Queue, url, data
120 | pnum += 1
121 |
--------------------------------------------------------------------------------
/test/test_cookies.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | # Copyright 2017 Mike Fährmann
5 | #
6 | # This program is free software; you can redistribute it and/or modify
7 | # it under the terms of the GNU General Public License version 2 as
8 | # published by the Free Software Foundation.
9 |
10 | import unittest
11 | from unittest import mock
12 |
13 | import logging
14 | import tempfile
15 | import http.cookiejar
16 | from os.path import join
17 |
18 | import gallery_dl.config as config
19 | import gallery_dl.extractor as extractor
20 |
21 | CKEY = ("cookies",)
22 |
23 |
24 | class TestCookiejar(unittest.TestCase):
25 |
26 | @classmethod
27 | def setUpClass(cls):
28 | cls.path = tempfile.TemporaryDirectory()
29 |
30 | cls.cookiefile = join(cls.path.name, "cookies.txt")
31 | with open(cls.cookiefile, "w") as file:
32 | file.write("""# HTTP Cookie File
33 | .example.org\tTRUE\t/\tFALSE\t253402210800\tNAME\tVALUE
34 | """)
35 |
36 | cls.invalid_cookiefile = join(cls.path.name, "invalid.txt")
37 | with open(cls.invalid_cookiefile, "w") as file:
38 | file.write("""# asd
39 | .example.org\tTRUE\t/\tFALSE\t253402210800\tNAME\tVALUE
40 | """)
41 |
42 | @classmethod
43 | def tearDownClass(cls):
44 | cls.path.cleanup()
45 | config.clear()
46 |
47 | def test_cookiefile(self):
48 | config.set(CKEY, self.cookiefile)
49 |
50 | cookies = extractor.find("test:").session.cookies
51 | self.assertEqual(len(cookies), 1)
52 |
53 | cookie = next(iter(cookies))
54 | self.assertEqual(cookie.domain, ".example.org")
55 | self.assertEqual(cookie.path , "/")
56 | self.assertEqual(cookie.name , "NAME")
57 | self.assertEqual(cookie.value , "VALUE")
58 |
59 | def test_invalid_cookiefile(self):
60 | self._test_warning(self.invalid_cookiefile, http.cookiejar.LoadError)
61 |
62 | def test_invalid_filename(self):
63 | self._test_warning(join(self.path.name, "nothing"), FileNotFoundError)
64 |
65 | def _test_warning(self, filename, exc):
66 | config.set(CKEY, filename)
67 | log = logging.getLogger("test")
68 | with mock.patch.object(log, "warning") as mock_warning:
69 | cookies = extractor.find("test:").session.cookies
70 | self.assertEqual(len(cookies), 0)
71 | self.assertEqual(mock_warning.call_count, 1)
72 | self.assertEqual(mock_warning.call_args[0][0], "cookies: %s")
73 | self.assertIsInstance(mock_warning.call_args[0][1], exc)
74 |
75 |
76 | class TestCookiedict(unittest.TestCase):
77 |
78 | def setUp(self):
79 | self.cdict = {"NAME1": "VALUE1", "NAME2": "VALUE2"}
80 | config.set(CKEY, self.cdict)
81 |
82 | def tearDown(self):
83 | config.clear()
84 |
85 | def test_dict(self):
86 | cookies = extractor.find("test:").session.cookies
87 | self.assertEqual(len(cookies), len(self.cdict))
88 | self.assertEqual(sorted(cookies.keys()), sorted(self.cdict.keys()))
89 | self.assertEqual(sorted(cookies.values()), sorted(self.cdict.values()))
90 |
91 | def test_domain(self):
92 | for category in ["exhentai", "nijie", "sankaku", "seiga"]:
93 | extr = _get_extractor(category)
94 | cookies = extr.session.cookies
95 | for key in self.cdict:
96 | self.assertTrue(key in cookies)
97 | for c in cookies:
98 | self.assertEqual(c.domain, extr.cookiedomain)
99 |
100 |
101 | class TestCookieLogin(unittest.TestCase):
102 |
103 | def tearDown(self):
104 | config.clear()
105 |
106 | def test_cookie_login(self):
107 | extr_cookies = {
108 | "exhentai": ("ipb_member_id", "ipb_pass_hash"),
109 | "nijie" : ("nemail", "nlogin"),
110 | "sankaku" : ("login", "pass_hash"),
111 | "seiga" : ("user_session",),
112 | }
113 | for category, cookienames in extr_cookies.items():
114 | cookies = {name: "value" for name in cookienames}
115 | config.set(CKEY, cookies)
116 | extr = _get_extractor(category)
117 | with mock.patch.object(extr, "_login_impl") as mock_login:
118 | extr.login()
119 | mock_login.assert_not_called()
120 |
121 |
122 | def _get_extractor(category):
123 | for extr in extractor.extractors():
124 | if extr.category == category and hasattr(extr, "_login_impl"):
125 | url = next(extr._get_tests())[0]
126 | return extr.from_url(url)
127 |
128 |
129 | if __name__ == "__main__":
130 | unittest.main()
131 |
--------------------------------------------------------------------------------
/gallery_dl/oauth.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2018 Mike Fährmann
4 | #
5 | # This program is free software; you can redistribute it and/or modify
6 | # it under the terms of the GNU General Public License version 2 as
7 | # published by the Free Software Foundation.
8 |
9 | """OAuth helper functions and classes"""
10 |
11 | import hmac
12 | import time
13 | import base64
14 | import random
15 | import string
16 | import hashlib
17 | import urllib.parse
18 |
19 | import requests
20 | import requests.auth
21 |
22 | from . import text
23 |
24 |
25 | def nonce(size, alphabet=string.ascii_letters):
26 | """Generate a nonce value with 'size' characters"""
27 | return "".join(random.choice(alphabet) for _ in range(size))
28 |
29 |
30 | def quote(value, quote=urllib.parse.quote):
31 | """Quote 'value' according to the OAuth1.0 standard"""
32 | return quote(value, "~")
33 |
34 |
35 | def concat(*args):
36 | """Concatenate 'args' as expected by OAuth1.0"""
37 | return "&".join(quote(item) for item in args)
38 |
39 |
40 | class OAuth1Session(requests.Session):
41 | """Extension to requests.Session to support OAuth 1.0"""
42 |
43 | def __init__(self, consumer_key, consumer_secret,
44 | token=None, token_secret=None):
45 |
46 | requests.Session.__init__(self)
47 | self.auth = OAuth1Client(
48 | consumer_key, consumer_secret,
49 | token, token_secret,
50 | )
51 |
52 | def rebuild_auth(self, prepared_request, response):
53 | if "Authorization" in prepared_request.headers:
54 | del prepared_request.headers["Authorization"]
55 | prepared_request.prepare_auth(self.auth)
56 |
57 |
58 | class OAuth1Client(requests.auth.AuthBase):
59 | """OAuth1.0a authentication"""
60 |
61 | def __init__(self, consumer_key, consumer_secret,
62 | token=None, token_secret=None):
63 |
64 | self.consumer_key = consumer_key
65 | self.consumer_secret = consumer_secret
66 | self.token = token
67 | self.token_secret = token_secret
68 |
69 | def __call__(self, request):
70 | oauth_params = [
71 | ("oauth_consumer_key", self.consumer_key),
72 | ("oauth_nonce", nonce(16)),
73 | ("oauth_signature_method", "HMAC-SHA1"),
74 | ("oauth_timestamp", str(int(time.time()))),
75 | ("oauth_version", "1.0"),
76 | ]
77 | if self.token:
78 | oauth_params.append(("oauth_token", self.token))
79 |
80 | signature = self.generate_signature(request, oauth_params)
81 | oauth_params.append(("oauth_signature", signature))
82 |
83 | request.headers["Authorization"] = "OAuth " + ",".join(
84 | key + '="' + value + '"' for key, value in oauth_params)
85 |
86 | return request
87 |
88 | def generate_signature(self, request, params):
89 | """Generate 'oauth_signature' value"""
90 | url, _, query = request.url.partition("?")
91 |
92 | params = params.copy()
93 | for key, value in text.parse_query(query).items():
94 | params.append((quote(key), quote(value)))
95 | params.sort()
96 | query = "&".join("=".join(item) for item in params)
97 |
98 | message = concat(request.method, url, query).encode()
99 | key = concat(self.consumer_secret, self.token_secret or "").encode()
100 | signature = hmac.new(key, message, hashlib.sha1).digest()
101 |
102 | return quote(base64.b64encode(signature).decode())
103 |
104 |
105 | class OAuth1API():
106 | """Base class for OAuth1.0 based API interfaces"""
107 | API_KEY = None
108 | API_SECRET = None
109 |
110 | def __init__(self, extractor):
111 | self.log = extractor.log
112 | self.extractor = extractor
113 |
114 | api_key = extractor.config("api-key", self.API_KEY)
115 | api_secret = extractor.config("api-secret", self.API_SECRET)
116 | token = extractor.config("access-token")
117 | token_secret = extractor.config("access-token-secret")
118 |
119 | if api_key and api_secret and token and token_secret:
120 | self.log.debug("Using OAuth1.0 authentication")
121 | self.session = OAuth1Session(
122 | api_key, api_secret, token, token_secret)
123 | self.api_key = None
124 | else:
125 | self.log.debug("Using api_key authentication")
126 | self.session = extractor.session
127 | self.api_key = api_key
128 |
129 | def request(self, url, method="GET", **kwargs):
130 | kwargs["fatal"] = None
131 | kwargs["session"] = self.session
132 | return self.extractor.request(url, method, **kwargs)
133 |
--------------------------------------------------------------------------------
/gallery_dl/extractor/shopify.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2019 Mike Fährmann
4 | #
5 | # This program is free software; you can redistribute it and/or modify
6 | # it under the terms of the GNU General Public License version 2 as
7 | # published by the Free Software Foundation.
8 |
9 | """Extractors for Shopify instances"""
10 |
11 | from .common import Extractor, Message, SharedConfigMixin, generate_extractors
12 | from .. import text
13 | import re
14 |
15 |
16 | class ShopifyExtractor(SharedConfigMixin, Extractor):
17 | """Base class for Shopify extractors"""
18 | basecategory = "shopify"
19 | filename_fmt = "{product[title]}_{num:>02}_{id}.{extension}"
20 | archive_fmt = "{id}"
21 |
22 | def __init__(self, match):
23 | Extractor.__init__(self, match)
24 | self.item_url = self.root + match.group(1)
25 |
26 | def request(self, url, **kwargs):
27 | kwargs["retries"] = float("inf")
28 | return Extractor.request(self, url, **kwargs)
29 |
30 | def items(self):
31 | data = self.metadata()
32 | yield Message.Version, 1
33 | yield Message.Directory, data
34 |
35 | headers = {"X-Requested-With": "XMLHttpRequest"}
36 | for url in self.products():
37 | response = self.request(
38 | url + ".json", headers=headers, fatal=False)
39 | if response.status_code >= 400:
40 | self.log.warning('Skipping %s ("%s: %s")',
41 | url, response.status_code, response.reason)
42 | continue
43 | product = response.json()["product"]
44 | del product["image"]
45 |
46 | for num, image in enumerate(product.pop("images"), 1):
47 | text.nameext_from_url(image["src"], image)
48 | image.update(data)
49 | image["product"] = product
50 | image["num"] = num
51 | yield Message.Url, image["src"], image
52 |
53 | def metadata(self):
54 | """Return general metadata"""
55 | return {}
56 |
57 | def products(self):
58 | """Return an iterable with all relevant product URLs"""
59 |
60 |
61 | class ShopifyCollectionExtractor(ShopifyExtractor):
62 | """Base class for collection extractors for Shopify based sites"""
63 | subcategory = "collection"
64 | directory_fmt = ("{category}", "{collection[title]}")
65 | pattern_fmt = r"(/collections/[\w-]+)/?(?:\?([^#]+))?(?:$|#)"
66 |
67 | def __init__(self, match):
68 | ShopifyExtractor.__init__(self, match)
69 | self.params = match.group(2)
70 |
71 | def metadata(self):
72 | return self.request(self.item_url + ".json").json()
73 |
74 | def products(self):
75 | params = text.parse_query(self.params)
76 | params["page"] = text.parse_int(params.get("page"), 1)
77 | search_re = re.compile(r"/collections/[\w-]+/products/[\w-]+")
78 |
79 | while True:
80 | page = self.request(self.item_url, params=params).text
81 | urls = search_re.findall(page)
82 | last = None
83 |
84 | if not urls:
85 | return
86 | for path in urls:
87 | if last == path:
88 | continue
89 | last = path
90 | yield self.root + path
91 | params["page"] += 1
92 |
93 |
94 | class ShopifyProductExtractor(ShopifyExtractor):
95 | """Base class for product extractors for Shopify based sites"""
96 | subcategory = "product"
97 | directory_fmt = ("{category}", "Products")
98 | pattern_fmt = r"((?:/collections/[\w-]+)?/products/[\w-]+)"
99 |
100 | def products(self):
101 | return (self.item_url,)
102 |
103 |
104 | EXTRACTORS = {
105 | "fashionnova": {
106 | "root": "https://www.fashionnova.com",
107 | "pattern": r"(?:www\.)?fashionnova\.com",
108 | "test-product": (
109 | ("https://www.fashionnova.com/products/essential-slide-red", {
110 | "pattern": r"https?://cdn\d*\.shopify.com/",
111 | "count": 3,
112 | }),
113 | ("https://www.fashionnova.com/collections/flats/products/name"),
114 | ),
115 | "test-collection": (
116 | ("https://www.fashionnova.com/collections/mini-dresses", {
117 | "range": "1-20",
118 | "count": 20,
119 | }),
120 | ("https://www.fashionnova.com/collections/mini-dresses/?page=1"),
121 | ("https://www.fashionnova.com/collections/mini-dresses#1"),
122 | ),
123 |
124 | },
125 | }
126 |
127 | generate_extractors(EXTRACTORS, globals(), (
128 | ShopifyProductExtractor,
129 | ShopifyCollectionExtractor,
130 | ))
131 |
--------------------------------------------------------------------------------