├── .gitignore ├── AjaxTest.ipynb ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── etsy ├── __init__.py ├── items.py ├── middlewares.py ├── pipelines.py ├── settings.py └── spiders │ ├── __init__.py │ ├── products.json │ └── search_products.py ├── outputs ├── products.csv └── products.xlsx ├── requirements.txt └── scrapy.cfg /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Windows files 107 | *.ini 108 | .vscode/ 109 | .DS_Store -------------------------------------------------------------------------------- /AjaxTest.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import requests\n", 10 | "\n", 11 | "cookies = {\n", 12 | " 'uaid': 'sAP7vHefE6jJvrsDbGklZvrkq-ljZACCGNc5RmA6cn5ytVJpYmaKkpVSlatpanx8aUFlsY93Sbalh0G-ubFvsa6zq3GOt5KOUnxefrmSVV5pTk4tAwA.' \n", 13 | "}\n", 14 | "\n", 15 | "headers = { \n", 16 | " 'x-csrf-token': '3:1549897497:9XIRfEbtgBGEHHdyl6VW3nno_bJw:98c271b297293f176aed5dacc8e60a2de27322fc525f9489dc54a34762b333f8', \n", 17 | "}\n", 18 | "\n", 19 | "data = {\n", 20 | " 'stats_sample_rate': '',\n", 21 | " 'specs[reviews][]': 'Listzilla_ApiSpecs_Reviews',\n", 22 | " 'specs[reviews][1][listing_id]': '519048483',\n", 23 | " 'specs[reviews][1][shop_id]': '13648052',\n", 24 | " 'specs[reviews][1][render_complete]': 'true'\n", 25 | "}\n", 26 | "\n", 27 | "response = requests.post('https://www.etsy.com/api/v3/ajax/bespoke/member/neu/specs/reviews', headers=headers, cookies=cookies, data=data)\n" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 40, 33 | "metadata": {}, 34 | "outputs": [ 35 | { 36 | "name": "stdout", 37 | "output_type": "stream", 38 | "text": [ 39 | "www.etsy.com/people/0svfagrm\n", 40 | "Feb 10, 2019\n", 41 | "5\n", 42 | "He arrived on schedule, and he was very well packaged. \n", 43 | "As soon as I unwrapped him I loved him straight away. \n", 44 | "I am amazed at the quality and finish. He was just perfect. If you like Groot then you will love this baby Groot. He is small but what else would you expect he is a baby after all. Great value for money.\n", 45 | "------------------------------\n", 46 | "www.etsy.com/people/tangatama\n", 47 | "Feb 8, 2019\n", 48 | "5\n", 49 | "super figurine elle est très cool\n", 50 | "------------------------------\n", 51 | "www.etsy.com/people/dni4ltav\n", 52 | "Feb 8, 2019\n", 53 | "5\n", 54 | "Good painting and really good quality printing- very light as it is printed but so cute ☺️\n", 55 | "------------------------------\n", 56 | "www.etsy.com/people/amber2117\n", 57 | "Feb 6, 2019\n", 58 | "5\n", 59 | "This is the cutest thing I’ve ever purchased!!! I love it!!!! You can tell it’s hand painted and done very well!! No complaints at all :)\n", 60 | "------------------------------\n", 61 | "www.etsy.com/people/vhvlqmzr\n", 62 | "Feb 6, 2019\n", 63 | "5\n", 64 | "Had my hopes set high after I saw the other reviews and pics. Got the parcel in good time, also received a notification advising the item has been shipped. Got the parcel in one piece and Baby Groot is so adorable. Great art work and care taken when making him. Thank you... It lived up to my expectations 🙂👍\n", 65 | "------------------------------\n", 66 | "www.etsy.com/people/ahclegion\n", 67 | "Feb 5, 2019\n", 68 | "5\n", 69 | "\n", 70 | "------------------------------\n", 71 | "www.etsy.com/people/evelynevdlinden\n", 72 | "Feb 5, 2019\n", 73 | "5\n", 74 | "\n", 75 | "------------------------------\n", 76 | "www.etsy.com/people/bri9511\n", 77 | "Feb 4, 2019\n", 78 | "5\n", 79 | "\n", 80 | "------------------------------\n", 81 | "www.etsy.com/people/sarahshirilla\n", 82 | "Feb 1, 2019\n", 83 | "5\n", 84 | "Perfect. Exactly as described and so adorable.\n", 85 | "------------------------------\n", 86 | "www.etsy.com/people/rlnniqxf\n", 87 | "Jan 26, 2019\n", 88 | "5\n", 89 | "First time buyer of this shop. Shipped on time. Just adorable!!! Fits in Palm of your hand. Feels like I'm truly holding baby Groot. Details are so \"life like\" to his character. You won't be disappointed. Got for my daughter for Christmas. HIGHLY RECOMMEND.\n", 90 | "------------------------------\n" 91 | ] 92 | } 93 | ], 94 | "source": [ 95 | "import json\n", 96 | "from parsel import Selector\n", 97 | "\n", 98 | "j = json.loads(response.content)\n", 99 | "html = j[\"output\"][\"reviews\"]\n", 100 | "\n", 101 | "sel = Selector(text=html)\n", 102 | "\n", 103 | "reviews_list = sel.xpath('//*[@class=\"listing-page__review col-group pl-xs-0 pr-xs-0\"]')\n", 104 | "for r in reviews_list:\n", 105 | " reviewer_profile = r.xpath(\".//*[@class='display-block']/parent::*//@href\").extract_first()\n", 106 | " review_date = r.xpath(\".//*[@class='text-link-underline display-inline-block mr-xs-1']/parent::*//text()\").extract()[2].strip()\n", 107 | " reviewer_rating = r.xpath('.//input[@name=\"rating\"]/@value').extract_first()\n", 108 | " review_content = \" \".join(r.xpath('.//div[@class=\"overflow-hidden\"]//text()').extract()).strip()\n", 109 | " \n", 110 | " print('www.etsy.com'+reviewer_profile)\n", 111 | " print(review_date)\n", 112 | " print(reviewer_rating)\n", 113 | " print(review_content)\n", 114 | " print('-'*30)\n", 115 | " " 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": null, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [] 124 | } 125 | ], 126 | "metadata": { 127 | "kernelspec": { 128 | "display_name": "Python 3", 129 | "language": "python", 130 | "name": "python3" 131 | }, 132 | "language_info": { 133 | "codemirror_mode": { 134 | "name": "ipython", 135 | "version": 3 136 | }, 137 | "file_extension": ".py", 138 | "mimetype": "text/x-python", 139 | "name": "python", 140 | "nbconvert_exporter": "python", 141 | "pygments_lexer": "ipython3", 142 | "version": "3.6.5" 143 | } 144 | }, 145 | "nbformat": 4, 146 | "nbformat_minor": 2 147 | } 148 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Patrick Alves 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | openpyxl = "*" 10 | Scrapy = "==2.2" 11 | 12 | [requires] 13 | python_version = "3.8" 14 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "f5fe8feb6d5ad9fb5bf06bed592e4a498741c29eaaa015bd8cbc2f269402b573" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.8" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "attrs": { 20 | "hashes": [ 21 | "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6", 22 | "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700" 23 | ], 24 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 25 | "version": "==20.3.0" 26 | }, 27 | "automat": { 28 | "hashes": [ 29 | "sha256:7979803c74610e11ef0c0d68a2942b152df52da55336e0c9d58daf1831cbdf33", 30 | "sha256:b6feb6455337df834f6c9962d6ccf771515b7d939bca142b29c20c2376bc6111" 31 | ], 32 | "version": "==20.2.0" 33 | }, 34 | "cffi": { 35 | "hashes": [ 36 | "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e", 37 | "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d", 38 | "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a", 39 | "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec", 40 | "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362", 41 | "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668", 42 | "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c", 43 | "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b", 44 | "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06", 45 | "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698", 46 | "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2", 47 | "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c", 48 | "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7", 49 | "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009", 50 | "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03", 51 | "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b", 52 | "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909", 53 | "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53", 54 | "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35", 55 | "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26", 56 | "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b", 57 | "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01", 58 | "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb", 59 | "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293", 60 | "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd", 61 | "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d", 62 | "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3", 63 | "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d", 64 | "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e", 65 | "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca", 66 | "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d", 67 | "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775", 68 | "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375", 69 | "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b", 70 | "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b", 71 | "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f" 72 | ], 73 | "version": "==1.14.4" 74 | }, 75 | "constantly": { 76 | "hashes": [ 77 | "sha256:586372eb92059873e29eba4f9dec8381541b4d3834660707faf8ba59146dfc35", 78 | "sha256:dd2fa9d6b1a51a83f0d7dd76293d734046aa176e384bf6e33b7e44880eb37c5d" 79 | ], 80 | "version": "==15.1.0" 81 | }, 82 | "cryptography": { 83 | "hashes": [ 84 | "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d", 85 | "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7", 86 | "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901", 87 | "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c", 88 | "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244", 89 | "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6", 90 | "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5", 91 | "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e", 92 | "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c", 93 | "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0", 94 | "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812", 95 | "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a", 96 | "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030", 97 | "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302" 98 | ], 99 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", 100 | "version": "==3.3.1" 101 | }, 102 | "cssselect": { 103 | "hashes": [ 104 | "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf", 105 | "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc" 106 | ], 107 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 108 | "version": "==1.1.0" 109 | }, 110 | "et-xmlfile": { 111 | "hashes": [ 112 | "sha256:614d9722d572f6246302c4491846d2c393c199cfa4edc9af593437691683335b" 113 | ], 114 | "version": "==1.0.1" 115 | }, 116 | "hyperlink": { 117 | "hashes": [ 118 | "sha256:47fcc7cd339c6cb2444463ec3277bdcfe142c8b1daf2160bdd52248deec815af", 119 | "sha256:c528d405766f15a2c536230de7e160b65a08e20264d8891b3eb03307b0df3c63" 120 | ], 121 | "version": "==20.0.1" 122 | }, 123 | "idna": { 124 | "hashes": [ 125 | "sha256:5205d03e7bcbb919cc9c19885f9920d622ca52448306f2377daede5cf3faac16", 126 | "sha256:c5b02147e01ea9920e6b0a3f1f7bb833612d507592c837a6c49552768f4054e1" 127 | ], 128 | "markers": "python_version >= '3.4'", 129 | "version": "==3.1" 130 | }, 131 | "incremental": { 132 | "hashes": [ 133 | "sha256:717e12246dddf231a349175f48d74d93e2897244939173b01974ab6661406b9f", 134 | "sha256:7b751696aaf36eebfab537e458929e194460051ccad279c72b755a167eebd4b3" 135 | ], 136 | "version": "==17.5.0" 137 | }, 138 | "itemadapter": { 139 | "hashes": [ 140 | "sha256:5327c2136353cb965b6b4ba564af002fd458691b8e30d3bd6b14c474d92c6b25", 141 | "sha256:cb7aaa577fefe2aa6f229ccf4d058e05f44e0178a98c8fb70ee4d95acfabb423" 142 | ], 143 | "markers": "python_version >= '3.6'", 144 | "version": "==0.2.0" 145 | }, 146 | "jdcal": { 147 | "hashes": [ 148 | "sha256:1abf1305fce18b4e8aa248cf8fe0c56ce2032392bc64bbd61b5dff2a19ec8bba", 149 | "sha256:472872e096eb8df219c23f2689fc336668bdb43d194094b5cc1707e1640acfc8" 150 | ], 151 | "version": "==1.4.1" 152 | }, 153 | "lxml": { 154 | "hashes": [ 155 | "sha256:0448576c148c129594d890265b1a83b9cd76fd1f0a6a04620753d9a6bcfd0a4d", 156 | "sha256:127f76864468d6630e1b453d3ffbbd04b024c674f55cf0a30dc2595137892d37", 157 | "sha256:1471cee35eba321827d7d53d104e7b8c593ea3ad376aa2df89533ce8e1b24a01", 158 | "sha256:2363c35637d2d9d6f26f60a208819e7eafc4305ce39dc1d5005eccc4593331c2", 159 | "sha256:2e5cc908fe43fe1aa299e58046ad66981131a66aea3129aac7770c37f590a644", 160 | "sha256:2e6fd1b8acd005bd71e6c94f30c055594bbd0aa02ef51a22bbfa961ab63b2d75", 161 | "sha256:366cb750140f221523fa062d641393092813b81e15d0e25d9f7c6025f910ee80", 162 | "sha256:42ebca24ba2a21065fb546f3e6bd0c58c3fe9ac298f3a320147029a4850f51a2", 163 | "sha256:4e751e77006da34643ab782e4a5cc21ea7b755551db202bc4d3a423b307db780", 164 | "sha256:4fb85c447e288df535b17ebdebf0ec1cf3a3f1a8eba7e79169f4f37af43c6b98", 165 | "sha256:50c348995b47b5a4e330362cf39fc503b4a43b14a91c34c83b955e1805c8e308", 166 | "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf", 167 | "sha256:535f067002b0fd1a4e5296a8f1bf88193080ff992a195e66964ef2a6cfec5388", 168 | "sha256:5be4a2e212bb6aa045e37f7d48e3e1e4b6fd259882ed5a00786f82e8c37ce77d", 169 | "sha256:60a20bfc3bd234d54d49c388950195d23a5583d4108e1a1d47c9eef8d8c042b3", 170 | "sha256:648914abafe67f11be7d93c1a546068f8eff3c5fa938e1f94509e4a5d682b2d8", 171 | "sha256:681d75e1a38a69f1e64ab82fe4b1ed3fd758717bed735fb9aeaa124143f051af", 172 | "sha256:68a5d77e440df94011214b7db907ec8f19e439507a70c958f750c18d88f995d2", 173 | "sha256:69a63f83e88138ab7642d8f61418cf3180a4d8cd13995df87725cb8b893e950e", 174 | "sha256:6e4183800f16f3679076dfa8abf2db3083919d7e30764a069fb66b2b9eff9939", 175 | "sha256:6fd8d5903c2e53f49e99359b063df27fdf7acb89a52b6a12494208bf61345a03", 176 | "sha256:791394449e98243839fa822a637177dd42a95f4883ad3dec2a0ce6ac99fb0a9d", 177 | "sha256:7a7669ff50f41225ca5d6ee0a1ec8413f3a0d8aa2b109f86d540887b7ec0d72a", 178 | "sha256:7e9eac1e526386df7c70ef253b792a0a12dd86d833b1d329e038c7a235dfceb5", 179 | "sha256:7ee8af0b9f7de635c61cdd5b8534b76c52cd03536f29f51151b377f76e214a1a", 180 | "sha256:8246f30ca34dc712ab07e51dc34fea883c00b7ccb0e614651e49da2c49a30711", 181 | "sha256:8c88b599e226994ad4db29d93bc149aa1aff3dc3a4355dd5757569ba78632bdf", 182 | "sha256:923963e989ffbceaa210ac37afc9b906acebe945d2723e9679b643513837b089", 183 | "sha256:94d55bd03d8671686e3f012577d9caa5421a07286dd351dfef64791cf7c6c505", 184 | "sha256:97db258793d193c7b62d4e2586c6ed98d51086e93f9a3af2b2034af01450a74b", 185 | "sha256:a9d6bc8642e2c67db33f1247a77c53476f3a166e09067c0474facb045756087f", 186 | "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc", 187 | "sha256:d8d3d4713f0c28bdc6c806a278d998546e8efc3498949e3ace6e117462ac0a5e", 188 | "sha256:e0bfe9bb028974a481410432dbe1b182e8191d5d40382e5b8ff39cdd2e5c5931", 189 | "sha256:f4822c0660c3754f1a41a655e37cb4dbbc9be3d35b125a37fab6f82d47674ebc", 190 | "sha256:f83d281bb2a6217cd806f4cf0ddded436790e66f393e124dfe9731f6b3fb9afe", 191 | "sha256:fc37870d6716b137e80d19241d0e2cff7a7643b925dfa49b4c8ebd1295eb506e" 192 | ], 193 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 194 | "version": "==4.6.2" 195 | }, 196 | "openpyxl": { 197 | "hashes": [ 198 | "sha256:18e11f9a650128a12580a58e3daba14e00a11d9e907c554a17ea016bf1a2c71b", 199 | "sha256:f7d666b569f729257082cf7ddc56262431878f602dcc2bc3980775c59439cdab" 200 | ], 201 | "index": "pypi", 202 | "version": "==3.0.5" 203 | }, 204 | "parsel": { 205 | "hashes": [ 206 | "sha256:70efef0b651a996cceebc69e55a85eb2233be0890959203ba7c3a03c72725c79", 207 | "sha256:9e1fa8db1c0b4a878bf34b35c043d89c9d1cbebc23b4d34dbc3c0ec33f2e087d" 208 | ], 209 | "version": "==1.6.0" 210 | }, 211 | "protego": { 212 | "hashes": [ 213 | "sha256:a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4" 214 | ], 215 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 216 | "version": "==0.1.16" 217 | }, 218 | "pyasn1": { 219 | "hashes": [ 220 | "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", 221 | "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", 222 | "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", 223 | "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", 224 | "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", 225 | "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", 226 | "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", 227 | "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", 228 | "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", 229 | "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", 230 | "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", 231 | "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", 232 | "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" 233 | ], 234 | "version": "==0.4.8" 235 | }, 236 | "pyasn1-modules": { 237 | "hashes": [ 238 | "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8", 239 | "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199", 240 | "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811", 241 | "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed", 242 | "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4", 243 | "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", 244 | "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", 245 | "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb", 246 | "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45", 247 | "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd", 248 | "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0", 249 | "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d", 250 | "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405" 251 | ], 252 | "version": "==0.2.8" 253 | }, 254 | "pycparser": { 255 | "hashes": [ 256 | "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", 257 | "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" 258 | ], 259 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 260 | "version": "==2.20" 261 | }, 262 | "pydispatcher": { 263 | "hashes": [ 264 | "sha256:5570069e1b1769af1fe481de6dd1d3a388492acddd2cdad7a3bde145615d5caf", 265 | "sha256:5be4a8be12805ef7d712dd9a93284fb8bc53f309867e573f653a72e5fd10e433" 266 | ], 267 | "version": "==2.0.5" 268 | }, 269 | "pyhamcrest": { 270 | "hashes": [ 271 | "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316", 272 | "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29" 273 | ], 274 | "markers": "python_version >= '3.5'", 275 | "version": "==2.0.2" 276 | }, 277 | "pyopenssl": { 278 | "hashes": [ 279 | "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51", 280 | "sha256:818ae18e06922c066f777a33f1fca45786d85edfe71cd043de6379337a7f274b" 281 | ], 282 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 283 | "version": "==20.0.1" 284 | }, 285 | "queuelib": { 286 | "hashes": [ 287 | "sha256:42b413295551bdc24ed9376c1a2cd7d0b1b0fa4746b77b27ca2b797a276a1a17", 288 | "sha256:ff43b5b74b9266f8df4232a8f768dc4d67281a271905e2ed4a3689d4d304cd02" 289 | ], 290 | "version": "==1.5.0" 291 | }, 292 | "scrapy": { 293 | "hashes": [ 294 | "sha256:8ce036a113ef9bc575a65b463994244e4298145f70bcdcf40b8847a2f528a1fb", 295 | "sha256:e6b724ef3d522bbce93bede7af18d1603b57cefcf589337b0f4c5b15ea0a5f29" 296 | ], 297 | "index": "pypi", 298 | "version": "==2.2" 299 | }, 300 | "service-identity": { 301 | "hashes": [ 302 | "sha256:001c0707759cb3de7e49c078a7c0c9cd12594161d3bf06b9c254fdcb1a60dc36", 303 | "sha256:0858a54aabc5b459d1aafa8a518ed2081a285087f349fe3e55197989232e2e2d" 304 | ], 305 | "version": "==18.1.0" 306 | }, 307 | "six": { 308 | "hashes": [ 309 | "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", 310 | "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" 311 | ], 312 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 313 | "version": "==1.15.0" 314 | }, 315 | "twisted": { 316 | "hashes": [ 317 | "sha256:040eb6641125d2a9a09cf198ec7b83dd8858c6f51f6770325ed9959c00f5098f", 318 | "sha256:147780b8caf21ba2aef3688628eaf13d7e7fe02a86747cd54bfaf2140538f042", 319 | "sha256:158ddb80719a4813d292293ac44ba41d8b56555ed009d90994a278237ee63d2c", 320 | "sha256:2182000d6ffc05d269e6c03bfcec8b57e20259ca1086180edaedec3f1e689292", 321 | "sha256:25ffcf37944bdad4a99981bc74006d735a678d2b5c193781254fbbb6d69e3b22", 322 | "sha256:3281d9ce889f7b21bdb73658e887141aa45a102baf3b2320eafcfba954fcefec", 323 | "sha256:356e8d8dd3590e790e3dba4db139eb8a17aca64b46629c622e1b1597a4a92478", 324 | "sha256:70952c56e4965b9f53b180daecf20a9595cf22b8d0935cd3bd664c90273c3ab2", 325 | "sha256:7408c6635ee1b96587289283ebe90ee15dbf9614b05857b446055116bc822d29", 326 | "sha256:7c547fd0215db9da8a1bc23182b309e84a232364cc26d829e9ee196ce840b114", 327 | "sha256:894f6f3cfa57a15ea0d0714e4283913a5f2511dbd18653dd148eba53b3919797", 328 | "sha256:94ac3d55a58c90e2075c5fe1853f2aa3892b73e3bf56395f743aefde8605eeaa", 329 | "sha256:a58e61a2a01e5bcbe3b575c0099a2bcb8d70a75b1a087338e0c48dd6e01a5f15", 330 | "sha256:c09c47ff9750a8e3aa60ad169c4b95006d455a29b80ad0901f031a103b2991cd", 331 | "sha256:ca3a0b8c9110800e576d89b5337373e52018b41069bc879f12fa42b7eb2d0274", 332 | "sha256:cd1dc5c85b58494138a3917752b54bb1daa0045d234b7c132c37a61d5483ebad", 333 | "sha256:cdbc4c7f0cd7a2218b575844e970f05a1be1861c607b0e048c9bceca0c4d42f7", 334 | "sha256:d267125cc0f1e8a0eed6319ba4ac7477da9b78a535601c49ecd20c875576433a", 335 | "sha256:d72c55b5d56e176563b91d11952d13b01af8725c623e498db5507b6614fc1e10", 336 | "sha256:d95803193561a243cb0401b0567c6b7987d3f2a67046770e1dccd1c9e49a9780", 337 | "sha256:e92703bed0cc21d6cb5c61d66922b3b1564015ca8a51325bd164a5e33798d504", 338 | "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467", 339 | "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4" 340 | ], 341 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 342 | "version": "==20.3.0" 343 | }, 344 | "w3lib": { 345 | "hashes": [ 346 | "sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53", 347 | "sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df" 348 | ], 349 | "version": "==1.22.0" 350 | }, 351 | "zope.interface": { 352 | "hashes": [ 353 | "sha256:05a97ba92c1c7c26f25c9f671aa1ef85ffead6cdad13770e5b689cf983adc7e1", 354 | "sha256:07d61722dd7d85547b7c6b0f5486b4338001fab349f2ac5cabc0b7182eb3425d", 355 | "sha256:0a990dcc97806e5980bbb54b2e46b9cde9e48932d8e6984daf71ef1745516123", 356 | "sha256:150e8bcb7253a34a4535aeea3de36c0bb3b1a6a47a183a95d65a194b3e07f232", 357 | "sha256:1743bcfe45af8846b775086471c28258f4c6e9ee8ef37484de4495f15a98b549", 358 | "sha256:1b5f6c8fff4ed32aa2dd43e84061bc8346f32d3ba6ad6e58f088fe109608f102", 359 | "sha256:21e49123f375703cf824214939d39df0af62c47d122d955b2a8d9153ea08cfd5", 360 | "sha256:21f579134a47083ffb5ddd1307f0405c91aa8b61ad4be6fd5af0171474fe0c45", 361 | "sha256:27c267dc38a0f0079e96a2945ee65786d38ef111e413c702fbaaacbab6361d00", 362 | "sha256:299bde0ab9e5c4a92f01a152b7fbabb460f31343f1416f9b7b983167ab1e33bc", 363 | "sha256:2ab88d8f228f803fcb8cb7d222c579d13dab2d3622c51e8cf321280da01102a7", 364 | "sha256:2ced4c35061eea623bc84c7711eedce8ecc3c2c51cd9c6afa6290df3bae9e104", 365 | "sha256:2dcab01c660983ba5e5a612e0c935141ccbee67d2e2e14b833e01c2354bd8034", 366 | "sha256:32546af61a9a9b141ca38d971aa6eb9800450fa6620ce6323cc30eec447861f3", 367 | "sha256:32b40a4c46d199827d79c86bb8cb88b1bbb764f127876f2cb6f3a47f63dbada3", 368 | "sha256:3cc94c69f6bd48ed86e8e24f358cb75095c8129827df1298518ab860115269a4", 369 | "sha256:42b278ac0989d6f5cf58d7e0828ea6b5951464e3cf2ff229dd09a96cb6ba0c86", 370 | "sha256:495b63fd0302f282ee6c1e6ea0f1c12cb3d1a49c8292d27287f01845ff252a96", 371 | "sha256:4af87cdc0d4b14e600e6d3d09793dce3b7171348a094ba818e2a68ae7ee67546", 372 | "sha256:4b94df9f2fdde7b9314321bab8448e6ad5a23b80542dcab53e329527d4099dcb", 373 | "sha256:4c48ddb63e2b20fba4c6a2bf81b4d49e99b6d4587fb67a6cd33a2c1f003af3e3", 374 | "sha256:4df9afd17bd5477e9f8c8b6bb8507e18dd0f8b4efe73bb99729ff203279e9e3b", 375 | "sha256:518950fe6a5d56f94ba125107895f938a4f34f704c658986eae8255edb41163b", 376 | "sha256:538298e4e113ccb8b41658d5a4b605bebe75e46a30ceca22a5a289cf02c80bec", 377 | "sha256:55465121e72e208a7b69b53de791402affe6165083b2ea71b892728bd19ba9ae", 378 | "sha256:588384d70a0f19b47409cfdb10e0c27c20e4293b74fc891df3d8eb47782b8b3e", 379 | "sha256:6278c080d4afffc9016e14325f8734456831124e8c12caa754fd544435c08386", 380 | "sha256:64ea6c221aeee4796860405e1aedec63424cda4202a7ad27a5066876db5b0fd2", 381 | "sha256:681dbb33e2b40262b33fd383bae63c36d33fd79fa1a8e4092945430744ffd34a", 382 | "sha256:6936aa9da390402d646a32a6a38d5409c2d2afb2950f045a7d02ab25a4e7d08d", 383 | "sha256:778d0ec38bbd288b150a3ae363c8ffd88d2207a756842495e9bffd8a8afbc89a", 384 | "sha256:8251f06a77985a2729a8bdbefbae79ee78567dddc3acbd499b87e705ca59fe24", 385 | "sha256:83b4aa5344cce005a9cff5d0321b2e318e871cc1dfc793b66c32dd4f59e9770d", 386 | "sha256:844fad925ac5c2ad4faaceb3b2520ad016b5280105c6e16e79838cf951903a7b", 387 | "sha256:8ceb3667dd13b8133f2e4d637b5b00f240f066448e2aa89a41f4c2d78a26ce50", 388 | "sha256:92dc0fb79675882d0b6138be4bf0cec7ea7c7eede60aaca78303d8e8dbdaa523", 389 | "sha256:9789bd945e9f5bd026ed3f5b453d640befb8b1fc33a779c1fe8d3eb21fe3fb4a", 390 | "sha256:a2b6d6eb693bc2fc6c484f2e5d93bd0b0da803fa77bf974f160533e555e4d095", 391 | "sha256:aab9f1e34d810feb00bf841993552b8fcc6ae71d473c505381627143d0018a6a", 392 | "sha256:abb61afd84f23099ac6099d804cdba9bd3b902aaaded3ffff47e490b0a495520", 393 | "sha256:adf9ee115ae8ff8b6da4b854b4152f253b390ba64407a22d75456fe07dcbda65", 394 | "sha256:aedc6c672b351afe6dfe17ff83ee5e7eb6ed44718f879a9328a68bdb20b57e11", 395 | "sha256:b7a00ecb1434f8183395fac5366a21ee73d14900082ca37cf74993cf46baa56c", 396 | "sha256:ba32f4a91c1cb7314c429b03afbf87b1fff4fb1c8db32260e7310104bd77f0c7", 397 | "sha256:cbd0f2cbd8689861209cd89141371d3a22a11613304d1f0736492590aa0ab332", 398 | "sha256:e4bc372b953bf6cec65a8d48482ba574f6e051621d157cf224227dbb55486b1e", 399 | "sha256:eccac3d9aadc68e994b6d228cb0c8919fc47a5350d85a1b4d3d81d1e98baf40c", 400 | "sha256:efd550b3da28195746bb43bd1d815058181a7ca6d9d6aa89dd37f5eefe2cacb7", 401 | "sha256:efef581c8ba4d990770875e1a2218e856849d32ada2680e53aebc5d154a17e20", 402 | "sha256:f057897711a630a0b7a6a03f1acf379b6ba25d37dc5dc217a97191984ba7f2fc", 403 | "sha256:f37d45fab14ffef9d33a0dc3bc59ce0c5313e2253323312d47739192da94f5fd", 404 | "sha256:f44906f70205d456d503105023041f1e63aece7623b31c390a0103db4de17537" 405 | ], 406 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 407 | "version": "==5.2.0" 408 | } 409 | }, 410 | "develop": {} 411 | } 412 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scraping Etsy 2 | 3 | This project was built using [Scrapy](https://scrapy.org/) (Scraping and Web Crawling Framework). 4 | 5 | It contains a set of Spiders to gather product's data from [Etsy Website](https://www.etsy.com). 6 | 7 | ## Prerequisites 8 | 9 | To run the Spiders, download Python 3 from [Python.org](https://www.python.org/). 10 | After install, you need to install some Python packages: 11 | ``` 12 | pip install -r requirements.txt 13 | 14 | ``` 15 | ## Usage 16 | 17 | ### Spider: search_products.py 18 | 19 | This Spider access the Etsy website and search for products based on a given search string. 20 | 21 | Supported parameters: 22 | * *search* - set the search string 23 | * *count_max* - limit the number of items/products to be scraped 24 | * *reviews_option* - set the method to get the product's reviews 25 | 26 | For example, to search for '3d printed' products go to the project's folder and run: 27 | ``` 28 | scrapy crawl search_products -a search='3d printed' 29 | ``` 30 | To save the results, use -o parameter: 31 | ``` 32 | scrapy crawl search_products -a search='3d printed' -o products.csv 33 | ``` 34 | The Spider will create a CSV and Excel files. 35 | 36 | To limit the number of products scraped, use the *count_max* parameter: 37 | ``` 38 | scrapy crawl search_products -a search='3d printed' -a count_max=10 -o products.csv 39 | ``` 40 | 41 | The *product reviews* data can be obtained in three ways: 42 | * 1 - Spider will get only the reviews in the product's page, that is, 4 reviews. This is the default and fastest option for scraping. 43 | * 2 - Spider will produce an Ajax request to get all reviews in the product's page (simulate the click in the *+More* button to load more reviews). In this option, the Spider will usually get 10 reviews. 44 | * 3 - Spider will visit the page with all store reviews (click in the *Read All Reviews* button) and get all the reviews for this specific product. As the Spider will visit several pages to get the reviews, this is the slower scraping option and there is a chance to get temporarily blocked by Etsy because of the high number of requests. 45 | 46 | To choose the option to scraping the reviews use the *-a reviews_option* parameter: 47 | ``` 48 | scrapy crawl search_products -a search='3d printed' -a reviews_option=3 -o products.csv 49 | ``` 50 | 51 | ## Scraping speed 52 | 53 | You can change the number of concurrent requests performed by Scrapy in the *setting.py* file. 54 | ``` 55 | CONCURRENT_REQUESTS = 10 56 | ``` 57 | Change this if you want to decrease the number of requests to avoid get blocking by Etsy. 58 | 59 | If you only need the products URLS, the scraping can be faster, just use the `urls_only` flag: 60 | 61 | ``` 62 | scrapy crawl search_products -a search='xbox controller elite' -o products.csv -a urls_only=true 63 | ``` 64 | -------------------------------------------------------------------------------- /etsy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cpatrickalves/scraping-etsy/2eb602c947b71b8b33f1cae53d39ca7e93eeac95/etsy/__init__.py -------------------------------------------------------------------------------- /etsy/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | from w3lib.html import remove_tags 10 | from scrapy.loader.processors import MapCompose, TakeFirst, Join 11 | 12 | # Remove extras spaces in strings 13 | def strip_space(value): 14 | return value.strip() 15 | 16 | # Put only one space between strings 17 | def normalize_space(value): 18 | return " ".join(value.split()) 19 | 20 | # This class defines the fields that will be created 21 | class ProductItem(scrapy.Item): 22 | 23 | # Each item may have a input and output processor 24 | # Each processor performs a series of transformations on the data before saving it 25 | 26 | product_id = scrapy.Field(output_processor=TakeFirst()) 27 | url = scrapy.Field(output_processor=TakeFirst()) 28 | title = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 29 | output_processor=TakeFirst()) 30 | description = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 31 | output_processor=Join(' ')) 32 | 33 | product_options = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 34 | output_processor=Join(',')) 35 | 36 | price = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 37 | output_processor=TakeFirst()) 38 | 39 | currency = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 40 | output_processor=TakeFirst()) 41 | 42 | rating = scrapy.Field(output_processor=TakeFirst()) 43 | 44 | number_of_reviews = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 45 | output_processor=TakeFirst()) 46 | count_of_images = scrapy.Field(output_processor=TakeFirst()) 47 | 48 | images_urls = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 49 | output_processor=Join(',')) 50 | 51 | overview = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 52 | output_processor=Join(',')) 53 | 54 | favorited_by = scrapy.Field(output_processor=TakeFirst()) 55 | store_name = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 56 | output_processor=TakeFirst()) 57 | 58 | store_location = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 59 | output_processor=TakeFirst()) 60 | 61 | return_location = scrapy.Field(input_processor=MapCompose(normalize_space, remove_tags, strip_space), 62 | output_processor=TakeFirst()) 63 | 64 | reviews = scrapy.Field() -------------------------------------------------------------------------------- /etsy/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | 10 | 11 | class EtsySpiderMiddleware(object): 12 | # Not all methods need to be defined. If a method is not defined, 13 | # scrapy acts as if the spider middleware does not modify the 14 | # passed objects. 15 | 16 | @classmethod 17 | def from_crawler(cls, crawler): 18 | # This method is used by Scrapy to create your spiders. 19 | s = cls() 20 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 21 | return s 22 | 23 | def process_spider_input(self, response, spider): 24 | # Called for each response that goes through the spider 25 | # middleware and into the spider. 26 | 27 | # Should return None or raise an exception. 28 | return None 29 | 30 | def process_spider_output(self, response, result, spider): 31 | # Called with the results returned from the Spider, after 32 | # it has processed the response. 33 | 34 | # Must return an iterable of Request, dict or Item objects. 35 | for i in result: 36 | yield i 37 | 38 | def process_spider_exception(self, response, exception, spider): 39 | # Called when a spider or process_spider_input() method 40 | # (from other spider middleware) raises an exception. 41 | 42 | # Should return either None or an iterable of Response, dict 43 | # or Item objects. 44 | pass 45 | 46 | def process_start_requests(self, start_requests, spider): 47 | # Called with the start requests of the spider, and works 48 | # similarly to the process_spider_output() method, except 49 | # that it doesn’t have a response associated. 50 | 51 | # Must return only requests (not items). 52 | for r in start_requests: 53 | yield r 54 | 55 | def spider_opened(self, spider): 56 | spider.logger.info('Spider opened: %s' % spider.name) 57 | 58 | 59 | class EtsyDownloaderMiddleware(object): 60 | # Not all methods need to be defined. If a method is not defined, 61 | # scrapy acts as if the downloader middleware does not modify the 62 | # passed objects. 63 | 64 | @classmethod 65 | def from_crawler(cls, crawler): 66 | # This method is used by Scrapy to create your spiders. 67 | s = cls() 68 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 69 | return s 70 | 71 | def process_request(self, request, spider): 72 | # Called for each request that goes through the downloader 73 | # middleware. 74 | 75 | # Must either: 76 | # - return None: continue processing this request 77 | # - or return a Response object 78 | # - or return a Request object 79 | # - or raise IgnoreRequest: process_exception() methods of 80 | # installed downloader middleware will be called 81 | return None 82 | 83 | def process_response(self, request, response, spider): 84 | # Called with the response returned from the downloader. 85 | 86 | # Must either; 87 | # - return a Response object 88 | # - return a Request object 89 | # - or raise IgnoreRequest 90 | return response 91 | 92 | def process_exception(self, request, exception, spider): 93 | # Called when a download handler or a process_request() 94 | # (from other downloader middleware) raises an exception. 95 | 96 | # Must either: 97 | # - return None: continue processing this exception 98 | # - return a Response object: stops process_exception() chain 99 | # - return a Request object: stops process_exception() chain 100 | pass 101 | 102 | def spider_opened(self, spider): 103 | spider.logger.info('Spider opened: %s' % spider.name) 104 | -------------------------------------------------------------------------------- /etsy/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | # This Pipeline processes several items scraped. 9 | class EtsyPipeline(object): 10 | def process_item(self, item, spider): 11 | 12 | # Format the price output 13 | if 'price' in item: 14 | # Check if there is a currency symbol 15 | if len(item['price'].split()) > 1: 16 | # Remove the currency symbol and the + signal 17 | item['price'] = item['price'].split()[1].replace('+','') 18 | else: 19 | # Remove the currency symbol and the + signal 20 | item['price'] = item['price'].replace('$','').replace('+','') 21 | 22 | # Remove the 'in' string 23 | if 'store_location' in item: 24 | item['store_location'] = item['store_location'].replace('in ', '') 25 | 26 | # Remove the 'From' string 27 | if 'return_location' in item: 28 | item['return_location'] = item['return_location'].replace('From ', '') 29 | 30 | # Sometimes the spider take the rate in the wrong format (ex: 48.333 instead of 4.8333) 31 | if 'rating' in item: 32 | rating = item['rating'] 33 | if float(rating) > 5: 34 | # Ex: Transform this 48.333 in 4.83) 35 | rating = rating[0] + '.' + rating[1:2] 36 | else: 37 | rating = round(float(rating), 2) 38 | 39 | item['rating'] = rating 40 | 41 | return item 42 | -------------------------------------------------------------------------------- /etsy/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for etsy project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # https://doc.scrapy.org/en/latest/topics/settings.html 9 | # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 10 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'etsy' 13 | 14 | SPIDER_MODULES = ['etsy.spiders'] 15 | NEWSPIDER_MODULE = 'etsy.spiders' 16 | 17 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 18 | USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36' 19 | 20 | # Obey robots.txt rules 21 | ROBOTSTXT_OBEY = False 22 | 23 | # Data fields that are exported to csv or Json output 24 | FEED_EXPORT_FIELDS = ['title', 'product_id', 'url', 'price', 'rating', 'number_of_reviews', 'product_options', 'count_of_images', 'images_urls', 'favorited_by', 'store_name', 'description', 'reviews'] 25 | 26 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 27 | CONCURRENT_REQUESTS = 2 28 | 29 | # Configure a delay for requests for the same website (default: 0) 30 | # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay 31 | # See also autothrottle settings and docs 32 | #DOWNLOAD_DELAY = 1 33 | #RANDOMIZE_DOWNLOAD_DELAY = True 34 | 35 | # The download delay setting will honor only one of: 36 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 37 | #CONCURRENT_REQUESTS_PER_IP = 16 38 | 39 | # Disable cookies (enabled by default) 40 | COOKIES_ENABLED = True 41 | 42 | # Disable Telnet Console (enabled by default) 43 | #TELNETCONSOLE_ENABLED = False 44 | 45 | # Override the default request headers: 46 | #DEFAULT_REQUEST_HEADERS = { 47 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 48 | # 'Accept-Language': 'en', 49 | #} 50 | 51 | # Enable or disable spider middlewares 52 | # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html 53 | #SPIDER_MIDDLEWARES = { 54 | # 'etsy.middlewares.EtsySpiderMiddleware': 543, 55 | #} 56 | 57 | # Enable or disable downloader middlewares 58 | # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 59 | #DOWNLOADER_MIDDLEWARES = { 60 | # 'etsy.middlewares.EtsyDownloaderMiddleware': 543, 61 | #} 62 | 63 | # Enable or disable extensions 64 | # See https://doc.scrapy.org/en/latest/topics/extensions.html 65 | #EXTENSIONS = { 66 | # 'scrapy.extensions.telnet.TelnetConsole': None, 67 | #} 68 | 69 | # Configure item pipelines 70 | # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html 71 | ITEM_PIPELINES = { 72 | 'etsy.pipelines.EtsyPipeline': 300, 73 | } 74 | 75 | # Enable and configure the AutoThrottle extension (disabled by default) 76 | # See https://doc.scrapy.org/en/latest/topics/autothrottle.html 77 | #AUTOTHROTTLE_ENABLED = True 78 | # The initial download delay 79 | #AUTOTHROTTLE_START_DELAY = 5 80 | # The maximum download delay to be set in case of high latencies 81 | #AUTOTHROTTLE_MAX_DELAY = 60 82 | # The average number of requests Scrapy should be sending in parallel to 83 | # each remote server 84 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 85 | # Enable showing throttling stats for every response received: 86 | #AUTOTHROTTLE_DEBUG = False 87 | 88 | # Enable and configure HTTP caching (disabled by default) 89 | # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 90 | #HTTPCACHE_ENABLED = True 91 | #HTTPCACHE_EXPIRATION_SECS = 3600 92 | #HTTPCACHE_DIR = 'httpcache' 93 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 94 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 95 | -------------------------------------------------------------------------------- /etsy/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /etsy/spiders/products.json: -------------------------------------------------------------------------------- 1 | [ 2 | {"title": "Pre-order. An adult fox. 3d printed bjd animal. Sizes 5 to 7 cm, removable heads on magnets. Ball jointed fox dolls to order. Red foxes", "product_id": "275174392", "url": "www.etsy.com/listing/275174392", "price": "1,236.62", "rating": 5.0, "number_of_votes": "117", "product_options": "Select a height: 5 (one head) cm (BRL 1,236.62), 5 (2 heads) cm (BRL 1,368.21), 6 (one head) cm (BRL 1,545.78), 6 (2 heads) cm (BRL 1,710.27), 7 (one head) cm (BRL 1,854.93), 7 (2 heads) cm (BRL 2,052.32)|Select a color: Red, Silver, Lunar, Winter", "count_of_images": 9, "favorited_by": "1350", "return_location": "Russia", "description": "\u25cf UPD 05.02.2019 It's available to order a new color - Winter \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550 The ball jointed doll Adult fox has several features: \u2022 The doll is made very movable and flexible. It easily takes poses and holds them. \u2022 The Fox can turn around, put the head low down as in the direct as in the side positions thanks to a double neck joint. \u2022 The body consists of 5 items, which allows it to arch the back, to fold (almost) a ball and turn to the side (see the photos). \u2022 Long and slender legs not only help the Fox to run fast and to dig burrows, but also to scratch behind the ear and close his eyes. \u2022 We make a special attention on the tail of the doll. It has a very attractive shape, that giving the feeling of fluffiness and volume. \u2022 An elegant nose makes a character face. \u2022 We have achieved a new, more intense ginger color, which we will use for all of the Fox dolls. When we played with the fox we understood that we didn't want to let it go from our hands! The sizes for adult fox are measured at the withers! And for comparison, an adult Fox of 5.5 cm at the withers looks like a teenager next to a Fox cub of 4,5 cm (for cubs the size is measured from the ground to the top of the ears). If you want to \"collect\" family, we recommend the adult fox of 7.5 cm is for the fox cub of 4,5 cm. You can place an order a fox cub here https://www.etsy.com/ru/listing/209255094/kukla-na-zakaz-lisenok-45-sm-55-sm-65-sm Learn more about the doll, its sizes and options on our website - http://walloyamorring.com/?page_id=539 About the doll \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550 Mold: Adult fox Size*: 5 cm, 6 cm and 7 cm Color: red, silver, lunar (lilac) Heads: 6 pcs (of each type) Eyes: 10 pcs Noomis**: from 15 * - the size is measured at the withers ** - it's our bonus points you can learn more here http://walloyamorring.com/?page_id=406 There are 2 sets of the doll. \"One head\" set means the doll has one head (with open eyes or sleeping) on your wish. \"2 heads\" set means that the doll has 2 heads, one of them has eyes and the other has closed eyes. There are 6 kinds of available heads to choose. They are Standard, Awaking, Sleeping, Sleepy, Open mouth and Tricky. They are available in 2 types - Realistic and Cartoon. See the last picture. Please leave heads and number of eyes in comment to the order when you place an order. Get touch with us before ordering if you are interested in more than 2 heads. Parts of the doll are printed with a high-quality industrial 3D printer (EOS Formiga P110) by SLS method. Material is white polyamide (EOS PA 2200). Then the parts are colored in 3 different ways, the coloring is fixed with a protective varnish. The doll is assembled to elastic (elastic thread or string), heads are removable on magnets, eyes made out of plastic. It usually takes about 2 - 3 months to make the doll. Feel free ask us if there is something unclear to you or you have any questions or you'd like to customize the doll. We'll be happy to consider your ideas. And also it'll be so wonderful if we can help you. Thank you for reading this and for your interest to our work! \u25cf UPD 05.02.2019 It's available to order a new color - Winter \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550 The ball jointed doll Adult fox has several features: \u2022 The doll is made very movable and flexible. It easily takes poses and holds them. \u2022 The Fox can turn around, put the head low down as in the direct as in the side positions thanks to a double neck joint. \u2022 The body consists of 5 items, which allows it to arch the back, to fold (almost) a ball and turn to the side (see the photos). \u2022 Long and slender legs not only help the Fox to run fast and to dig burrows, but also to scratch behind the ear and close his eyes. \u2022 We make a special attention on the tail of the doll. It has a very attractive shape, that giving the feeling of fluffiness and volume. \u2022 An elegant nose makes a character face. \u2022 We have achieved a new, more intense ginger color, which we will use for all of the Fox dolls. When we played with the fox we understood that we didn't want to let it go from our hands! The sizes for adult fox are measured at the withers! And for comparison, an adult Fox of 5.5 cm at the withers looks like a teenager next to a Fox cub of 4,5 cm (for cubs the size is measured from the ground to the top of the ears). If you want to \"collect\" family, we recommend the adult fox of 7.5 cm is for the fox cub of 4,5 cm. You can place an order a fox cub here https://www.etsy.com/ru/listing/209255094/kukla-na-zakaz-lisenok-45-sm-55-sm-65-sm Learn more about the doll, its sizes and options on our website - http://walloyamorring.com/?page_id=539 About the doll \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550 Mold: Adult fox Size*: 5 cm, 6 cm and 7 cm Color: red, silver, lunar (lilac) Heads: 6 pcs (of each type) Eyes: 10 pcs Noomis**: from 15 * - the size is measured at the withers ** - it's our bonus points you can learn more here http://walloyamorring.com/?page_id=406 There are 2 sets of the doll. \"One head\" set means the doll has one head (with open eyes or sleeping) on your wish. \"2 heads\" set means that the doll has 2 heads, one of them has eyes and the other has closed eyes. There are 6 kinds of available heads to choose. They are Standard, Awaking, Sleeping, Sleepy, Open mouth and Tricky. They are available in 2 types - Realistic and Cartoon. See the last picture. Please leave heads and number of eyes in comment to the order when you place an order. Get touch with us before ordering if you are interested in more than 2 heads. Parts of the doll are printed with a high-quality industrial 3D printer (EOS Formiga P110) by SLS method. Material is white polyamide (EOS PA 2200). Then the parts are colored in 3 different ways, the coloring is fixed with a protective varnish. The doll is assembled to elastic (elastic thread or string), heads are removable on magnets, eyes made out of plastic. It usually takes about 2 - 3 months to make the doll. Feel free ask us if there is something unclear to you or you have any questions or you'd like to customize the doll. We'll be happy to consider your ideas. And also it'll be so wonderful if we can help you. Thank you for reading this and for your interest to our work! + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/lupurt \nRating: 5 \nDate: Dec 24, 2018 \nContent: The fox is amazing, a true piece of art! All tiny details are perfect, very high quality - painting is amazing, realistic and professional.I especially like how posable is the doll. I'm a doll collector so I have a good base of comparison and I have to say that Dmitry and Svetlana really raised the bar - not just their doll is amazing but they are wonderful people to work with: very helpful, open to questions, and great in comms. I highly recommend the shop and their dolls. \r\nThank you very much again for your kindness!!! I wish you a happy holdiays!!\n\nReview number: 2 \nProfile: www.etsy.com/people/Superviki111 \nRating: 5 \nDate: Aug 3, 2018 \nContent: I must say, I\u2019m really impressed. \n \nShipping: This fox was made in exactly 3 months, and it took about a week to get here so definitelly 10/10. \n \nCustomer service: I have to say, they have the best customer service I have ever experienced. They answered every single one of my questions and were really nice and kind. I also received a gift from them for being the first customer from my country, which is extremely nice. 12/10 \n \nProduct: The fox is amazing! She is beautiful, really light in weight and poses like a real fox. The paint job is done stunningly and I absolutely love the little paws she has. 10/10 \n \nOveral: It came fast and was packaged really securely. I also received a gift since I bought her on Friday 13th. Overally the package is 100/10. I couldn\u2019t be happier. Definitelly buying again in the future. THANK YOU! :)\n \n \n \n I must say, I\u2019m really impressed. \n \nShipping: This fox was made in exactly 3 months, and it took about a week to get here so definitelly 10/10. \n \nCustomer service: I have to say, they have the best customer service I have ever experienced. They answered every single one of my questions and were really nice and kind. I also received a gift from them for being the first customer from my country, which is extremely nice. 12/10 \n \nProduct: The fox is amazing! She is beautiful, really light in weight and poses like a real fox. The paint job is done stunningly and I absolutely love the...\n \n more\n\nReview number: 3 \nProfile: www.etsy.com/people/nightmareangel1908 \nRating: 5 \nDate: Feb 7, 2018 \nContent: I love my little fox it is absolutely beautiful and poses so much better than I had hoped!!!\n\nReview number: 4 \nProfile: www.etsy.com/people/jornie1 \nRating: 5 \nDate: Nov 12, 2017 \nContent: A fabulous little doll and totally worth the wait! The expression is terrific and posing this little fox is so much fun."]}, 3 | {"title": "HELL BOY 2019 3D print - Digital STL file", "product_id": "689241591", "url": "www.etsy.com/listing/689241591", "price": "59.45", "product_options": "", "count_of_images": 10, "description": "HELL BOY 2019 BUST [ This is digital file - and will not refund after purchased] format file : STL 3 parts for printing: size current: X:148 (mm) Y:200 (mm) Z:114 (mm) The STL is for personal use only. Do not copy or redistribute work if you have any question or problem with model(change the scale, missing files of something else ),Feel free contact to my personal email: blackstar.dvh [!at] gmail.com thank you !", "reviews": [""]}, 4 | {"title": "CaseCorder: 3D Printed Folding Case and Kickstand for Handheld, Tabletop, or Tripod Video (iPhone and Samsung)", "product_id": "586138253", "url": "www.etsy.com/listing/586138253", "price": "47.56", "rating": 0.0, "number_of_votes": "8", "product_options": "Select a color: White, Black, Pink, Blue|Select an option: iPhone 5, iPhone 6, iPhone 6 Plus, iPhone 7, iPhone 7 Plus, iPhone 8, iPhone X, iPhone XR, Samsung Galaxy S3, Samsung Galaxy S5, Samsung Galaxy Note8, Samsung Galaxy S8", "count_of_images": 4, "favorited_by": "26", "return_location": "United States", "description": "Please note that this does not include the GorillaPod Tripod shown in the picture. This is for the adaptive case only. Welcome to CaseCorder! The CaseCorder is a device that mimics the ergonomics of a camcorder, which makes it easier to shoot video without the hassle or cost of carrying a dedicated video recording device. It works well for handheld video, shooting from a flat surface, or for mounting to a tripod. In addition to its original purpose, the CaseCorder can be used for anything which requires the phone to be held stable in a vertical or horizontal orientation. This includes things such as video viewing, video chat, or even GPS navigation. For more information, you can see the video at http://www.casecorder.com, or see Knoptop's review at https://www.youtube.com/watch?v=8DXDWHj1iM0. CaseCorder is available for the following phones: iPhone 5, iPhone 6, iPhone 6 plus, iPhone 7, iPhone 7 plus, iPhone 8, iPhone X, iPhone XR, Samsung Galaxy S3, Samsung Galaxy S5, Samsung Galaxy S8, Samsung Galaxy Note 8. If you want it for another phone, just shoot me a message! And is available in the following colors: Black, White, Blue, Pink If you're interested in other phone models, or in combining colors, let me know, and I'll see what I can do! Please note that this does not include the GorillaPod Tripod shown in the picture. This is for the adaptive case only. Welcome to CaseCorder! The CaseCorder is a device that mimics the ergonomics of a camcorder, which makes it easier to shoot video without the hassle or cost of carrying a dedicated video recording device. It works well for handheld video, shooting from a flat surface, or for mounting to a tripod. In addition to its original purpose, the CaseCorder can be used for anything which requires the phone to be held stable in a vertical or horizontal orientation. This includes things such as video viewing, video chat, or even GPS navigation. For more information, you can see the video at http://www.casecorder.com, or see Knoptop's review at https://www.youtube.com/watch?v=8DXDWHj1iM0. CaseCorder is available for the following phones: iPhone 5, iPhone 6, iPhone 6 plus, iPhone 7, iPhone 7 plus, iPhone 8, iPhone X, iPhone XR, Samsung Galaxy S3, Samsung Galaxy S5, Samsung Galaxy S8, Samsung Galaxy Note 8. If you want it for another phone, just shoot me a message! And is available in the following colors: Black, White, Blue, Pink If you're interested in other phone models, or in combining colors, let me know, and I'll see what I can do! + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/londonvallery \nRating: 5 \nDate: Jan 11, 2018 \nContent: Amazing product and amazing manufacturer! If I could give 6 stars, I would!\n\nReview number: 2 \nProfile: www.etsy.com/people/wildrosebotanicals \nRating: 5 \nDate: Oct 24, 2017 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/londonvallery \nRating: 5 \nDate: Sep 8, 2017 \nContent: Absolutely amazing! It'll work perfectly!\n\nReview number: 4 \nProfile: www.etsy.com/people/pianogirl12347 \nRating: 1 \nDate: Jan 13, 2017 \nContent: I'm disappointed because I thought it came with the kickstand. It didn't."]}, 5 | {"title": "3D printed LOVE sculpture", "product_id": "651791689", "url": "www.etsy.com/listing/651791689", "price": "27.71", "rating": 5.0, "number_of_votes": "4", "product_options": "Select a color: Brown, Green, Pink, White, Black, Red", "count_of_images": 2, "favorited_by": "5", "return_location": "United States", "description": "3D printed LOVE sculpture. Item is 3in x 3in by 2in. You can stand it up or lay it down absolutely beautiful. This is printed in PLA plastic. Any questions please ask!", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/ojsisytp \nRating: 5 \nDate: Feb 16, 2019 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/gkdez5u0 \nRating: 5 \nDate: Jan 20, 2019 \nContent: I absolutely love this item! It keeps everything organized on my desk. No more searching drawers for the correct flash drive. Great customer service and quick shipping. Thanks!\n\nReview number: 3 \nProfile: www.etsy.com/people/wpct5v2w \nRating: 5 \nDate: Jan 18, 2019 \nContent: quick shipping. Item is as described. \ud83d\udc4d\ud83c\udffc\ud83d\udc4d\ud83c\udffc\n\nReview number: 4 \nProfile: www.etsy.com/people/sagebrown1 \nRating: 5 \nDate: Dec 21, 2018 \nContent: "]}, 6 | {"title": "Seven 3d printed BJD Dragon Medieval", "product_id": "636142608", "url": "www.etsy.com/listing/636142608", "price": "207.69", "rating": 4.92, "number_of_votes": "315", "product_options": "Select a color: Black (BRL 253.63), Black w/ Sparkles [Sold out, waitlist available], White(Milky) (BRL 253.63), Silver (BRL 253.63), Iron Red (BRL 272.69), Purple (BRL 253.63), Green (BRL 253.63), Pink (BRL 253.63), Blue (BRL 272.69), Orange (BRL 207.69), light blue (BRL 253.63), Custom [Sold out, waitlist available], Marble [Sold out, waitlist available], Red (BRL 253.63), Deep Blue (BRL 253.63), Military Green (BRL 282.20), Skin-Fair complexion (BRL 282.20), Skin- Beige (BRL 282.20), Baby Pink (BRL 253.63), Lime Green (BRL 253.63), GOLD - Silky [Sold out, waitlist available], COPPER - Silky [Sold out, waitlist available]|Select an option: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, Random", "count_of_images": 10, "favorited_by": "189", "return_location": "United States", "description": "See what's printing today on our facebook live broadcast: www.facebook.com/JollyBuild-169979676913663 \"Seven\" is a 3D Printed Dragon, Ball jointed and Articulated. Wings have black fleece cloth as a webbing. It will ship fully assembled. **Message me if you want the CLAW on the wing on a different color YOU NEED TO KNOW: -Seven may have some flaws which is normal on 3d prints but I will do my best to minimize it. -overtime, the joints might get loose- It cannot be used as a toy. (We will include extra parts of the parts that have a high possibility of breaking off. SIZE: About 24 inches Wide (wingspan), 17inches head to tail. Material: PLA Plastic (Earth Friendly) Realistic Dragon eyes (Glass Eyes) Fleece cloth for wing webbings. *spotless clean. *Quality print. **Will be shipped in a BOX to avoid any damage during shipping. \"Braq\" is a 3d printed dragon, there will be some flaws which is normal on 3d prints. Other Related Items: Braq BJD (original): www.etsy.com/listing/554886949 Horned/Spiked elder dragon: www.etsy.com/listing/562404678 DIY Unassembled Braq (CHEAPER): www.etsy.com/listing/551986400 DIY Unassembled Spiked Braqolie dragon(CHEAPER) https://www.etsy.com/listing/588988219 Sea Serpent: https://www.etsy.com/listing/598430517 Dragon egg: www.etsy.com/listing/546746858 Seven the Articulated Dragon by 7Fish Published on May 12, 2018 www.thingiverse.com/thing:2884250 See what's printing today on our facebook live broadcast: www.facebook.com/JollyBuild-169979676913663 \"Seven\" is a 3D Printed Dragon, Ball jointed and Articulated. Wings have black fleece cloth as a webbing. It will ship fully assembled. **Message me if you want the CLAW on the wing on a different color YOU NEED TO KNOW: -Seven may have some flaws which is normal on 3d prints but I will do my best to minimize it. -overtime, the joints might get loose- It cannot be used as a toy. (We will include extra parts of the parts that have a high possibility of breaking off. SIZE: About 24 inches Wide (wingspan), 17inches head to tail. Material: PLA Plastic (Earth Friendly) Realistic Dragon eyes (Glass Eyes) Fleece cloth for wing webbings. *spotless clean. *Quality print. **Will be shipped in a BOX to avoid any damage during shipping. \"Braq\" is a 3d printed dragon, there will be some flaws which is normal on 3d prints. Other Related Items: Braq BJD (original): www.etsy.com/listing/554886949 Horned/Spiked elder dragon: www.etsy.com/listing/562404678 DIY Unassembled Braq (CHEAPER): www.etsy.com/listing/551986400 DIY Unassembled Spiked Braqolie dragon(CHEAPER) https://www.etsy.com/listing/588988219 Sea Serpent: https://www.etsy.com/listing/598430517 Dragon egg: www.etsy.com/listing/546746858 Seven the Articulated Dragon by 7Fish Published on May 12, 2018 www.thingiverse.com/thing:2884250 + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/PinkDragoness \nRating: 5 \nDate: Mar 12, 2019 \nContent: I really love my lil dude dragon <3 these things are amazing and I HIGHLY recommend them! I am beyond happy . All of them are amazing <3\n\nReview number: 2 \nProfile: www.etsy.com/people/xylii \nRating: 5 \nDate: Mar 9, 2019 \nContent: Very nice dragon. Well made and looks great. Just don't get too close to his mouth until be gets used to you. Almost bit my finger.\n\nReview number: 3 \nProfile: www.etsy.com/people/leuufj3c \nRating: 5 \nDate: Mar 6, 2019 \nContent: Oh my goodness he is amazing, I always dreamed of something like this when I was little. And here it is. It\u2019s beautifully made and I love it. Thank you so much\n\nReview number: 4 \nProfile: www.etsy.com/people/redfox1997 \nRating: 5 \nDate: Feb 6, 2019 \nContent: This is by far my favorite of the two types of dragon I ordered from this shop! So fun! Thank you!!"]}, 7 | {"title": "Desk Tidy / Pencil Cup / Stationary Storage - 3D Printed", "product_id": "573945408", "url": "www.etsy.com/listing/573945408", "price": "73.72", "rating": 4.97, "number_of_votes": "32", "product_options": "Select a color: White, Pink, Teal, Blue, Grey 1, Grey 2, Grey 3", "count_of_images": 6, "favorited_by": "96", "return_location": "United Kingdom", "description": "3D printed geometric faceted organisation pot, perfectly sized to fit an assortment of pens, pencils, glue sticks etc. The printing affect gives a ribbed appearance and feels great to the touch. The origami shape gives an elegant and modern look to this pot and a great addition to your home, office or workplace. I have designed these pots to be 3D printed in an unconventional way using a thick nozzle and 1 continuous spiral of plastic to create the unique surface texture of the product. It is then hand finished and hand assembled by myself in the studio. The base has a cork bottom to stop scratching surfaces and slipping. They measure 7.5cm diameter and 8cm tall. If you would like other colours that are not shown or different sizes for specific items, please drop me a message and I will see what I can do! Available with standard shipping and upgrade to 1st class in UK for just 60p more. Standard Royal Mail shipping for worldwide buyers.", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/ChantalsGifts \nRating: 5 \nDate: Apr 12, 2018 \nContent: Lovely item. Looks great thanks\n\nReview number: 2 \nProfile: www.etsy.com/people/ecovenee \nRating: 5 \nDate: Feb 25, 2019 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/ecovenee \nRating: 5 \nDate: Feb 25, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/villaelisabetta \nRating: 5 \nDate: Jan 11, 2019 \nContent: beautiful and original. well done!"]}, 8 | {"title": "Mini Enigma Machine - 3D Printed!", "product_id": "602719869", "url": "www.etsy.com/listing/602719869", "price": "63.42", "rating": 4.99, "number_of_votes": "720", "product_options": "", "count_of_images": 3, "favorited_by": "67", "return_location": "United States", "description": "This tiny, 2\" Enigma Machine was designed using open source CAD software, 3D printed on a Makerbot Replicator, and hand painted and assembled. A great addition to your desk or studio. Includes one machine. Includes tiny fact sheet with trivia on the original machine. Not suitable for ages 5 and under due to small parts. More information at www.rabbitengineering.com", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/bjworyk \nRating: 5 \nDate: Dec 26, 2018 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/desorkin \nRating: 5 \nDate: Oct 23, 2018 \nContent: Very cool item. High quality, realistic. The hinges are threads but work well, and were undamaged in shipment.\n\nReview number: 3 \nProfile: www.etsy.com/people/ejahng \nRating: 5 \nDate: Oct 14, 2018 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/DigitalGuyX \nRating: 5 \nDate: Sep 21, 2018 \nContent: Awesomely cool Enigma Machine. Will this thing help me win a war?"]}, 9 | {"title": "3D Printed Bowl", "product_id": "526638678", "url": "www.etsy.com/listing/526638678", "price": "237.81", "rating": 5.0, "number_of_votes": "4", "product_options": "Select a width: 12 inches (BRL 594.53), 7 inches (BRL 237.81)", "count_of_images": 5, "favorited_by": "11", "return_location": "United States", "description": "3D Printed voronoi bowl Material - PLA Finish - None Dimensions - 7\" x 7\" x 2.7\" OR Dimensions - 12\" x 12\" x 4.5\" For Bulk orders email us at info [!at] tangiblecreative.com with the subject ETSY Tangiblecreative.com info [!at] tangiblecreative.com", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/ffreiheit \nRating: 5 \nDate: Feb 17, 2019 \nContent: Really nice coaster. Nice thickness, details all look good, etc.\n\nReview number: 2 \nProfile: www.etsy.com/people/ciara0421 \nRating: 5 \nDate: Feb 9, 2019 \nContent: Love this planter for propagating my plants! Shipping was quick and seller even threw in two coasters. Will definitely be ordering more.\n\nReview number: 3 \nProfile: www.etsy.com/people/stopmailingme2 \nRating: 5 \nDate: Dec 5, 2017 \nContent: Item is unique and very detailed. Item was well packed and came quickly. With this being a new shop I can hardly wait to see what is offered in the future\n\nReview number: 4 \nProfile: www.etsy.com/people/kcangialosi24 \nRating: 5 \nDate: Jul 10, 2017 \nContent: Item arrived in excellent condition, and was shipped and delivered quickly. I'm impressed by the care and attention that TangibleCreative put into the product/packaging, and look forward to seeing what designs they come up with next."]}, 10 | {"title": "Dilophosaurus Shower Head | 3D printed", "product_id": "558104344", "url": "www.etsy.com/listing/558104344", "price": "118.87", "rating": 4.97, "number_of_votes": "178", "product_options": "Select an option: Copper, Chrome, Pewter, Custom", "count_of_images": 5, "favorited_by": "126", "return_location": "United States", "description": "Bring Jurassic Park straight to your bathroom with this 3D printed, water-spitting dino! Features a detailed dilophosaurus skull with a wide, contoured frill and built-in shower head fitting complete with 87 nozzles, printed in sturdy ABS plastic. This product includes one dilophosaurus shower head and 6 inches of Teflon pipe thread tape to seal the connection between the shower head and the shower arm. Dimensions: This shower head is ~6.5 inches (16.5 cm) across the span of the frills. Note on color choices: Our default is to finish this product with a coat of beautiful, bright copper, pewter or chrome spray paint for an overall cohesive and modern style. If however, you would like to customize your color choices, we would be happy to accommodate you. We can print in a fairly wide array of ABS plastic colors, and also have other spray paints available too. We can even hand paint your print to your custom specifications for a small additional fee, if you would prefer. Please select \"custom\" at checkout if you want something beyond the three default colors and then contact us to specify what color options you would like. *Many of our products can be personalized with names, dates, short phrases, etc. to meet your needs. Please see our \u201cAbout\u201d section for additional details. ************************************************************************************************************** At StarkContrasts, we select the right plastic for each product and carefully dial in our machine settings to get you the best quality we can. The technology of 3D printing is amazing, but it's not perfect! To work, 3D printers lay down extruded plastic in thin precise layers which creates a texture called \u201clayer grain\u201d in parallel to the face it was printed on. Please be aware that the strength of 3D printed parts are weakest in the adhesion between this layer grain (it\u2019s very similar to wood in this way). Additionally, just like working with a hot glue gun, 3D prints may have some very mild stringing (appears like stray spider threads) that we missed when cleaning the print. We're sorry if we missed them! Bring Jurassic Park straight to your bathroom with this 3D printed, water-spitting dino! Features a detailed dilophosaurus skull with a wide, contoured frill and built-in shower head fitting complete with 87 nozzles, printed in sturdy ABS plastic. This product includes one dilophosaurus shower head and 6 inches of Teflon pipe thread tape to seal the connection between the shower head and the shower arm. Dimensions: This shower head is ~6.5 inches (16.5 cm) across the span of the frills. Note on color choices: Our default is to finish this product with a coat of beautiful, bright copper, pewter or chrome spray paint for an overall cohesive and modern style. If however, you would like to customize your color choices, we would be happy to accommodate you. We can print in a fairly wide array of ABS plastic colors, and also have other spray paints available too. We can even hand paint your print to your custom specifications for a small additional fee, if you would prefer. Please select \"custom\" at checkout if you want something beyond the three default colors and then contact us to specify what color options you would like. *Many of our products can be personalized with names, dates, short phrases, etc. to meet your needs. Please see our \u201cAbout\u201d section for additional details. ************************************************************************************************************** At StarkContrasts, we select the right plastic for each product and carefully dial in our machine settings to get you the best quality we can. The technology of 3D printing is amazing, but it's not perfect! To work, 3D printers lay down extruded plastic in thin precise layers which creates a texture called \u201clayer grain\u201d in parallel to the face it was printed on. Please be aware that the strength of 3D printed parts are weakest in the adhesion between this layer grain (it\u2019s very similar to wood in this way). Additionally, just like working with a hot glue gun, 3D prints may have some very mild stringing (appears like stray spider threads) that we missed when cleaning the print. We're sorry if we missed them! + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/kenmac138 \nRating: 4 \nDate: Feb 23, 2019 \nContent: I wished it were a little larger but its kitschy decor.serves its purpose and looks neat\n\nReview number: 2 \nProfile: www.etsy.com/people/sahanes \nRating: 5 \nDate: Apr 13, 2018 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/bjf7uij4 \nRating: 1 \nDate: Jan 4, 2018 \nContent: What a cheap piece of junk! First off it\u2019s way smaller then shown, second the threads on the head aren\u2019t the correct size and won\u2019t screw on. Third trying to screw it on the bottom jaw broke fall because it\u2019s so small you can\u2019t get you hand on it and it\u2019s so poorly made it couldn\u2019t take the twisting. Please don\u2019t waste your money on this it will only disappoint the kids when it doesn\u2019t work.\n\nReview number: 4 \nProfile: www.etsy.com/people/9halrdts \nRating: 5 \nDate: Mar 13, 2019 \nContent: amazing quality and beautifully painted. ordered mid december and recieved it early march, but it was totally worth the wait! definitely going to order two more pairs of my other cats!"]}, 11 | {"title": "3D Print - Iron Giant inspired figure - 16'' tall and eyes light up red or green!", "product_id": "649894067", "url": "www.etsy.com/listing/649894067", "price": "594.49", "rating": 4.93, "number_of_votes": "37", "product_options": "", "count_of_images": 5, "favorited_by": "90", "return_location": "United States", "description": "\u2022 This figure is made of PLA and airbrushed with a light metallic silver. \u2022 This figure stands 16'' tall and has 9 points of articulation, the finished piece will come with a doll stand for display. \u2022 This figure is powered by 1 CR2032 which will be easy to change in the future. \u2022 This figure has a sigle bi-color LED (red/green) with a 3 way switch so you can choose either color to diplay This is a display piece, it is meant to be posed and left but most of the parts are removable if they get damaged. If you have a damaged piece let me know and I\u2019ll tell you if I can send a replacement part out. If the damaged part can not be easily replaced then message me anyways and I\u2019ll see if there\u2019s something I can do. PLA is sturdy in general but to make an interactive design as close to the original as possible, compromises needed to be made in both the structure and design. - This piece is delicate and requires a bit of care to manipulate, it may not be suitable for children as there are also small pieces. I also offer design service for custom pieces! Message me if you have any questions or special requests :) All of the 3d prints that I offer have been designed by me. \u2022 This figure is made of PLA and airbrushed with a light metallic silver. \u2022 This figure stands 16'' tall and has 9 points of articulation, the finished piece will come with a doll stand for display. \u2022 This figure is powered by 1 CR2032 which will be easy to change in the future. \u2022 This figure has a sigle bi-color LED (red/green) with a 3 way switch so you can choose either color to diplay This is a display piece, it is meant to be posed and left but most of the parts are removable if they get damaged. If you have a damaged piece let me know and I\u2019ll tell you if I can send a replacement part out. If the damaged part can not be easily replaced then message me anyways and I\u2019ll see if there\u2019s something I can do. PLA is sturdy in general but to make an interactive design as close to the original as possible, compromises needed to be made in both the structure and design. - This piece is delicate and requires a bit of care to manipulate, it may not be suitable for children as there are also small pieces. I also offer design service for custom pieces! Message me if you have any questions or special requests :) All of the 3d prints that I offer have been designed by me. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/shantaynelson1 \nRating: 5 \nDate: Nov 16, 2018 \nContent: great detail and just so dang awesome! this was for my son and was like a dream he was so excited. the changing eye color is the perfect touch. More delicate than I thought for play, more of a display item but still perfect for the iron giant lover! Shipped quickly and great communication.\n\nReview number: 2 \nProfile: www.etsy.com/people/hlqultae \nRating: 5 \nDate: Mar 9, 2019 \nContent: What else can I say about this project that was made for me. Based on my favorite movie, the iron giant, Hogarths ray gun from the film is accurate and awsome! Item was made very well, and was made fast and shipped fast as well! The seller took the the time to make sure every detail was on point and was very accurate to the film. He takes the time to make sure every project he creates meets the satisfaction of the customers needs. And gurss what, of course he meet my expectations and more! I highly recommend this seller and can\u2019t say enough about this project. I can\u2019t wait to add this to my iron giant pieces and have family and friends mention we\u2019re you can buy this, but you can\u2019t, so if you have a project from a film or want something that is a one of a kind piece, then look no further, and get your piece made from him today!\n \n \n \n What else can I say about this project that was made for me. Based on my favorite movie, the iron giant, Hogarths ray gun from the film is accurate and awsome! Item was made very well, and was made fast and shipped fast as well! The seller took the the time to make sure every detail was on point and was very accurate to the film. He takes the time to make sure every project he creates meets the satisfaction of the customers needs. And gurss what, of course he meet my expectations and more! I highly recommend this seller and can\u2019t say enough about this project. I can\u2019t wait to add...\n \n more\n\nReview number: 3 \nProfile: www.etsy.com/people/steinfly1992 \nRating: 5 \nDate: Feb 26, 2019 \nContent: shipped fast looks great glade to have a piece of my childhood.\n\nReview number: 4 \nProfile: www.etsy.com/people/woodeNES \nRating: 5 \nDate: Feb 26, 2019 \nContent: As hoped for. Can't wait to finish it off!"]}, 12 | {"title": "3D Printed Ceramic Twisting Triangular Vase, White, 12\" Tall Large Vase, elegant gift, modern home decor", "product_id": "647973531", "url": "www.etsy.com/listing/647973531", "price": "574.71", "rating": 5.0, "number_of_votes": "1", "product_options": "", "count_of_images": 3, "favorited_by": "4", "return_location": "United States", "description": "The elegant triangular form of this vase twists, creating a sense of movement. The surface features an intricate woven pattern that creates a subtle texture. It is a statement piece whether displayed alone or with a flower arrangement. Since each piece is 3D printed individually in a natural material, and then refined and glazed by hand, slight variations in the surface and glazing are part of its beauty. Like any handcrafted item, no two are exactly alike. Made from stoneware ceramics, it is available in two finish options. Flowers not included. 4.75in x 5in x 12in", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/dfiros \nRating: 5 \nDate: Oct 13, 2018 \nContent: I was so excited when this arrived! Really unique texture and shape that I'm loving on my desk, even without flowers. \n \nAnd I love that it's 3D printed in clay and not plastic. I haven't seen that anywhere else!"]}, 13 | {"title": "Settlers of Catan Pieces 3D Print Base Set Ships Seafarers Catan Holder Card Holder Replacement Pieces Complete Set", "product_id": "585286745", "url": "www.etsy.com/listing/585286745", "price": "20.85", "rating": 4.96, "number_of_votes": "319", "product_options": "Select a color: Black, Gray, White, Transparent - White, Transparent - Blue, Blue, Turquoise, Purple, Red, Pink, Hot Pink, Orange, Yellow, Green, Other - Add note|Select an option: Base Set (BRL 20.85), Base Set w/Ships (BRL 37.10), Ships only (BRL 20.85), Game Holder Standard (BRL 21.05), Card Tray Pair (BRL 37.10)", "count_of_images": 7, "favorited_by": "59", "return_location": "United States", "description": "Spring Sale! No Minimum required! These are 3D Printed Catan pieces made with PLA filament. Base Set includes: 15 Road Pieces 5 Settlement Pieces 4 City Pieces w/Ships: 15 Ship Pieces Card Tray Pair: 2 Card Trays (3 Card Slots per Tray) Pieces will come with a FREE bag or packet to protect the pieces and to prevent losing one. Check out my shop for other sets and colors! https://www.etsy.com/shop/TheHotlineBling Catan matching color pouches: https://www.etsy.com/listing/510731045/ Wooden sets: Base set: https://www.etsy.com/listing/254210178/ Seafarers set: https://www.etsy.com/listing/521064622/ Cities and Knights set: https://www.etsy.com/listing/484059368/ Complete set with expansions: https://www.etsy.com/listing/534929889/ These are 3D Printed Catan pieces made with PLA filament. Base Set includes: 15 Road Pieces 5 Settlement Pieces 4 City Pieces w/Ships: 15 Ship Pieces Card Tray Pair: 2 Card Trays (3 Card Slots per Tray) Pieces will come with a FREE bag or packet to protect the pieces and to prevent losing one. Check out my shop for other sets and colors! https://www.etsy.com/shop/TheHotlineBling Catan matching color pouches: https://www.etsy.com/listing/510731045/ Wooden sets: Base set: https://www.etsy.com/listing/254210178/ Seafarers set: https://www.etsy.com/listing/521064622/ Cities and Knights set: https://www.etsy.com/listing/484059368/ Complete set with expansions: https://www.etsy.com/listing/534929889/ + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/stephanieb1977 \nRating: 5 \nDate: Feb 9, 2019 \nContent: This shipped very quickly and the pieces are very well made. We will definitely order again!!!\n\nReview number: 2 \nProfile: www.etsy.com/people/NichelleTamra \nRating: 5 \nDate: Jan 27, 2019 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/NichelleTamra \nRating: 5 \nDate: Jan 27, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/NichelleTamra \nRating: 5 \nDate: Jan 27, 2019 \nContent: "]}, 14 | {"title": "Stealth 3D Printed clutch", "product_id": "577864852", "url": "www.etsy.com/listing/577864852", "price": "1,953.89", "rating": 5.0, "number_of_votes": "9", "product_options": "", "count_of_images": 6, "favorited_by": "22", "return_location": "Singapore", "description": "Geometric 3D printed clutch is a finalist of prestigious jewelry and accessories competition by GreenBird and Swarovski. The main body is 3D printed in nylon with all the moving parts and hinges. It is finished with the Swarovski crystals. The clutch is very light. This is a perfect accessory for the evening events and even though nylon is a flexible and durable material - the clutch should be handled with care. Please keep in mind that this item is made to order and it will take 5 weeks to 3D print, fix the crystals and ship it to you.", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/etsyweasel \nRating: 5 \nDate: Jan 11, 2019 \nContent: Thank you very much--she loves it!\n\nReview number: 2 \nProfile: www.etsy.com/people/eleanor35 \nRating: 5 \nDate: Jan 8, 2019 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/arianave88 \nRating: 5 \nDate: Apr 23, 2018 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/btsecker \nRating: 5 \nDate: Dec 5, 2017 \nContent: Love this ring - can't wait to wear it. Thank you for the great service :-)"]}, 15 | {"title": "Mini DEC PDP-8/I - 3D Printed!", "product_id": "582672678", "url": "www.etsy.com/listing/582672678", "price": "39.64", "rating": 4.99, "number_of_votes": "720", "product_options": "", "count_of_images": 2, "favorited_by": "9", "return_location": "United States", "description": "This tiny, 2\" Digital Equipment Corporation PDP-8/I was designed using open source CAD software, 3D printed on a Makerbot Replicator, and hand painted and assembled. A great addition to your desk or studio. Includes one computer. Includes tiny fact sheet with trivia on the original machine. Not suitable for ages 5 and under due to small parts. More information at www.rabbitengineering.com", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/fortyseven \nRating: 5 \nDate: Apr 24, 2018 \nContent: Excellent work as always! Almost expect it to actually turn on. ;)\n\nReview number: 2 \nProfile: www.etsy.com/people/jshapiro73 \nRating: 5 \nDate: Mar 10, 2019 \nContent: Another Home Run from Rabbit Engineering.\n\nReview number: 3 \nProfile: www.etsy.com/people/HopeKilledKraken \nRating: 5 \nDate: Mar 5, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/4mikkydi \nRating: 5 \nDate: Mar 2, 2019 \nContent: This little NeXTcube is very small but impressively detailed. I love it! It shipped fairly quickly, and arrived extremely well-packed. It's a tiny work of art -- the combined pieces won't even cover a typical business card. Overall it's a fantastic product if you're into NeXT stuff. I wish they offered the NeXT Laser Printer as part of the set."]}, 16 | {"title": "Wood Plastic 3D Printed Succulent Planters", "product_id": "616167044", "url": "www.etsy.com/listing/616167044", "price": "67.38", "rating": 4.84, "number_of_votes": "67", "product_options": "Select an option: Squatty Spiral, Geometirc, Tall Cylinder, Slanted Spiral", "count_of_images": 5, "favorited_by": "13", "return_location": "United States", "description": "Succulent planters that were crafted with CAD (computer aided design) software and then 3D printed out of plastic! These are our Wood plastic planters. That's right! It is made from a plastic with wood fibers so it is sand-able and stain-able! (See our \"Assorted 3D Printed Succulent Planters\" listing for more planter colors.) Pick a color for your Squatty Spiral Planter --> Then, go find yourself a succulent to plant in to one of these bad boys!! **Need all 4 of the planters to make a set? Message for pricing.", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/kayleighpetramale \nRating: 5 \nDate: Mar 10, 2019 \nContent: If I could give this shop 500 stars I would. Extremely easy to work with despite my error in address. I look forward to using this product.\n\nReview number: 2 \nProfile: www.etsy.com/people/armiller81 \nRating: 5 \nDate: Mar 5, 2019 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/armiller81 \nRating: 5 \nDate: Mar 5, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/amandagarcia353 \nRating: 5 \nDate: Mar 4, 2019 \nContent: Was very helpful and it came so quick first was damaged during transit and they sent a replacement immediately! Thank you so much great product and customer service."]}, 17 | {"title": "Amulet of Daylight / Eclipse, 3d Printed, rechargeable LED, Unofficial.", "product_id": "502146914", "url": "www.etsy.com/listing/502146914", "price": "305.42", "rating": 4.91, "number_of_votes": "233", "product_options": "Select a color: Black/Red with Gold (BRL 342.29), Black/Red (BRL 315.96), Silver/Blue + Gold (BRL 342.29), Silver/Blue (BRL 315.96), Chrome/Blue (BRL 305.42), Silver/Red (BRL 315.96), Silver/Red with Gold (BRL 342.29)|Select a style: Troll Language, English", "count_of_images": 10, "favorited_by": "814", "return_location": "United Kingdom", "description": "Choose from the Amulet of Daylight Toy (Silver & Blue or Chrome) and/or the Amulet of Eclipse Toy (Black & Red): 3d Printed & hand finished to order! A great gift idea or collectible item for all fans out there \ud83d\ude03 Now daylight is yours to command! This Amulet has a push button on the back to turn the LED\u2019s on and off and a USB charging port on the bottom. The arms can be rotated. USB cable Included. 4 Light settings: 3 brightness settings and flashing mode. Each press of the power button will cycle through the brightness's. A long press of the power button will make the Amulet flash. ***Please note to only charge the amulet via a laptop/PC USB ports or portable power banks to avoid the charging cable overloading. 5 volts maximum.*** The options shown in the photos are the painted silver/blue + gold and the painted black/red + gold options. Outer ring diameter is 90mm. 3D printed in PLA plastic *EPILEPSY WARNING* *FLASHING LIGHTS* *FREE UK POSTAGE* All items are shipped tracked Technical Info: All of my items are 3D printed on Ultimaker printers using PLA Filaments. I mostly print items in parts and assemble for the best quality and finish each product by hand using acrylic sprays and or paints. All my items are designed by myself. Any design I do can be altered if there is something specific you would like changing, just drop me a message. All of my work is a representations of the items they portray. They are not official replicas or copies and I have no affiliation with the companies that originally designed them. My items are simply created for you to enjoy and celebrate the things you like. What is a 3d printed item?: 3D printing or additive manufacturing is a process of making three dimensional solid objects from a digital file. The creation of a 3D printed object is achieved using additive processes. In an additive process an object is created by laying down successive layers of material until the object is created. Each of these layers can be seen as a thinly sliced horizontal cross-section of the eventual object. Licensing Info: Unpainted items: The unpainted options to most items in my store are created on a 3d printer but do not undergo any of the final processing that the painted item does. The item is printed, removed from the printer and assembled only. Any STL files purchased from me are done so under a non commercial and non share licence: Non-Commercial \u2014 You may not use the material for commercial purposes. NoDerivatives \u2014 If you remix, transform, or build upon the material, you may not distribute the modified material. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits. Choose from the Amulet of Daylight Toy (Silver & Blue or Chrome) and/or the Amulet of Eclipse Toy (Black & Red): 3d Printed & hand finished to order! A great gift idea or collectible item for all fans out there \ud83d\ude03 Now daylight is yours to command! This Amulet has a push button on the back to turn the LED\u2019s on and off and a USB charging port on the bottom. The arms can be rotated. USB cable Included. 4 Light settings: 3 brightness settings and flashing mode. Each press of the power button will cycle through the brightness's. A long press of the power button will make the Amulet flash. ***Please note to only charge the amulet via a laptop/PC USB ports or portable power banks to avoid the charging cable overloading. 5 volts maximum.*** The options shown in the photos are the painted silver/blue + gold and the painted black/red + gold options. Outer ring diameter is 90mm. 3D printed in PLA plastic *EPILEPSY WARNING* *FLASHING LIGHTS* *FREE UK POSTAGE* All items are shipped tracked Technical Info: All of my items are 3D printed on Ultimaker printers using PLA Filaments. I mostly print items in parts and assemble for the best quality and finish each product by hand using acrylic sprays and or paints. All my items are designed by myself. Any design I do can be altered if there is something specific you would like changing, just drop me a message. All of my work is a representations of the items they portray. They are not official replicas or copies and I have no affiliation with the companies that originally designed them. My items are simply created for you to enjoy and celebrate the things you like. What is a 3d printed item?: 3D printing or additive manufacturing is a process of making three dimensional solid objects from a digital file. The creation of a 3D printed object is achieved using additive processes. In an additive process an object is created by laying down successive layers of material until the object is created. Each of these layers can be seen as a thinly sliced horizontal cross-section of the eventual object. Licensing Info: Unpainted items: The unpainted options to most items in my store are created on a 3d printer but do not undergo any of the final processing that the painted item does. The item is printed, removed from the printer and assembled only. Any STL files purchased from me are done so under a non commercial and non share licence: Non-Commercial \u2014 You may not use the material for commercial purposes. NoDerivatives \u2014 If you remix, transform, or build upon the material, you may not distribute the modified material. No additional restrictions \u2014 You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/gevwp2ws \nRating: 5 \nDate: Mar 6, 2019 \nContent: my nephew loves it\n\nReview number: 2 \nProfile: www.etsy.com/people/ki1w4orh \nRating: 5 \nDate: Feb 2, 2019 \nContent: \n\nReview number: 3 \nProfile: www.etsy.com/people/elodielacassagne \nRating: 4 \nDate: Jan 23, 2019 \nContent: Tr\u00e8s bon article, r\u00e9aliste mais un peu gros pour la main des enfants.\n\nReview number: 4 \nProfile: www.etsy.com/people/ericaadams2009 \nRating: 5 \nDate: Jan 4, 2019 \nContent: "]}, 18 | {"title": "3D Printed Toothbrush Rug Needle", "product_id": "643643947", "url": "www.etsy.com/listing/643643947", "price": "15.85", "rating": 4.98, "number_of_votes": "2919", "product_options": "Select a color: Pink [Sold out, waitlist available], Yellow [Sold out, waitlist available], Green [Sold out, waitlist available], Red, Purple [Sold out, waitlist available], Blue, Orange [Sold out, waitlist available], Light Blue, White, Black, GITD-Blue [Sold out, waitlist available], GITD-Green", "count_of_images": 4, "favorited_by": "23", "return_location": "United States", "description": "The toothbrush rug needles are 4\" in length, 1/8\" thick. The eye of the needle is 3/16\" wide by 5/8\" long. The needles are printed from Polylactic acid (PLA). PLA is derived from renewable resources like cornstarch and tapioca roots and is biodegradable. The only trade off for not sitting in a landfill for future super intelligent apes to find is the glass transition temperature of PLA (temperature where the plastic gets soft) is 60\u201365 \u00b0C. (140-149 \u00b0F). So do not keep them in your oven, your car in the summer or anywhere that you would not want to hang out.", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/NanaTat4Life \nRating: 5 \nDate: Feb 10, 2019 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/mmjenny8281 \nRating: 5 \nDate: Dec 4, 2018 \nContent: This well made needle has big hole, size is also very easy to use. It is fun to chose a lot of colors! Thank you very much!\n\nReview number: 3 \nProfile: www.etsy.com/people/mmjenny8281 \nRating: 5 \nDate: Dec 4, 2018 \nContent: This well made needle has big hole, size is also very easy to use. It is fun to chose a lot of colors! Thank you very much!\n\nReview number: 4 \nProfile: www.etsy.com/people/mmjenny8281 \nRating: 5 \nDate: Dec 4, 2018 \nContent: This well made needle has big hole, size is also very easy to use. It is fun to chose a lot of colors! Thank you very much! \nAnd thank you very much for your cute service! A cute group of sheep was made at my house \u266a"]}, 19 | {"title": "ORC/OGRE Dildo Customizable 3D printed 6 inch Vaginal or Anal Pleasure NerdClimax, sexy time, Bedroom Toys, Harness, Fantasy dildo", "product_id": "554003162", "url": "www.etsy.com/listing/554003162", "price": "148.59", "rating": 3.94, "number_of_votes": "48", "product_options": "Select a width: 1.7 inches (BRL 148.59 - BRL 198.18), 2.2 inches (BRL 200.12 - BRL 237.81)|Select a color: White (BRL 148.59 - BRL 200.12), Black (BRL 148.59 - BRL 200.12), Beige (BRL 148.59 - BRL 200.12), Brown (BRL 148.59 - BRL 200.12), Gray (BRL 148.59 - BRL 200.12), Green (BRL 148.59 - BRL 200.12), Pink (BRL 148.59 - BRL 200.12), Purple (BRL 148.59 - BRL 200.12), Red (BRL 148.59 - BRL 200.12), Silver (BRL 148.59 - BRL 200.12), GlowInTheDark (BRL 176.34 - BRL 223.90), GlowInTheDark Blue (BRL 176.34 - BRL 223.90), Temperature changer (BRL 198.18 - BRL 237.81)", "count_of_images": 9, "favorited_by": "132", "return_location": "United States", "description": "This is Hard Objects if your looking for a silicone version, please go back and click the silicone version of this object. NerdClimax We recommend if you are looking for a more realistic Feel please request in silicone as this product is hard plastic. Thank you ORC/OGRE Dildo Customizable 3D printed 6 inches Vaginal or Anal Pleasure NerdClimax, sexy time, Bedroom Toys, Harness, Fantasy dildo Bigger is Better! Imagine becoming taken over by an Orc/Ogre and all they want to do is pleasure you fully; Don't miss this amazing Orc/Ogre Dildo that you can feel the dildo veins as it slides down. Now pick your own color to customize every aspect of your toy. Now harness Accessible just purchase the dildo you need and email us what size your O ring is and we will take it from there. If you would like to request a color please let us know and we will do our best to make it! Shipping : Please understand if you select two day shipping this means that it's two days after the item is created. Thank you for understanding and sorry for any inconvenience is this may have caused. All items come to you safe and secure and most important all items come in unmark packages they just say from a Nerd! Thank you for checking us out and check some of our other products out as well as stay tuned for new NerdClimax Please understand that this item although is very strong it still must well be maintained to be stable. This is Hard Objects if your looking for a silicone version, please go back and click the silicone version of this object. NerdClimax We recommend if you are looking for a more realistic Feel please request in silicone as this product is hard plastic. Thank you ORC/OGRE Dildo Customizable 3D printed 6 inches Vaginal or Anal Pleasure NerdClimax, sexy time, Bedroom Toys, Harness, Fantasy dildo Bigger is Better! Imagine becoming taken over by an Orc/Ogre and all they want to do is pleasure you fully; Don't miss this amazing Orc/Ogre Dildo that you can feel the dildo veins as it slides down. Now pick your own color to customize every aspect of your toy. Now harness Accessible just purchase the dildo you need and email us what size your O ring is and we will take it from there. If you would like to request a color please let us know and we will do our best to make it! Shipping : Please understand if you select two day shipping this means that it's two days after the item is created. Thank you for understanding and sorry for any inconvenience is this may have caused. All items come to you safe and secure and most important all items come in unmark packages they just say from a Nerd! Thank you for checking us out and check some of our other products out as well as stay tuned for new NerdClimax Please understand that this item although is very strong it still must well be maintained to be stable. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/bqvt1sle \nRating: 5 \nDate: Mar 5, 2019 \nContent: Exactly what i wanted. Great customer service.\n\nReview number: 2 \nProfile: www.etsy.com/people/bqvt1sle \nRating: 4 \nDate: Feb 27, 2019 \nContent: The size is ok. Could be a little softer.\n\nReview number: 3 \nProfile: www.etsy.com/people/PossumKingg \nRating: 5 \nDate: Feb 20, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/SynjoDeonecros \nRating: 3 \nDate: Feb 6, 2019 \nContent: It took a while to ship, getting stuck in Indianapolis for a while, but I'm not going to complain about it. My main issues are the alterations needed to make it printable (which aren't as unsightly as it could've been, and gives it a more alien look, but still, principle of the thing), the fact that it's very firm (much firmer than I'd like, and - given the following complaint, makes chaffing a problem), and it wasn't sanded down at all, meaning I can see and feel the texture. It does glow rather well, and it's still very usable as a dildo, just not as good as I thought it would be. Maybe if I had been able to pay for it at a larger size, it could've been better, and I admit, I didn't specify a firmness, so the full hardness isn't their fault, so I won't be asking for a refund, but it's neither great nor bad, just fine as-is.\n \n \n \n It took a while to ship, getting stuck in Indianapolis for a while, but I'm not going to complain about it. My main issues are the alterations needed to make it printable (which aren't as unsightly as it could've been, and gives it a more alien look, but still, principle of the thing), the fact that it's very firm (much firmer than I'd like, and - given the following complaint, makes chaffing a problem), and it wasn't sanded down at all, meaning I can see and feel the texture. It does glow rather well, and it's still very usable as a dildo, just not as...\n \n more"]}, 20 | {"title": "V-Buck Pack 3D Printed with Glow-in-the-Dark Inserts | Fortnite", "product_id": "649121701", "url": "www.etsy.com/listing/649121701", "price": "21.06", "rating": 4.89, "number_of_votes": "28", "product_options": "Select an option: Pack of Two (BRL 40.02), Pack of Three (BRL 60.03), Pack of Four (BRL 80.04), Pack of Five (BRL 100.05), Pack of Six (BRL 120.06), Pack of Seven (BRL 140.07), Pack of Eight (BRL 160.08), Pack of Nine (BRL 180.09), Pack of Ten (BRL 200.11), Single (Unpacked) (BRL 21.06)", "count_of_images": 6, "favorited_by": "74", "return_location": "United Kingdom", "description": "Vindertech Bucks, or V-Bucks for short, are the premium currency of Fortnite - Epic Game's amazing battle royale game that's taken the world by storm. V-Bucks are also packed in a special, Fortnite Bag similar to how they are in-game! These V-Bucks are the perfect addition to any Fortnite Fan's collection! They're excellent as party favours, Christmas stocking fillers or birthday presents! These V-Bucks are printed in 5 different types of PLA plastic. This is printed as one solid piece using multi-material printing methods. The glow-in-the-dark strength may vary, and is best when charged using UV or daylight. Every V-Buck is 3D printed to order. * Diameter: 40mm * Weight: 7g * Materials Used: PLA (polylactide) plastic. This is not official merchandise, but fan-created content. Default shipping method is Royal Mail 2nd Class (Domestic, Free) or Royal Mail International (tracked). I advise customers to purchase the tracking upgrade to ensure your order's safe delivery, as I cannot be held responsible for lost parcels. Standard shipping to US & Canada can take between 2 and 4 weeks. Shipping times vary and are not guaranteed. As this product is 3D printed, every attempt to reduce quality issues has been taken. However, due to the nature of 3D printing, some issues may still be present. Vindertech Bucks, or V-Bucks for short, are the premium currency of Fortnite - Epic Game's amazing battle royale game that's taken the world by storm. V-Bucks are also packed in a special, Fortnite Bag similar to how they are in-game! These V-Bucks are the perfect addition to any Fortnite Fan's collection! They're excellent as party favours, Christmas stocking fillers or birthday presents! These V-Bucks are printed in 5 different types of PLA plastic. This is printed as one solid piece using multi-material printing methods. The glow-in-the-dark strength may vary, and is best when charged using UV or daylight. Every V-Buck is 3D printed to order. * Diameter: 40mm * Weight: 7g * Materials Used: PLA (polylactide) plastic. This is not official merchandise, but fan-created content. Default shipping method is Royal Mail 2nd Class (Domestic, Free) or Royal Mail International (tracked). I advise customers to purchase the tracking upgrade to ensure your order's safe delivery, as I cannot be held responsible for lost parcels. Standard shipping to US & Canada can take between 2 and 4 weeks. Shipping times vary and are not guaranteed. As this product is 3D printed, every attempt to reduce quality issues has been taken. However, due to the nature of 3D printing, some issues may still be present. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/charliebeesnees \nRating: 5 \nDate: Dec 9, 2018 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/xuzeuqa0 \nRating: 5 \nDate: Mar 10, 2019 \nContent: Super satisfied ! item arrived earlier than expected, Overall well packaged and looks very good even if its not painted i would suggest this to any fan of titanfall and APEX legends\n\nReview number: 3 \nProfile: www.etsy.com/people/vpchqvxf \nRating: 5 \nDate: Mar 1, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/luxibuxc \nRating: 4 \nDate: Feb 18, 2019 \nContent: Nice quality, just a slight little issue at the bottom where the 3d printing didn't cover it all. But still very good :)"]}, 21 | {"title": "Brain Lamp - rechargeable - water proof - includes remote control- 3D Printed", "product_id": "642330385", "url": "www.etsy.com/listing/642330385", "price": "313.12", "rating": 4.86, "number_of_votes": "145", "product_options": "", "count_of_images": 10, "favorited_by": "68", "return_location": "United States", "description": "3D Printed Brain Lamp Printed in clear Earth friendly PLA plastic Design Made from an actual MRI brain scan LED light inside can change to 16 different colors and 4 different transition modes. You can also control brightness. Includes remote control and charging cable Has 8-10 hr battery and is rechargeable Originally based on this model, then modified by me: https://www.thingiverse.com/thing:620846", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/wonderwomen2 \nRating: 5 \nDate: Mar 13, 2019 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/girgr6qd \nRating: 5 \nDate: Mar 4, 2019 \nContent: Great design and articulation. Fine work.\n\nReview number: 3 \nProfile: www.etsy.com/people/SamuraiDiva \nRating: 5 \nDate: Feb 24, 2019 \nContent: \n\nReview number: 4 \nProfile: www.etsy.com/people/taylorkilgore5 \nRating: 5 \nDate: Feb 22, 2019 \nContent: Really cool item, thank you"]}, 22 | {"title": "Men in Black, Noisy Cricket, Cos-play Prop , Replica, Model Kit, 3d printed", "product_id": "640839739", "url": "www.etsy.com/listing/640839739", "price": "76.12", "rating": 5.0, "number_of_votes": "2", "product_options": "", "count_of_images": 5, "favorited_by": "33", "return_location": "Slovenia", "description": "This item is a 3D printed version of Men in Blacks Noisy Cricket. A Prop Replica, 3D Printed Model Kit. This is a kit item, meaning that you will receive the parts straight off of the 3D printer, and you will have to assemble and add all of the finishing touches yourself. Parts are printed in PLA (ABS or PETG on request). PLA is biodegradable, lightweight and durable. This is a Replica, Prop and has no working parts. perfect for co-splay, display or kit assembly enthusiast --TERMS-- 3D printed props and prop kits are sold with a zero return, zero refund policy. 3d printed items, even finished items, will have small imperfections, period. Photos show you exactly what you're getting and please understand that while fairly durable, these props are meant as display items, and can be scratched or broken if mishandled. Parts come straight from the print-bed every print is a one of a kind but dimensional identical. This item normally ships in 3-5 business days, but may take four on occasion if workload is heavy. Also, kits may come in colors other than what is pictured. They are printed using whatever color filament we have on hand at the time. This item is a 3D printed version of Men in Blacks Noisy Cricket. A Prop Replica, 3D Printed Model Kit. This is a kit item, meaning that you will receive the parts straight off of the 3D printer, and you will have to assemble and add all of the finishing touches yourself. Parts are printed in PLA (ABS or PETG on request). PLA is biodegradable, lightweight and durable. This is a Replica, Prop and has no working parts. perfect for co-splay, display or kit assembly enthusiast --TERMS-- 3D printed props and prop kits are sold with a zero return, zero refund policy. 3d printed items, even finished items, will have small imperfections, period. Photos show you exactly what you're getting and please understand that while fairly durable, these props are meant as display items, and can be scratched or broken if mishandled. Parts come straight from the print-bed every print is a one of a kind but dimensional identical. This item normally ships in 3-5 business days, but may take four on occasion if workload is heavy. Also, kits may come in colors other than what is pictured. They are printed using whatever color filament we have on hand at the time. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/3byf7cmf \nRating: 5 \nDate: Oct 16, 2018 \nContent: parfait, tr\u00e8s r\u00e9aliste. merci beaucoup\n\nReview number: 2 \nProfile: www.etsy.com/people/jrpalone \nRating: 5 \nDate: Oct 16, 2018 \nContent: Very well made item. Was delivered promptly and safely"]}, 23 | {"title": "3D-Printed Stellated Dodecahedron Pendant", "product_id": "684618991", "url": "www.etsy.com/listing/684618991", "price": "79.27", "rating": 5.0, "number_of_votes": "6", "product_options": "Select a material: 14k Gold Plated (BRL 138.72 - BRL 190.25), 14k Rose Gold Plated (BRL 138.72 - BRL 190.25), 18k Gold Plated (BRL 138.72 - BRL 190.25), Rhodium Plated (BRL 138.72 - BRL 190.25), Natural Silver (BRL 158.54 - BRL 210.07), Polished Silver (BRL 178.36 - BRL 229.89), Natural Brass (BRL 99.09 - BRL 150.61), Polished Brass (BRL 118.91 - BRL 170.43), Natural Bronze (BRL 99.09 - BRL 150.61), Polished Bronze (BRL 118.91 - BRL 170.43), Gold Plated Steel (BRL 79.27 - BRL 130.80), Nickel Steel (BRL 79.27 - BRL 130.80), Bronze Steel (BRL 79.27 - BRL 130.80), Black Steel (BRL 79.27 - BRL 130.80)|Select an option: Silver Chain 16 inch (BRL 110.98 - BRL 210.07), Silver Chain 20 inch (BRL 118.91 - BRL 217.99), Gold Chain 16 inch (BRL 110.98 - BRL 210.07), Gold Chain 20 inch (BRL 118.91 - BRL 217.99), Black Satin Cord30in (BRL 79.27 - BRL 178.36), SilverBeadChain20in (BRL 118.91 - BRL 217.99), SilverBeadChain24in (BRL 130.80 - BRL 229.89), SilverBevelChain16in (BRL 91.16 - BRL 190.25), SilverBevelChain20in (BRL 99.09 - BRL 198.18)", "count_of_images": 8, "favorited_by": "12", "return_location": "Canada", "description": "A small 20mm wide Stellated Dodecahedron Pendant available in a variety of precious metals & finishes. A 30-inch black Satin Cord is included if you do not select a chain option. Chain options: - Silver Beveled Cable Chain 16 inches/40.6cm - Silver Beveled Cable Chain 20 inches/50.8cm - Silver Bead Chain 20 inches/50.8cm - Silver Bead Chain 24 inches/60.9cm - Silver Round Cable Chain 16 inches/40.6cm - Silver Round Cable Chain 20 inches/50.8cm -Yellow Gold-Filled Cable Chain 16 inches/40.6cm -Yellow Gold-Filled Cable Chain 20 inches/50.8cm - Black Satin Cord 30 inches/70.6cm Pendant metals & finishes: - Natural Silver - Polished Silver - Rhodium Plated Brass - 14k Gold Plated Brass - 18k Gold Plated Brass - 14k Rose Gold Plated Brass - Natural Brass - Polished Brass - Natural Bronze - Polished Bronze - Polished Gold Plated Steel - Polished Nickel Steel - Polished Black Steel - Polished Bronze Steel Please allow 1 -2 weeks for this item to be created and shipped. A small 20mm wide Stellated Dodecahedron Pendant available in a variety of precious metals & finishes. A 30-inch black Satin Cord is included if you do not select a chain option. Chain options: - Silver Beveled Cable Chain 16 inches/40.6cm - Silver Beveled Cable Chain 20 inches/50.8cm - Silver Bead Chain 20 inches/50.8cm - Silver Bead Chain 24 inches/60.9cm - Silver Round Cable Chain 16 inches/40.6cm - Silver Round Cable Chain 20 inches/50.8cm -Yellow Gold-Filled Cable Chain 16 inches/40.6cm -Yellow Gold-Filled Cable Chain 20 inches/50.8cm - Black Satin Cord 30 inches/70.6cm Pendant metals & finishes: - Natural Silver - Polished Silver - Rhodium Plated Brass - 14k Gold Plated Brass - 18k Gold Plated Brass - 14k Rose Gold Plated Brass - Natural Brass - Polished Brass - Natural Bronze - Polished Bronze - Polished Gold Plated Steel - Polished Nickel Steel - Polished Black Steel - Polished Bronze Steel Please allow 1 -2 weeks for this item to be created and shipped. + More - Less", "reviews": ["Review number: 1 \nProfile: www.etsy.com/people/jenniferespinosa1717 \nRating: 5 \nDate: Feb 7, 2019 \nContent: \n\nReview number: 2 \nProfile: www.etsy.com/people/TheOldWaysWitchery \nRating: 5 \nDate: Jan 23, 2019 \nContent: Wonderful. Thank you \ud83d\ude0a\n\nReview number: 3 \nProfile: www.etsy.com/people/mapydvfx \nRating: 5 \nDate: Jan 17, 2019 \nContent: Il fornitore del negozio ha tardato, ma il negozio mi ha restituito i soldi spesi e mi ha inviato comunque il ciondolo: ringrazio dunque Matthew Chin e la 3DDreamworldDesigns. \n \n \nENG: The store's vendor was late, but the store returned the money and sent me the pendant anyway: I thank Matthew Chin and 3DDreamworldDesigns.\n\nReview number: 4 \nProfile: www.etsy.com/people/tenshihidenka \nRating: 5 \nDate: Jan 15, 2019 \nContent: Matthew was absolutely amazing! I cannot express enough the craftsmanship and even more, the patience. I asked him to recreate a swivel fob pendant and then 3d print it in sterling silver. Not only did he (impressively!) succeed in doing so, but he was also extraordinarily patient and answered any and all questions I had every step of the way. In addition to his angelic patience, he created two models of the swivel fob, one with a more secure swivel mechanism and one that was exactly like the original. He also 3d printed both in simple plastic first to check that the swivel mechanism works and sent them to me for my approval before printing the sterling silver piece. I had hoped to upload a photo of the completed swivel fob pendant with both stones in place but I will update it later when the stones are set. A* service!\n \n \n \n Matthew was absolutely amazing! I cannot express enough the craftsmanship and even more, the patience. I asked him to recreate a swivel fob pendant and then 3d print it in sterling silver. Not only did he (impressively!) succeed in doing so, but he was also extraordinarily patient and answered any and all questions I had every step of the way. In addition to his angelic patience, he created two models of the swivel fob, one with a more secure swivel mechanism and one that was exactly like the original. He also 3d printed both in simple plastic first to check that the swivel mechanism...\n \n more"]}, 24 | {"title": "Near and Far 3D Printed Organizer", "product_id": "686129419", "url": "www.etsy.com/listing/686129419", "price": "79.23", "product_options": "Select a color: Blue, Green, Red, Yellow, Black, White, Silver, Orange, Translucent, Translucent Green, Translucent Blue", "count_of_images": 10, "return_location": "United States", "description": "3D printed organizers for the board game Near and Far. Ditch plastic bags and messy piles. Cut down on set-up and clean-up time. This organizer has been designed to fit neatly in the Near and Far box. The box insert holds all cards, characters and stands, reputation markers, and pack bird tokens. The goods organizer holds all coin tokens, faction tokens, food tokens, and gems. Organizers take up little space in the box, and on the table. Goods organizer comes with a lid Organizers are made to order in a color of your choice and will be manufactured and shipped promptly. Unlike wooden organizers no assembly, glue, paint, or sealant required. This is a third-party accessory designed and manufactured by 3D Print Wizard. All product names are trademarks of their respective owners and are not associated or affiliated with 3D Print Wizard in any way. 3D printed organizers for the board game Near and Far. Ditch plastic bags and messy piles. Cut down on set-up and clean-up time. This organizer has been designed to fit neatly in the Near and Far box. The box insert holds all cards, characters and stands, reputation markers, and pack bird tokens. The goods organizer holds all coin tokens, faction tokens, food tokens, and gems. Organizers take up little space in the box, and on the table. Goods organizer comes with a lid Organizers are made to order in a color of your choice and will be manufactured and shipped promptly. Unlike wooden organizers no assembly, glue, paint, or sealant required. This is a third-party accessory designed and manufactured by 3D Print Wizard. All product names are trademarks of their respective owners and are not associated or affiliated with 3D Print Wizard in any way. + More - Less", "reviews": [""]} 25 | ] -------------------------------------------------------------------------------- /etsy/spiders/search_products.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | #============================================================================== 3 | #title :search_products.py 4 | #description :A spider to scrape etsy.com products based on a search string. 5 | #author :Patrick Alves (cpatrickalves@gmail.com) 6 | #last Update :12-02-2019 7 | #usage :scrapy crawl search_products -a search='3d printed' -o products.csv 8 | #python version :3.6 9 | #============================================================================== 10 | 11 | 12 | import scrapy 13 | import os 14 | import sys 15 | import csv 16 | import glob 17 | import json 18 | from openpyxl import Workbook 19 | from scrapy.http import Request 20 | from etsy.items import ProductItem 21 | from scrapy.loader import ItemLoader 22 | 23 | # Spider Class 24 | class ProductsSpider(scrapy.Spider): 25 | # Spider name 26 | name = 'search_products' 27 | allowed_domains = ['etsy.com'] 28 | start_urls = ['https://www.etsy.com/'] 29 | 30 | # Max number of items 31 | COUNT_MAX = 10**100 32 | # Count the number of items scraped 33 | COUNTER = 0 34 | 35 | # Get only the products URLs 36 | URLS_ONLY = False 37 | 38 | # Set the method to get the product reviews 39 | # If set to 1 (default), Spider will get only the reviews in the product's page, the default value is 4 reviews [FAST SCRAPING] 40 | # If set to 2, Spider will produce a Ajax request to get all reviews in the product's page, that is, a maximum of 10 reviews 41 | # If set to 3, Spider will visit the page with all store reviews and get all the reviews for this specific product [SLOWER SCRAPING] 42 | reviews_opt = None 43 | 44 | def __init__(self, search, reviews_option=1, count_max=None, urls_only=False, *args, **kwargs): 45 | if search: 46 | # Build the search URL 47 | self.start_urls = ['https://www.etsy.com/search?q={}&ref=pagination&page=1'.format(search)] 48 | # Set the maximum number of items to be scraped 49 | if count_max: 50 | self.COUNT_MAX = int(count_max) 51 | 52 | # Get only the products URLs 53 | self.URLS_ONLY = bool(urls_only) 54 | # Set the chosen review option 55 | self.reviews_opt = int(reviews_option) 56 | 57 | super(ProductsSpider, self).__init__(*args, **kwargs) 58 | 59 | 60 | # Parse the first page result and go to the next page 61 | def parse(self, response): 62 | 63 | # Get the list of products from html response 64 | products_list = response.xpath('//div[@data-search-results=""]/div//li//a/@href').extract() 65 | products_id_list = [product_href.split("/")[4] for product_href in products_list] 66 | 67 | # For each product extracts the product URL 68 | print(f"#### FOUND {len(products_id_list)} PRODUCTS") 69 | 70 | if self.URLS_ONLY: 71 | for product_id in products_id_list: 72 | 73 | # Create the ItemLoader object that stores each product information 74 | l = ItemLoader(item=ProductItem(), response=response) 75 | 76 | product_url = f'https://www.etsy.com/listing/{product_id}' 77 | l.add_value('url', product_url) 78 | yield l.load_item() 79 | 80 | else: 81 | for product_id in products_id_list: 82 | product_url = f'https://www.etsy.com/listing/{product_id}' 83 | # Stops if the COUNTER reaches the maximum set value 84 | if self.COUNTER < self.COUNT_MAX: 85 | # Go to the product's page to get the data 86 | yield scrapy.Request(product_url, callback=self.parse_product, dont_filter=True) 87 | 88 | # Pagination - Go to the next page 89 | current_page_number = int(response.url.split('=')[-1]) 90 | next_page_number = current_page_number + 1 91 | # Build the next page URL 92 | next_page_url = '='.join(response.url.split('=')[:-1]) + '=' + str(next_page_number) 93 | 94 | # If the current list is not empty 95 | if len(products_id_list) > 0: 96 | yield scrapy.Request(next_page_url) 97 | 98 | 99 | # Get the HTML from product's page and get the data 100 | def parse_product(self, response): 101 | 102 | # Stops if the COUNTER reaches the maximum set value 103 | if self.COUNTER >= self.COUNT_MAX: 104 | raise scrapy.exceptions.CloseSpider(reason='COUNT_MAX value reached - {} items'.format(self.COUNT_MAX)) 105 | 106 | # Check if the product is available 107 | no_available_message = response.xpath('//h2[contains(text(), "Darn")]') 108 | if no_available_message: 109 | return [] 110 | 111 | # Create the ItemLoader object that stores each product information 112 | l = ItemLoader(item=ProductItem(), response=response) 113 | 114 | # Get the product ID (ex: 666125766) 115 | product_id = response.url.split('/')[4] 116 | l.add_value('product_id', product_id) 117 | 118 | # Get the produc Title 119 | #l.add_xpath('title', '//meta[@property="og:title"]/@content') 120 | l.add_xpath('title', '//div[@data-component="listing-page-title-component"]/h1/text()') 121 | #l.add_xpath('title', "//h1[@data-listing-id='{}']".format(response.url.split('/')[4])) 122 | 123 | # Get the product price 124 | l.add_xpath('price', '//*[contains(@data-buy-box-region, "price")]//p') 125 | 126 | # Get the product URL (ex: www.etsy.com/listing/666125766) 127 | l.add_value('url', '/'.join(response.url.split('/')[2:5])) 128 | 129 | # Get the product description 130 | l.add_xpath('description', '//div[@data-id="description-text"]/div/p/text()') 131 | 132 | # Get each product option and save in a list 133 | product_options = [] 134 | product_options_list = response.xpath('//*[contains(@id, "inventory-variation-select")]') 135 | for options in product_options_list: 136 | # Get list of options 137 | temp_list = options.xpath('.//text()').extract() 138 | # Remove '\n' strings 139 | temp_list = list(map(lambda s: s.strip(), temp_list)) 140 | # Remove empty strings ('') 141 | temp_list = list(filter(lambda s: s != '', temp_list)) 142 | 143 | # Filter the 'Quantity' option 144 | if temp_list[0] != '1': 145 | # Create the final string: 146 | # example: "Select a color: White, Black, Red, Silver" 147 | product_options.append(temp_list[0] +': ' + ', '.join(temp_list[1:])) 148 | 149 | # Separate each option with a | (pipe) symbol 150 | l.add_value('product_options', '|'.join(product_options)) 151 | 152 | # Get the product rating (ex: 4.8 ) 153 | l.add_xpath('rating', '//a[@href="#reviews"]//input[@name="rating"]/@value') 154 | 155 | # Get the number of votes (number of reviews) 156 | l.add_xpath('number_of_reviews', '//button[@id="same-listing-reviews-tab"]/span/text()') 157 | 158 | # Count the number of product images 159 | images_sel = response.xpath('//ul[@data-carousel-pagination-list=""]/li/img/@data-src-delay').extract() 160 | l.add_value('count_of_images', len(images_sel)) 161 | l.add_value('images_urls', images_sel) 162 | 163 | # Get the product overview 164 | #l.add_xpath('overview', '//*[@class="listing-page-overview-component"]//li') 165 | 166 | # Get the number of people that add the product in favorites 167 | l.add_xpath('favorited_by', '//*[@id="item-overview"]//*[contains(@href, "/favoriters")]/text()', re='(\d+)') 168 | l.add_xpath('favorited_by', '//*[@class="listing-page-favorites-link"]/text()', re='(\d+)') 169 | l.add_xpath('favorited_by', '//a[contains(text(), " favorites")]/text()', re='(\d+)') 170 | 171 | # Get the name of the Store and location 172 | l.add_xpath('store_name', '//div[@id="listing-page-cart"]//span/text()') 173 | #l.add_xpath('store_location', '//*[@id="shop-info"]/div') 174 | #l.add_xpath('return_location', "//*[@class='js-estimated-delivery']/following-sibling::div") 175 | 176 | # Use the chosen method to get the reviews 177 | self.logger.info('Reviews scraping option: ' + str(self.reviews_opt)) 178 | 179 | # Option 3 - All reviews 180 | if self.reviews_opt == 3: 181 | # Getting all Reviews 182 | store_name = response.xpath('//span[@itemprop="title"]//text()').extract_first() 183 | # Build the reviews URL 184 | rev_url = "https://www.etsy.com/shop/{}/reviews?ref=l2-see-more-feedback".format(store_name) 185 | data = {'itemLoader':l, 'product_id':product_id} 186 | 187 | # Go to the all reviews page 188 | yield Request(rev_url, meta=data, callback=self.parse_reviews) 189 | 190 | # Option 2 - Ajax request 191 | elif self.reviews_opt == 2: 192 | # Creating the Ajax request 193 | # Getting the session cookie 194 | get_cookie = response.request.headers['Cookie'].split(b';')[0].split(b'=') 195 | cookies = {get_cookie[0].decode("utf-8"):get_cookie[1].decode("utf-8")} 196 | 197 | # Getting the x-csrf-token 198 | headers = {'x-csrf-token': response.xpath("//*[@name='_nnc']/@value").extract_first()} 199 | 200 | # Shop Id 201 | shop_id = response.xpath("//*[@property='og:image']/@content").extract_first().split('/')[3] 202 | 203 | formdata = { 204 | 'stats_sample_rate': '', 205 | 'specs[reviews][]': 'Listzilla_ApiSpecs_Reviews', 206 | 'specs[reviews][1][listing_id]': product_id, 207 | 'specs[reviews][1][shop_id]': shop_id, 208 | 'specs[reviews][1][render_complete]': 'true' 209 | } 210 | 211 | data = {'itemLoader':l, 'product_id':product_id} 212 | ajax_url = "https://www.etsy.com/api/v3/ajax/bespoke/member/neu/specs/reviews" 213 | 214 | yield scrapy.FormRequest(ajax_url, headers=headers, cookies=cookies, 215 | meta=data, formdata=formdata, 216 | callback=self.parse_ajax_response) 217 | # Option 1 218 | else: 219 | # Dict that saves all the reviews data 220 | reviews_data = [] 221 | reviews_counter = 1 222 | 223 | # Get the data from each review 224 | all_reviews = response.xpath('//*[@class="listing-page__review col-group pl-xs-0 pr-xs-0"]') 225 | # Process each review 226 | for r in all_reviews: 227 | 228 | # Get the profile URL of the reviewer 229 | reviewer_profile = r.xpath(".//*[@class='display-block']/parent::*//@href").extract_first() 230 | if reviewer_profile: 231 | # Build the full profile url 232 | reviewer_profile = 'www.etsy.com' + reviewer_profile 233 | else: 234 | # If the profile is inactive there is no profile url 235 | continue 236 | 237 | review_date = r.xpath(".//*[@class='text-link-underline display-inline-block mr-xs-1']/parent::*//text()").extract()[2].strip() 238 | reviewer_rating = r.xpath('.//input[@name="rating"]/@value').extract_first() 239 | review_content = " ".join(r.xpath('.//div[@class="overflow-hidden"]//text()').extract()).strip() 240 | 241 | # Build the review string 242 | rev_data = "Review number: {} \nProfile: {} \nRating: {} \nDate: {} \nContent: {}".format(reviews_counter, reviewer_profile, reviewer_rating, review_date, review_content) 243 | 244 | # Save into the list 245 | reviews_data.append(rev_data) 246 | reviews_counter += 1 247 | 248 | # Saves all reviews data 249 | l.add_value('reviews', "\n\n".join(reviews_data)) 250 | 251 | # Increment the items counter 252 | self.COUNTER += 1 253 | print('\nProducts scraped: {}\n'.format(self.COUNTER)) 254 | 255 | yield l.load_item() 256 | 257 | 258 | # Parse the Ajax response (Json) and extract reviews data 259 | def parse_ajax_response(self, response): 260 | # Get the itemLoader object from parser_products 261 | l = response.meta['itemLoader'] 262 | 263 | # Dict that saves all the reviews data 264 | reviews_data = [] 265 | reviews_counter = 1 266 | 267 | # Loads the Json data 268 | j = json.loads(response.text) 269 | html = j["output"]["reviews"] 270 | # Create the Selector 271 | sel = scrapy.Selector(text=html) 272 | 273 | # Get the data from each review 274 | all_reviews = sel.xpath('//*[@class="listing-page__review col-group pl-xs-0 pr-xs-0"]') 275 | # Process each review 276 | for r in all_reviews: 277 | 278 | # Get the profile URL of the reviewer 279 | reviewer_profile = r.xpath(".//*[@class='display-block']/parent::*//@href").extract_first() 280 | if reviewer_profile: 281 | # Build the full profile url 282 | reviewer_profile = 'www.etsy.com' + reviewer_profile 283 | else: 284 | # If the profile is inactive there is no profile url 285 | continue 286 | 287 | review_date = r.xpath(".//*[@class='text-link-underline display-inline-block mr-xs-1']/parent::*//text()").extract()[2].strip() 288 | reviewer_rating = r.xpath('.//input[@name="rating"]/@value').extract_first() 289 | review_content = " ".join(r.xpath('.//div[@class="overflow-hidden"]//text()').extract()).strip() 290 | 291 | # Build the string 292 | rev_data = "Review number: {} \nProfile: {} \nRating: {} \nDate: {} \nContent: {}".format(reviews_counter, reviewer_profile, reviewer_rating, review_date, review_content) 293 | 294 | # Saves the string in a list 295 | reviews_data.append(rev_data) 296 | reviews_counter += 1 297 | 298 | # aves all reviews data 299 | l.add_value('reviews', "\n\n".join(reviews_data)) 300 | 301 | # Increment the items counter 302 | self.COUNTER += 1 303 | print('\nProducts scraped: {}\n'.format(self.COUNTER)) 304 | 305 | yield l.load_item() 306 | 307 | 308 | # Parse the Store reviews page 309 | def parse_reviews(self, response): 310 | # Get the itemLoader object from parser_products 311 | l = response.meta['itemLoader'] 312 | 313 | # Dict that saves all the reviews data 314 | # Check if this is the first access or if there is data from another reviews page 315 | if 'reviews_data' in response.meta.keys(): 316 | reviews_data = response.meta['reviews_data'] 317 | reviews_counter = response.meta['reviews_counter'] 318 | else: 319 | reviews_data = [] 320 | reviews_counter = 1 321 | 322 | # Get the data from each review 323 | all_reviews = response.xpath("//*[@data-region='review']") 324 | 325 | # Process each review 326 | for r in all_reviews: 327 | 328 | # Get the product id of the review 329 | product_id = response.xpath("//*[@data-region='listing']//@href").extract_first().split('/')[4] 330 | 331 | # Check if this is the product in analysis 332 | if response.meta['product_id'] == product_id: 333 | # Get the profile URL of the reviewer 334 | reviewer_profile = r.xpath(".//*[@class='shop2-review-attribution']//@href").extract_first() 335 | if reviewer_profile: 336 | # Shorter version of the profile url 337 | reviewer_profile = reviewer_profile.split('?')[0] 338 | else: 339 | # If the profile is inactive there is no profile url 340 | continue 341 | 342 | reviewer_rating = r.xpath('.//input[@name="rating"]/@value').extract_first() 343 | review_date = r.xpath(".//*[@class='shop2-review-attribution']//text()").extract()[2].replace('on ','').strip() 344 | review_content = " ".join(r.xpath('.//div[@class="text-gray-lighter"]//text()').extract()).strip() 345 | 346 | # Build the string 347 | rev_data = "Review number: {} \nProfile: {} \nRating: {} \nDate: {} \nContent: {}".format(reviews_counter, reviewer_profile, reviewer_rating, review_date, review_content) 348 | 349 | # Saves the string in a list 350 | reviews_data.append(rev_data) 351 | reviews_counter += 1 352 | 353 | # Go to the next reviews page 354 | next_page_url = response.xpath("//*[contains(text(),'Next page')]/parent::*/@href").extract_first() 355 | # Check if there is a next page 356 | if next_page_url: 357 | # Save the current data 358 | data = {'itemLoader':l, 'product_id':product_id, 'reviews_data':reviews_data, 'reviews_counter':reviews_counter} 359 | # Build the request 360 | yield Request(next_page_url, meta=data, callback=self.parse_reviews) 361 | 362 | else: 363 | # If there is no next page, saves the data 364 | # Saves the data 365 | l.add_value('reviews', "\n\n".join(reviews_data)) 366 | # Increment the items counter 367 | self.COUNTER += 1 368 | print('\nProducts scraped: {}\n'.format(self.COUNTER)) 369 | 370 | yield l.load_item() 371 | 372 | 373 | # Create the Excel file 374 | def close(self, reason): 375 | 376 | # Check if there is a CSV file in arguments 377 | csv_found = False 378 | for arg in sys.argv: 379 | if '.csv' in arg: 380 | csv_found = True 381 | 382 | if csv_found: 383 | self.logger.info('Creating Excel file') 384 | # Get the last csv file created 385 | csv_file = max(glob.iglob('*.csv'), key=os.path.getctime) 386 | 387 | wb = Workbook() 388 | ws = wb.active 389 | 390 | with open(csv_file, 'r', encoding='utf-8') as f: 391 | for row in csv.reader(f): 392 | # Check if the row is not empty 393 | if row: 394 | ws.append(row) 395 | # Saves the file 396 | wb.save(csv_file.replace('.csv', '') + '.xlsx') 397 | -------------------------------------------------------------------------------- /outputs/products.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cpatrickalves/scraping-etsy/2eb602c947b71b8b33f1cae53d39ca7e93eeac95/outputs/products.xlsx -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | attrs==20.3.0 2 | Automat==20.2.0 3 | cffi==1.14.4 4 | constantly==15.1.0 5 | cryptography==3.3.1 6 | cssselect==1.1.0 7 | et-xmlfile==1.0.1 8 | hyperlink==20.0.1 9 | idna==3.1 10 | incremental==17.5.0 11 | itemadapter==0.2.0 12 | itemloaders==1.0.4 13 | jdcal==1.4.1 14 | jmespath==0.10.0 15 | lxml==4.6.2 16 | openpyxl==3.0.5 17 | parsel==1.6.0 18 | Protego==0.1.16 19 | pyasn1==0.4.8 20 | pyasn1-modules==0.2.8 21 | pycparser==2.20 22 | PyDispatcher==2.0.5 23 | PyHamcrest==2.0.2 24 | pyOpenSSL==20.0.1 25 | queuelib==1.5.0 26 | Scrapy==2.2.0 27 | service-identity==18.1.0 28 | six==1.15.0 29 | Twisted==20.3.0 30 | w3lib==1.22.0 31 | zope.interface==5.2.0 32 | -------------------------------------------------------------------------------- /scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = etsy.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = etsy 12 | --------------------------------------------------------------------------------