├── log └── .gitkeep ├── ali responses └── .gitkeep ├── docs └── source │ ├── .nojekyll │ ├── _static │ └── .gitkeep │ ├── set_creds.rst │ ├── get_new_items.rst │ ├── webscraperchewy.rst │ ├── webscraperebay.rst │ ├── webscraperetsy.rst │ ├── webscraperwmt.rst │ ├── webscraperali.rst │ ├── webscraperwayfair.rst │ ├── fbmp_mobile_sel_list.rst │ ├── ebay_desktop_sel_list.rst │ ├── etsy_desktop_sel_list.rst │ ├── fbmp_desktop_sel_list.rst │ ├── user_guide.rst │ ├── index.rst │ ├── getting_started.md │ └── conf.py ├── ebay responses └── .gitkeep ├── etsy responses └── .gitkeep ├── wmt responses └── .gitkeep ├── chewy responses └── .gitkeep ├── wayfair responses └── .gitkeep ├── Dropshipping Items ├── new_links_ebay.csv ├── new_links_etsy.csv ├── DROPSHIPPING_SPREADSHEET.xlsx ├── new_links_ali.csv ├── new_links_wayfair.csv ├── new_items_ali.csv └── new_items_wayfair.csv ├── requirements.txt ├── linters ├── .pydocstyle ├── .jscpd.json ├── .markdown-lint.yml ├── .flake8 ├── .hadolint.yaml └── .pylintrc ├── .github ├── ISSUE_TEMPLATE │ ├── custom.md │ ├── feature_request.md │ └── bug_report.md ├── stale.yml ├── FUNDING.yml ├── PULL_REQUEST_TEMPLATE │ └── pull_request_template.md └── workflows │ └── python-lint-analyze-test-build-sphinx.yml ├── log_config.yaml ├── .gitignore ├── Pipfile ├── get_new_items.py ├── README.md ├── .pre-commit-config.yaml ├── set_creds.py ├── webscraperwmt.py ├── fbmp_mobile_sel_list.py ├── .gitlab └── .gitlab-ci-py.yml ├── xpath_params.yaml ├── ebay_desktop_sel_list.py ├── etsy_desktop_sel_list.py ├── webscraperetsy.py ├── webscraperali.py ├── webscraperebay.py └── fbmp_desktop_sel_list.py /log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ali responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/.nojekyll: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ebay responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /etsy responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /wmt responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /chewy responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/source/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /wayfair responses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Dropshipping Items/new_links_ebay.csv: -------------------------------------------------------------------------------- 1 | link -------------------------------------------------------------------------------- /Dropshipping Items/new_links_etsy.csv: -------------------------------------------------------------------------------- 1 | link -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ehgp/ecommerce_autolister/HEAD/requirements.txt -------------------------------------------------------------------------------- /linters/.pydocstyle: -------------------------------------------------------------------------------- 1 | [pydocstyle] 2 | # https://www.pydocstyle.org/en/stable/usage.html#configuration-files 3 | convention = google 4 | -------------------------------------------------------------------------------- /Dropshipping Items/DROPSHIPPING_SPREADSHEET.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ehgp/ecommerce_autolister/HEAD/Dropshipping Items/DROPSHIPPING_SPREADSHEET.xlsx -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /docs/source/set_creds.rst: -------------------------------------------------------------------------------- 1 | Set Creds 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: set_creds 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/get_new_items.rst: -------------------------------------------------------------------------------- 1 | Get New Items 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: get_new_items 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperchewy.rst: -------------------------------------------------------------------------------- 1 | Chewy Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperchewy 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperebay.rst: -------------------------------------------------------------------------------- 1 | eBay Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperebay 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperetsy.rst: -------------------------------------------------------------------------------- 1 | Etsy Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperetsy 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperwmt.rst: -------------------------------------------------------------------------------- 1 | Walmart Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperwmt 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperali.rst: -------------------------------------------------------------------------------- 1 | Ali Express Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperali 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/webscraperwayfair.rst: -------------------------------------------------------------------------------- 1 | Wayfair Web Scraper 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: webscraperwayfair 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/fbmp_mobile_sel_list.rst: -------------------------------------------------------------------------------- 1 | FBMP Mobile Autolister 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: fbmp_mobile_sel_list 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/ebay_desktop_sel_list.rst: -------------------------------------------------------------------------------- 1 | eBay Desktop Autolister 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: ebay_desktop_sel_list 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/etsy_desktop_sel_list.rst: -------------------------------------------------------------------------------- 1 | Etsy Desktop Autolister 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: etsy_desktop_sel_list 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /docs/source/fbmp_desktop_sel_list.rst: -------------------------------------------------------------------------------- 1 | FBMP Desktop Autolister 2 | =============================== 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | .. automodule:: fbmp_desktop_sel_list 7 | :members: 8 | :inherited-members: 9 | :undoc-members: -------------------------------------------------------------------------------- /linters/.jscpd.json: -------------------------------------------------------------------------------- 1 | { 2 | "threshold": 0, 3 | "reporters": ["html", "console", "badge"], 4 | "ignore": [".github/workflows/python-lint-analyze-test-build-sphinx.yml","/node_modules/jscpd/","/linters/",".gitlab/.gitlab-ci-py.yml"], 5 | "absolute": true 6 | } -------------------------------------------------------------------------------- /Dropshipping Items/new_links_ali.csv: -------------------------------------------------------------------------------- 1 | link 2 | https://www.aliexpress.com/item/1005001805778039.html? 3 | https://www.aliexpress.com/item/1005001909720513.html? 4 | https://www.aliexpress.com/item/1005001873139959.html? 5 | https://www.aliexpress.com/item/4000163754909.html? 6 | https://www.aliexpress.com/item/1000008623030.html? 7 | https://www.aliexpress.com/item/33010721339.html? 8 | -------------------------------------------------------------------------------- /Dropshipping Items/new_links_wayfair.csv: -------------------------------------------------------------------------------- 1 | link 2 | https://www.aliexpress.com/item/1005001805778039.html? 3 | https://www.aliexpress.com/item/1005001909720513.html? 4 | https://www.aliexpress.com/item/1005001873139959.html? 5 | https://www.aliexpress.com/item/4000163754909.html? 6 | https://www.aliexpress.com/item/1000008623030.html? 7 | https://www.aliexpress.com/item/33010721339.html? 8 | -------------------------------------------------------------------------------- /docs/source/user_guide.rst: -------------------------------------------------------------------------------- 1 | User Guide 2 | ================================================ 3 | .. toctree:: 4 | :maxdepth: 2 5 | 6 | set_creds 7 | fbmp_desktop_sel_list 8 | fbmp_mobile_sel_list 9 | etsy_desktop_sel_list 10 | ebay_desktop_sel_list 11 | get_new_items 12 | webscraperali 13 | webscraperchewy 14 | webscraperebay 15 | webscraperetsy 16 | webscraperwayfair 17 | webscraperwmt 18 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. eCommerce Autolister documentation main file 2 | 3 | eCommerce Autolister 4 | ================================================================ 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | getting_started 9 | user_guide 10 | 11 | Author 12 | ================================================================ 13 | ehgp 14 | 15 | Indices and tables 16 | ================================================================ 17 | * :ref:`genindex` 18 | * :ref:`modindex` 19 | * :ref:`search` 20 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /log_config.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | 3 | disable_existing_loggers: True 4 | formatters: 5 | simple: 6 | format: "%(asctime)s %(name)-5s %(message)s" 7 | datefmt: "%Y-%m-%d %H:%M:%S" 8 | extended: 9 | format: "%(asctime)s %(name)s %(levelname)s: %(message)s" 10 | 11 | handlers: 12 | console: 13 | class: logging.StreamHandler 14 | level: INFO 15 | formatter: simple 16 | stream: ext://sys.stdout 17 | file: 18 | class: logging.FileHandler 19 | level: INFO 20 | formatter: simple 21 | filename: log/log.log 22 | 23 | loggers: 24 | __main__: 25 | level: INFO 26 | handlers: [console, file] 27 | propagate: False 28 | 29 | selenium: 30 | level: INFO 31 | handlers: [console, file] 32 | propagate: False -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an issue becomes stale 2 | daysUntilStale: 60 3 | # Number of days of inactivity before a stale issue is closed 4 | daysUntilClose: 7 5 | # Issues with these labels will never be considered stale 6 | exemptLabels: 7 | - pinned 8 | - security 9 | - help wanted 10 | # Label to use when marking an issue as stale 11 | staleLabel: wontfix 12 | # Comment to post when marking an issue as stale. Set to `false` to disable 13 | markComment: > 14 | This issue has been automatically marked as stale because it has not had 15 | recent activity. It will be closed if no further activity occurs. Thank you 16 | for your contributions. 17 | # Comment to post when closing a stale issue. Set to `false` to disable 18 | closeComment: false -------------------------------------------------------------------------------- /linters/.markdown-lint.yml: -------------------------------------------------------------------------------- 1 | # MD013/line-length - Line length 2 | MD013: 3 | # Number of characters 4 | line_length: 10000 5 | # Number of characters for headings 6 | heading_line_length: 80 7 | # Number of characters for code blocks 8 | code_block_line_length: 80 9 | # Include code blocks 10 | code_blocks: true 11 | # Include tables 12 | tables: true 13 | # Include headings 14 | headings: true 15 | # Include headings 16 | headers: true 17 | # Strict length checking 18 | strict: false 19 | # Stern length checking 20 | stern: false 21 | 22 | # MD049/emphasis-style - Emphasis style should be consistent 23 | MD049: 24 | # Emphasis style should be consistent 25 | style: "consistent" 26 | 27 | # MD045/no-alt-text - Images should have alternate text (alt text) 28 | MD045: false 29 | 30 | # MD033/no-inline-html - Inline HTML 31 | MD033: false 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | 1. Go to '...' 17 | 2. Click on '....' 18 | 3. Scroll down to '....' 19 | 4. See error 20 | 21 | **Expected behavior** 22 | A clear and concise description of what you expected to happen. 23 | 24 | **Screenshots** 25 | If applicable, add screenshots to help explain your problem. 26 | 27 | **Desktop (please complete the following information):** 28 | 29 | - OS: [e.g. iOS] 30 | - Browser [e.g. chrome, safari] 31 | - Version [e.g. 22] 32 | 33 | **Mobile phone (please complete the following information):** 34 | 35 | - Device: [e.g. iPhone6] 36 | - OS: [e.g. iOS8.1] 37 | - Browser [e.g. stock browser, safari] 38 | - Version [e.g. 22] 39 | 40 | **Additional context** 41 | Add any other context about the problem here. 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | .vscode 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | env/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *,cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Log 54 | **/*.log 55 | **/*.csv 56 | 57 | # Sphinx documentation 58 | docs/_build/ 59 | 60 | # PyBuilder 61 | target/ 62 | 63 | *.profile 64 | .client_secrets.json 65 | Pipfile.lock 66 | chromedriver.exe 67 | **/*.dat 68 | **/*.html -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [packages] 7 | ipython = {version = "*", index="pypi"} 8 | requests = {version = "==2.24.0", index="pypi"} 9 | selenium = {version = "==3.141.0", index="pypi"} 10 | pandas = {version = "==1.3.3", index="pypi"} 11 | pathlib2 = {version = "==2.3.5", index="pypi"} 12 | keyring = {version = "==22.3.0", index="pypi"} 13 | pyyaml = {version = "==5.4.1", index="pypi"} 14 | bs4 = {version = "==0.0.1", index="pypi"} 15 | pyautogui = "==0.9.53" 16 | openpyxl = {version = "==3.0.7", index="pypi"} 17 | lxml = {version = "==4.9.1", index="pypi"} 18 | Sphinx = {version = "==4.4.0", index="pypi"} 19 | sphinx-rtd-theme = {version = "==1.0.0", index="pypi"} 20 | recommonmark = {version = "==0.7.1", index="pypi"} 21 | flake8 = {version = "==3.9.0", index="pypi"} 22 | pydocstyle = {version = "==6.0.0", index="pypi"} 23 | black = {version = "==22.3.0", index="pypi"} 24 | chromedriver-autoinstaller = {git = "https://github.com/chadfrost/python-chromedriver-autoinstaller.git", ref = "master"} 25 | geckodriver-autoinstaller = {git = "https://github.com/VBobCat/python-geckodriver-autoinstaller.git", ref = "patch-1"} 26 | -------------------------------------------------------------------------------- /linters/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # Doc: https://flake8.pycqa.org/en/latest/user/configuration.html 3 | # E203: whitespace before ':' (black does not add whitespace) 4 | # W503: line break before binary operator (black does not add line breaks) 5 | # F403: 'from module import *' used; unable to detect undefined names 6 | # F401: module imported but unused 7 | # E501: line too long (88 > 79 characters) 8 | # E402: module level import not at top of file 9 | # E731: do not assign a lambda expression, use a def 10 | # E741: do not use variables named 'l', 'O', or 'I' 11 | # E722: do not use bare 'except' 12 | # E203: whitespace before ':' 13 | # E266: too many leading '#' for block comment 14 | # E501: line too long (88 > 79 characters) 15 | # E722: do not use bare 'except' 16 | # E741: do not use variables named 'l', 'O', or 'I' 17 | # W503: line break before binary operator 18 | # W504: line break after binary operator 19 | # W605: invalid escape sequence '\.' 20 | # F403: 'from module import *' used; unable to detect undefined names 21 | # max-complexity: McCabe complexity 22 | # extend-ignore: ignore additional errors 23 | # max-line-length: max line length 24 | # select: select errors and warnings 25 | # ignore: ignore errors and warnings 26 | 27 | max-line-length = 88 28 | extend-ignore = E203, W503, F403, F401, E501, E402, E731, E741, E722, E203, E266, E501, E722, E741, W503, W504, W605, F403 29 | max-complexity = 18 30 | select = B,C,E,F,W,T4,B9 31 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # # These are supported funding model platforms 2 | 3 | # github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | # patreon: # Replace with a single Patreon username 5 | # open_collective: # Replace with a single Open Collective username 6 | # ko_fi: # Replace with a single Ko-fi username 7 | # tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | # community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | # liberapay: # Replace with a single Liberapay username 10 | # issuehunt: # Replace with a single IssueHunt username 11 | # otechie: # Replace with a single Otechie username 12 | # lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | # custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | 15 | # # Donations 16 | # # I accept donations in cryptocurrencies, anything helps. Thank you. 17 | 18 | # BTC: 3G3zsvcxgomdERYTSjeX4iBJYMfFfCgFmn 19 | 20 | # ETH: 0x227cc9c06db03563300fa7c2d0b0a34b370f5987 21 | 22 | # DOGE: DNNsSrk767w9K1eaqc2tQSvJ4mzfBpw4RP 23 | 24 | # BNB: bnb1rlka4xf6h8p8nlpf8szczmcyugdktptstgham0 25 | 26 | # XMR: 86vsoW6jsTzcvGZxKVG1PxfsSqUzrMuqvKxLGCXZ3RcNY7VyvhcgiimciW5ZsHyrKUGCpqFPjDG7iMu9sSoveZDxMeGpqCb 27 | 28 | # ZCASH: t1fkojdhoTTQmrPSExCLMuV6D3a2jxESGtL 29 | 30 | # ADA: DdzFFzCqrhtBuwQRtRKNSVca58HDwicLx5aDWn8K5pyg36665BL5s6WBLAc9bCTxWk15MFiefoerCRiuxysW7Sy4RQJ6UM2vWXoCg98z 31 | 32 | github: ["ehgp"] 33 | -------------------------------------------------------------------------------- /get_new_items.py: -------------------------------------------------------------------------------- 1 | """Get New Items. 2 | 3 | Automatically gets new links from products from your lists 4 | 5 | Author: ehgp 6 | """ 7 | import os 8 | from pathlib import Path 9 | 10 | import pandas as pd 11 | 12 | import webscraperali 13 | import webscraperchewy 14 | import webscraperebay 15 | import webscraperetsy 16 | import webscraperwayfair 17 | 18 | path = Path(os.getcwd()) 19 | Path("log").mkdir(parents=True, exist_ok=True) 20 | 21 | binary_path = Path(path, "chromedriver.exe") 22 | 23 | new_items_ali = pd.read_csv(Path(path, "Dropshipping Items", "new_links_ali.csv")) 24 | 25 | new_items_ebay = pd.read_csv(Path(path, "Dropshipping Items", "new_links_ebay.csv")) 26 | 27 | new_items_etsy = pd.read_csv(Path(path, "Dropshipping Items", "new_links_etsy.csv")) 28 | 29 | new_items_wayfair = pd.read_csv( 30 | Path(path, "Dropshipping Items", "new_links_wayfair.csv") 31 | ) 32 | 33 | if len(new_items_ali) > 0: 34 | webscraperali.listingscraper(new_items_ali["link"].to_list()) 35 | 36 | if len(new_items_ebay) > 0: 37 | webscraperebay.listingscraper(new_items_ebay["link"].to_list()) 38 | 39 | if len(new_items_etsy) > 0: 40 | webscraperetsy.listingscraper(new_items_etsy["link"].to_list()) 41 | 42 | if len(new_items_wayfair) > 0: 43 | webscraperwayfair.listingscraper(new_items_wayfair["link"].to_list()) 44 | 45 | # etsy checks 4 pages, ali 2, ebay 2, wayfair all 46 | webscraperali.linkscraperali() 47 | webscraperchewy.linkscraperchewy() 48 | webscraperebay.linkscraperebay() 49 | webscraperetsy.linkscraperetsy() 50 | webscraperwayfair.linkscraperwayfair() 51 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE/pull_request_template.md: -------------------------------------------------------------------------------- 1 | # Pull Request Template 2 | 3 | ## Description 4 | 5 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. 6 | 7 | Fixes # (issue) 8 | 9 | ## Type of change 10 | 11 | Please delete options that are not relevant. 12 | 13 | - [ ] Bugfix (non-breaking change which fixes an issue) 14 | - [ ] New feature (non-breaking change which adds functionality) 15 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 16 | - [ ] This change requires a documentation update 17 | 18 | ## How Has This Been Tested? 19 | 20 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration 21 | 22 | - [ ] Test A 23 | - [ ] Test B 24 | 25 | **Test Configuration**: 26 | 27 | - Firmware version: 28 | - Hardware: 29 | - Toolchain: 30 | - SDK: 31 | 32 | ## Checklist 33 | 34 | - [ ] My code follows the style guidelines of this project 35 | - [ ] I have performed a self-review of my own code 36 | - [ ] I have commented my code, particularly in hard-to-understand areas 37 | - [ ] I have made corresponding changes to the documentation 38 | - [ ] My changes generate no new warnings 39 | - [ ] I have added tests that prove my fix is effective or that my feature works 40 | - [ ] New and existing unit tests pass locally with my changes 41 | - [ ] Any dependent changes have been merged and published in downstream modules 42 | - [ ] I have checked my code and corrected any misspellings 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # eCommerce Autolister 2 | 3 | ## Introduction 4 | 5 | Autolisting bot that will perform listing functions in both FBMP and Etsy. Ebay is a work in progress 6 | 7 | ## Documentation 8 | 9 | Documentation found [here](https://ehgp.github.io/ecommerce_autolister) 10 | 11 | ## Dependencies 12 | 13 | Your command center is located in Dropshipping Items/DROPSHIPPING_SPREADSHEET.xlsx 14 | 15 | Must use set_creds.py file with user email and password for code to work and authenticate 16 | with the marketplace and webscraper you want 17 | 18 | Python 3.8.8 19 | pipenv 2021.11.23 20 | 21 | ## Install 22 | 23 | * Must use set_creds.py file with user email and password for code to work and authenticate 24 | with the marketplace and webscraper you want 25 | 26 | * cd to the directory 27 | 28 | * pipenv install 29 | 30 | * pipenv run python fbmp_desktop_sel_list.py 31 | 32 | ## Setup 33 | 34 | chromedriver.exe for your chrome version must be installed from [here](https://sites.google.com/chromium.org/driver/downloads) 35 | 36 | ## Donations 37 | 38 | I accept donations in cryptocurrencies, anything helps. Thank you. 39 | 40 | BTC: 41 | 3G3zsvcxgomdERYTSjeX4iBJYMfFfCgFmn 42 | 43 | ETH: 44 | 0x227cc9c06db03563300fa7c2d0b0a34b370f5987 45 | 46 | DOGE: 47 | DNNsSrk767w9K1eaqc2tQSvJ4mzfBpw4RP 48 | 49 | BNB: 50 | bnb1rlka4xf6h8p8nlpf8szczmcyugdktptstgham0 51 | 52 | XMR: 53 | 86vsoW6jsTzcvGZxKVG1PxfsSqUzrMuqvKxLGCXZ3RcNY7VyvhcgiimciW5ZsHyrKUGCpqFPjDG7iMu9sSoveZDxMeGpqCb 54 | 55 | ZCASH: 56 | t1fkojdhoTTQmrPSExCLMuV6D3a2jxESGtL 57 | 58 | ADA: 59 | DdzFFzCqrhtBuwQRtRKNSVca58HDwicLx5aDWn8K5pyg36665BL5s6WBLAc9bCTxWk15MFiefoerCRiuxysW7Sy4RQJ6UM2vWXoCg98z 60 | 61 | ## Author 62 | [ehgp](ehgp.github.io) 63 | -------------------------------------------------------------------------------- /docs/source/getting_started.md: -------------------------------------------------------------------------------- 1 | # eCommerce Autolister 2 | 3 | ## Introduction 4 | 5 | Autolisting bot that will perform listing functions in both FBMP and Etsy. Ebay is a work in progress 6 | 7 | ## Documentation 8 | 9 | Documentation found [here](https://ehgp.github.io/ecommerce_autolister) 10 | 11 | ## Dependencies 12 | 13 | Your command center is located in Dropshipping Items/DROPSHIPPING_SPREADSHEET.xlsx 14 | 15 | Must use set_creds.py file with user email and password for code to work and authenticate 16 | with the marketplace and webscraper you want 17 | 18 | Python 3.8.8 19 | pipenv 2021.11.23 20 | 21 | ## Install 22 | 23 | * Must use set_creds.py file with user email and password for code to work and authenticate 24 | with the marketplace and webscraper you want 25 | 26 | * cd to the directory 27 | 28 | * pipenv install 29 | 30 | * pipenv run python fbmp_desktop_sel_list.py 31 | 32 | ## Setup 33 | 34 | chromedriver.exe for your chrome version must be installed from [here](https://sites.google.com/chromium.org/driver/downloads) 35 | 36 | ## Donations 37 | 38 | I accept donations in cryptocurrencies, anything helps. Thank you. 39 | 40 | BTC: 41 | 3G3zsvcxgomdERYTSjeX4iBJYMfFfCgFmn 42 | 43 | ETH: 44 | 0x227cc9c06db03563300fa7c2d0b0a34b370f5987 45 | 46 | DOGE: 47 | DNNsSrk767w9K1eaqc2tQSvJ4mzfBpw4RP 48 | 49 | BNB: 50 | bnb1rlka4xf6h8p8nlpf8szczmcyugdktptstgham0 51 | 52 | XMR: 53 | 86vsoW6jsTzcvGZxKVG1PxfsSqUzrMuqvKxLGCXZ3RcNY7VyvhcgiimciW5ZsHyrKUGCpqFPjDG7iMu9sSoveZDxMeGpqCb 54 | 55 | ZCASH: 56 | t1fkojdhoTTQmrPSExCLMuV6D3a2jxESGtL 57 | 58 | ADA: 59 | DdzFFzCqrhtBuwQRtRKNSVca58HDwicLx5aDWn8K5pyg36665BL5s6WBLAc9bCTxWk15MFiefoerCRiuxysW7Sy4RQJ6UM2vWXoCg98z 60 | 61 | ## Author 62 | [ehgp](ehgp.github.io) 63 | -------------------------------------------------------------------------------- /linters/.hadolint.yaml: -------------------------------------------------------------------------------- 1 | ignored: 2 | - DL3008 # Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` 3 | - DL3018 # Pin versions in apk add. Instead of `apk add ` use `apk add =` 4 | - DL4000 # Set the SHELL option -o pipefail before RUN with a pipe in it. If you are using /bin/sh in an alpine image or if your shell is symlinked to busybox then consider explicitly setting your SHELL to /bin/ash, or disable this check 5 | - DL4001 # Either use Wget or Curl but not both 6 | - DL4006 # Set the SHELL option -o pipefail before RUN with a pipe in it. If you are using /bin/sh in an alpine image or if your shell is symlinked to busybox then consider explicitly setting your SHELL to /bin/ash, or disable this check 7 | - DL3008 # warning: Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` 8 | - DL3009 #info: Delete the apt-get lists after installing something 9 | - DL3042 #warning: Avoid use of cache directory with pip. Use `pip install --no-cache-dir ` 10 | - DL3013 #warning: Pin versions in pip. Instead of `pip install ` use `pip install ==` or `pip install --requirement ` 11 | - DL3042 #warning: Avoid use of cache directory with pip. Use `pip install --no-cache-dir ` 12 | - DL3059 #info: Multiple consecutive `RUN` instructions. Consider consolidation. 13 | - DL4000 #info: Multiple `CMD` instructions found. If you list more than one `CMD` then only the last `CMD` will take effect 14 | - DL4001 #info: Either use Wget or Curl but not both 15 | - DL4006 #info: Specify a maintainer of the Dockerfile 16 | 17 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ".json$" 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.0.1 5 | hooks: 6 | - id: check-added-large-files 7 | - id: check-case-conflict 8 | - id: check-json 9 | - id: check-merge-conflict 10 | - id: check-symlinks 11 | - id: check-toml 12 | - id: check-yaml 13 | - id: debug-statements 14 | - id: detect-private-key 15 | - id: end-of-file-fixer 16 | - id: mixed-line-ending 17 | - id: trailing-whitespace 18 | 19 | - repo: https://github.com/pycqa/flake8 20 | rev: 5.0.4 21 | hooks: 22 | - id: flake8 23 | args: [--config, linters/.flake8] 24 | 25 | - repo: https://github.com/psf/black 26 | rev: 22.3.0 27 | hooks: 28 | - id: black 29 | 30 | - repo: https://github.com/pycqa/pylint 31 | rev: v2.15.5 32 | hooks: 33 | - id: pylint 34 | args: [--rcfile, linters/.pylintrc] 35 | 36 | - repo: https://github.com/pycqa/pydocstyle 37 | rev: 6.1.1 38 | hooks: 39 | - id: pydocstyle 40 | args: [--config, linters/.pydocstyle] 41 | 42 | - repo: https://github.com/pycqa/isort 43 | rev: 5.11.5 44 | hooks: 45 | - id: isort 46 | name: isort (python) 47 | args: [--profile, black] 48 | - id: isort 49 | name: isort (cython) 50 | types: [cython] 51 | - id: isort 52 | name: isort (pyi) 53 | types: [pyi] 54 | 55 | # - repo: https://github.com/antonbabenko/pre-commit-terraform 56 | # rev: v1.74.1 57 | # hooks: 58 | # - id: terraform_fmt 59 | # args: 60 | # - --args=-write=true 61 | -------------------------------------------------------------------------------- /set_creds.py: -------------------------------------------------------------------------------- 1 | """Set Creds. 2 | 3 | Leverages Keyring to add your credentials for each portal and then retrieves them in code. 4 | author: ehgp 5 | """ 6 | import datetime as dt 7 | import logging 8 | import logging.config 9 | import os 10 | from getpass import getpass, getuser 11 | from pathlib import Path 12 | 13 | import keyring 14 | import yaml 15 | 16 | # Paths 17 | path = Path(os.getcwd()) 18 | 19 | # Logging 20 | Path("log").mkdir(parents=True, exist_ok=True) 21 | log_config = Path(path, "log_config.yaml") 22 | timestamp = "{:%Y_%m_%d_%H_%M_%S}".format(dt.datetime.now()) 23 | with open(log_config, "r") as log_file: 24 | config_dict = yaml.safe_load(log_file.read()) 25 | # Append date stamp to the file name 26 | log_filename = config_dict["handlers"]["file"]["filename"] 27 | base, extension = os.path.splitext(log_filename) 28 | base2 = "_" + os.path.splitext(os.path.basename(__file__))[0] + "_" 29 | log_filename = "{}{}{}{}".format(base, base2, timestamp, extension) 30 | config_dict["handlers"]["file"]["filename"] = log_filename 31 | logging.config.dictConfig(config_dict) 32 | logger = logging.getLogger(__name__) 33 | 34 | # Leverages Windows Credential Manager and Mac Keychain to store credentials 35 | # and also makes environment variables that store your credentials in your computer 36 | dsns = [ 37 | "FACEBOOK_EMAIL", 38 | "FACEBOOK_PASSWORD", 39 | # "ALI_EMAIL", 40 | # "ALI_PASSWORD", 41 | # "CHEWY_EMAIL", 42 | # "CHEWY_PASSWORD", 43 | "EBAY_EMAIL", 44 | "EBAY_PASSWORD", 45 | # "ETSY_EMAIL", 46 | # "ETSY_PASSWORD", 47 | # "WAYFAIR_EMAIL", 48 | # "WAYFAIR_PASSWORD", 49 | # "WALMART_EMAIL", 50 | # "WALMART_PASSWORD", 51 | ] 52 | user = getuser() 53 | for dsn in dsns: 54 | if keyring.get_password(dsn, user) is None: 55 | prompt = f"Please input {dsn}: " 56 | password = getpass(prompt=prompt, stream=None) 57 | keyring.set_password(dsn, user, password) 58 | -------------------------------------------------------------------------------- /webscraperwmt.py: -------------------------------------------------------------------------------- 1 | """WALMART IMAGE SCRAPER. 2 | 3 | Scrapes images from your list on WALMART 4 | 5 | Author: ehgp 6 | """ 7 | import json 8 | import os 9 | import re 10 | import shutil 11 | import string 12 | from os import listdir 13 | from os.path import isfile, join 14 | from pathlib import Path 15 | 16 | import requests 17 | from bs4 import BeautifulSoup 18 | 19 | 20 | def format_filename(s): 21 | """Take a string and return a valid filename constructed from the string. 22 | 23 | Uses a whitelist approach: any characters not present in valid_chars are 24 | removed. Also spaces are replaced with underscores. 25 | 26 | Note: this method may produce invalid filenames such as ``, `.` or `..` 27 | When I use this method I prepend a date string like '2009_01_15_19_46_32_' 28 | and append a file extension like '.txt', so I avoid the potential of using 29 | an invalid filename. 30 | 31 | """ 32 | valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) 33 | filename = "".join(c for c in s if c in valid_chars) 34 | return filename 35 | 36 | 37 | path = Path(os.getcwd()) 38 | Path("log").mkdir(parents=True, exist_ok=True) 39 | URL = "https://www.walmart.com/ip/LED-Face-Mask-Luminous-Programmable-Message-Display-Mask-Rechargeable/978504239" 40 | title = "Bluetooth App LED Mask Customize" 41 | 42 | # # def imagewebscraperwmt(URL,title): 43 | 44 | page = requests.get(URL) 45 | 46 | soup = BeautifulSoup(page.content, "html.parser") 47 | 48 | imgfilename = re.findall( 49 | r"(http:|https:)(\/\/.walmartimages.com[^\"\']*)(96.png|96.jpg|96.jpeg|96.gif|96.png|96.svg|96.webp)", 50 | str(soup), 51 | ) 52 | 53 | imgfilename = ["".join(i) for i in imgfilename] 54 | imgfilename = list(dict.fromkeys(imgfilename)) 55 | 56 | os.makedirs(Path(path, "Dropshipping Items", title), exist_ok=True) 57 | 58 | for idx, url in enumerate(imgfilename): 59 | with open(Path(path, "Dropshipping Items", title, f"{title}{idx}.jpg"), "wb") as f: 60 | response = requests.get(url.replace("96", "300")) 61 | f.write(response.content) 62 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | """Configuration file for the Sphinx documentation builder.""" 2 | # # This file only contains a selection of the most common options. 3 | # For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html 4 | # # -- Path setup -------------------------------------------------------------- # 5 | # If extensions (or modules to document with autodoc) are in another directory, # add these directories 6 | # to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath 7 | # to make it absolute, like shown here. 8 | import datetime as dt 9 | import os 10 | import sys 11 | 12 | sys.path.insert(0, os.path.abspath("../..")) 13 | # # -- Project information ----------------------------------------------------- 14 | project = "eCommerce Autolister" 15 | author = "ehgp" 16 | copyright = "%s ehgp. All rights reserved." % (dt.date.today().year) 17 | # The full version, including alpha/beta/rc tags 18 | release = "0.1.0" 19 | # -- General configuration --------------------------------------------------- 20 | # Add any Sphinx extension module names here, as strings. 21 | # They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. 22 | extensions = [ 23 | "sphinx.ext.autodoc", 24 | "sphinx.ext.napoleon", 25 | "sphinx.ext.coverage", 26 | "recommonmark", 27 | ] 28 | napoleon_google_docstring = True 29 | napoleon_numpy_docstring = False 30 | napoleon_use_param = False 31 | napoleon_use_ivar = True 32 | html_show_sourcelink = False 33 | # Add any paths that contain templates here, relative to this directory. 34 | templates_path = ["_templates"] 35 | # List of patterns, relative to source directory, that match files and 36 | # directories to ignore when looking for source files. 37 | # This pattern also affects html_static_path and html_extra_path. 38 | exclude_patterns = [] 39 | # The master toctree document. 40 | master_doc = "index" 41 | # -- Options for HTML output ------------------------------------------------- 42 | # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. 43 | html_theme = "sphinx_rtd_theme" 44 | # HTML Theme Options 45 | html_theme_options = { 46 | "display_version": True, 47 | "collapse_navigation": False, 48 | # "logo_only": True, 49 | } 50 | # Image for the top of the sidebar & favicon 51 | # html_logo = "_static/logo.jpg" 52 | # html_favicon = "_static/logo.jpg" 53 | 54 | # Add any paths that contain custom static files (such as style sheets) here, 55 | # relative to this directory. They are copied after the builtin static files, 56 | # so a file named "default.css" will overwrite the builtin "default.css". 57 | html_static_path = ["_static"] 58 | -------------------------------------------------------------------------------- /Dropshipping Items/new_items_ali.csv: -------------------------------------------------------------------------------- 1 | ,new_title,new_description,new_link,new_price,new_ship_price,new_ship_how 2 | 0,durable Exercise Treadmill with Handrail Ultra-thin Foldable Running Walkingpad With LCD Display Indoor Mute Fitness Equipment,"Cheap Treadmills, Buy Quality Sports & Entertainment Directly Suppliers:durable Exercise Treadmill with Handrail Ultra thin Foldable Running Walkingpad With LCD Display Indoor Mute Fitness Equipment Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/1005001805778039.html?,225.00,Free Shipping,to United States via USPS 3 | 1,Electric Mechanical Treadmill Household Gym Treadmill Running Machine with LED Display Safe Bar Folding Silent Mini Treadmill,"Cheap Treadmills, Buy Quality Sports & Entertainment Directly Suppliers:Electric Mechanical Treadmill Household Gym Treadmill Running Machine with LED Display Safe Bar Folding Silent Mini Treadmill Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/1005001909720513.html?,171.94,Free Shipping,to United States via USPS 4 | 2,US STORE Folding Electric Treadmill Home Electric Treadmill Ultra-thin Mini Fitness Equipment Foldable Home Treadmill Machine,"Cheap Treadmills, Buy Quality Sports & Entertainment Directly Suppliers:US STORE Folding Electric Treadmill Home Electric Treadmill Ultra thin Mini Fitness Equipment Foldable Home Treadmill Machine Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/1005001873139959.html?,190.33,Free Shipping,to United States via USPS 5 | 3,Abdominal Muscle Trainer EMS Electric Muscle Exerciser Machine For Body Slim Weight Loss Fat Burning Workout Home GymEquipment,"Cheap Accessories, Buy Quality Sports & Entertainment Directly Suppliers:Abdominal Muscle Trainer EMS Electric Muscle Exerciser Machine For Body Slim Weight Loss Fat Burning Workout Home GymEquipment Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/4000163754909.html?,3.82 - 20.34,Shipping: 5.45,to United States via USPS 6 | 4,Body Slimming EMS Hip Trainer Muscle Stimulator Electric Butt Buttock Vibration Massager Bodybuilding Machine Home Workout Gym,"Cheap Integrated Fitness Equipments, Buy Quality Sports & Entertainment Directly Suppliers:Body Slimming EMS Hip Trainer Muscle Stimulator Electric Butt Buttock Vibration Massager Bodybuilding Machine Home Workout Gym Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/1000008623030.html?,7.45,Free Shipping,to United States via Standard Shipping 7 | 5,Electric EMS Foot Massager Foot Muscle Stimulator Massager Wireless Low Frequency Feet Physiotherapy ABS Stimulator Massage Mat,"Online Shopping at a cheapest price for Automotive, Phones & Accessories, Computers & Electronics, Fashion, Beauty & Health, Home & Garden, Toys & Sports, Weddings & Events and more; just about anything else Enjoy ✓Free Shipping ! ✓Limited Time Sale✓Easy Return.",https://www.aliexpress.com/item/33010721339.html?,11.99 - 19.99,Free Shipping,to United States via USPS 8 | -------------------------------------------------------------------------------- /fbmp_mobile_sel_list.py: -------------------------------------------------------------------------------- 1 | """Selenium FB Mobile Marketplace Dropshipping Automator (SFBMMPDA). 2 | 3 | Allows user to leverage an excel sheet to automatically add products to FBMP Mobile 4 | 5 | Author: ehgp 6 | """ 7 | import datetime as dt 8 | import logging 9 | import logging.config 10 | import os 11 | import random 12 | import re 13 | import string 14 | import sys 15 | import time 16 | from getpass import getuser 17 | from os import listdir 18 | from os.path import isfile, join 19 | from pathlib import Path 20 | 21 | import chromedriver_autoinstaller 22 | import keyring 23 | import pandas as pd 24 | import pyautogui 25 | import yaml 26 | from selenium import webdriver 27 | from selenium.webdriver.chrome.options import Options 28 | from selenium.webdriver.common.by import By 29 | from selenium.webdriver.common.keys import Keys 30 | from selenium.webdriver.support import expected_conditions as EC 31 | from selenium.webdriver.support.ui import WebDriverWait 32 | 33 | from webscraperali import imagewebscraperali 34 | from webscraperchewy import imagewebscraperchewy 35 | from webscraperebay import imagewebscraperebay 36 | from webscraperetsy import imagewebscraperetsy 37 | from webscraperwayfair import imagewebscraperwayfair 38 | 39 | 40 | def format_filename(s): 41 | """Take a string and return a valid filename constructed from the string. 42 | 43 | Uses a whitelist approach: any characters not present in valid_chars are 44 | removed. 45 | 46 | Note: this method may produce invalid filenames such as ``, `.` or `..` 47 | When I use this method I prepend a date string like '2009_01_15_19_46_32_' 48 | and append a file extension like '.txt', so I avoid the potential of using 49 | an invalid filename. 50 | 51 | """ 52 | valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) 53 | filename = "".join(re.sub("[^A-Za-z0-9]+", " ", c) for c in s if c in valid_chars) 54 | filename = " ".join(filename.split()) 55 | return filename 56 | 57 | 58 | def _load_config(): 59 | """Load the configuration yaml and return dictionary of setttings. 60 | 61 | Returns: 62 | yaml as a dictionary. 63 | """ 64 | config_path = os.path.dirname(os.path.realpath(__file__)) 65 | config_path = os.path.join(config_path, "xpath_params.yaml") 66 | with open(config_path, "r") as config_file: 67 | config_defs = yaml.safe_load(config_file.read()) 68 | 69 | if config_defs.values() is None: 70 | raise ValueError("parameters yaml file incomplete") 71 | 72 | return config_defs 73 | 74 | 75 | # chrome_options = Options() 76 | # chrome_options.add_argument("--headless") 77 | # chrome_options.add_argument("--window-size=1920x1080") 78 | # from utils.encryption import create_encrypted_config, load_encrypted_config 79 | # os.makedirs(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET', exist_ok = True) 80 | # os.rmdir(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET') 81 | # Creds 82 | user = getuser() 83 | fb_email = keyring.get_password("FACEBOOK_EMAIL", user) 84 | fb_pass = keyring.get_password("FACEBOOK_PASSWORD", user) 85 | 86 | # Paths 87 | path = Path(os.getcwd()) 88 | # binary_path = Path(path, "chromedriver.exe") 89 | chromedriver_autoinstaller.install() 90 | dropship_sh_path = Path(path, "Dropshipping Items", "DROPSHIPPING_SPREADSHEET.xlsx") 91 | dropship_path = Path(path, "Droppping Items") 92 | 93 | # Logging 94 | Path("log").mkdir(parents=True, exist_ok=True) 95 | log_config = Path(path, "log_config.yaml") 96 | timestamp = "{:%Y_%m_%d_%H_%M_%S}".format(dt.datetime.now()) 97 | with open(log_config, "r") as log_file: 98 | config_dict = yaml.safe_load(log_file.read()) 99 | # Append date stamp to the file name 100 | log_filename = config_dict["handlers"]["file"]["filename"] 101 | base, extension = os.path.splitext(log_filename) 102 | base2 = "_" + os.path.splitext(os.path.basename(__file__))[0] + "_" 103 | log_filename = "{}{}{}{}".format(base, base2, timestamp, extension) 104 | config_dict["handlers"]["file"]["filename"] = log_filename 105 | logging.config.dictConfig(config_dict) 106 | logger = logging.getLogger(__name__) 107 | 108 | 109 | # OpenBrowser 110 | driver = webdriver.Chrome() 111 | driver.get("https://facebook.com/") 112 | 113 | # Logging 114 | email = driver.find_element_by_id("email") 115 | password = driver.find_element_by_id("pass") 116 | # facebook username input 117 | email.send_keys(fb_email) 118 | # facebook password input 119 | password.send_keys(fb_pass) 120 | 121 | 122 | time.sleep(5) 123 | driver.find_element_by_name("login").click() 124 | # Redicret to fb marketplace 125 | time.sleep(2) 126 | driver.get("https://m.facebook.com/marketplace/selling/item/") 127 | 128 | # Filling up form 129 | driver.find_element_by_xpath( 130 | "/html/body/div[1]/div/div[4]/div/div[1]/div/form/div[1]/div/div[1]/div/div/div/div/div/div[1]" 131 | ).click() 132 | time.sleep(4) # waiting for window popup to open 133 | pyautogui.write(r"C:\Users\ehgp\Pictures\6V.png") # path of File 134 | pyautogui.press("enter") 135 | 136 | time.sleep(5) 137 | title = driver.find_element_by_name("title").send_keys("title teste") 138 | 139 | time.sleep(2) 140 | price = driver.find_element_by_name("price").send_keys("50") 141 | 142 | time.sleep(2) 143 | driver.find_element_by_xpath( 144 | "/html/body/div[1]/div/div[4]/div/div[1]/div/form/div[2]/div/div/div[4]/div/div[1]/div/div[2]/input" 145 | ).click() 146 | 147 | time.sleep(2) 148 | driver.find_element_by_xpath( 149 | "/html/body/div[2]/div/div[2]/div/div/div[23]/div[1]" 150 | ).click() 151 | # driver.find_element_by_xpath("/html/body/div[2]/div/div[2]/div/div/div[35]/div[2]").click() 152 | 153 | time.sleep(2) 154 | description = driver.find_element_by_name("description").send_keys("description teste") 155 | 156 | time.sleep(2) 157 | driver.find_element_by_xpath( 158 | "/html/body/div[1]/div/div[4]/div/div[1]/div/form/div[2]/div/div/div[7]/div[3]" 159 | ).click() 160 | -------------------------------------------------------------------------------- /.github/workflows/python-lint-analyze-test-build-sphinx.yml: -------------------------------------------------------------------------------- 1 | # Workflow will use Python 3.8 in lint stages using flake8, black and pydocstyle, analyze stage with codeQL, test stage using pytest and build stage using Sphinx 2 | name: python-lint-analyze-test-build-sphinx 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Checkout code 14 | uses: actions/checkout@v3 15 | with: 16 | # Full git history is needed to get a proper list of changed files within `super-linter` 17 | fetch-depth: 0 18 | 19 | - name: Lint Code Base 20 | uses: github/super-linter/slim@v4 21 | env: 22 | VALIDATE_ALL_CODEBASE: false 23 | VALIDATE_JSCPD: false 24 | VALIDATE_PYTHON_MYPY: false 25 | DEFAULT_BRANCH: main 26 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | pre-commit: 29 | runs-on: ubuntu-latest 30 | strategy: 31 | fail-fast: false 32 | matrix: 33 | python-version: [3.8] 34 | steps: 35 | - name: Checkout repository 36 | uses: actions/checkout@v3 37 | 38 | - name: lint 39 | uses: pre-commit/action@v3.0.0 40 | with: 41 | extra_args: --all-files 42 | 43 | analyze: 44 | name: Analyze 45 | runs-on: ubuntu-latest 46 | permissions: 47 | actions: read 48 | contents: read 49 | security-events: write 50 | 51 | strategy: 52 | fail-fast: false 53 | matrix: 54 | language: ["python"] 55 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 56 | # Learn more about CodeQL language support at https://git.io/codeql-language-support 57 | 58 | steps: 59 | - name: Checkout repository 60 | uses: actions/checkout@v3 61 | 62 | # Initializes the CodeQL tools for scanning. 63 | - name: Initialize CodeQL 64 | uses: github/codeql-action/init@v2 65 | with: 66 | languages: ${{ matrix.language }} 67 | # If you wish to specify custom queries, you can do so here or in a config file. 68 | # By default, queries listed here will override any specified in a config file. 69 | # Prefix the list here with "+" to use these queries and those in the config file. 70 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 71 | 72 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 73 | # If this step fails, then you should remove it and run the build manually (see below) 74 | - name: Autobuild 75 | uses: github/codeql-action/autobuild@v2 76 | 77 | # ℹ️ Command-line programs to run using the OS shell. 78 | # 📚 https://git.io/JvXDl 79 | 80 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 81 | # and modify them (or add more) to build your code if your project 82 | # uses a compiled language 83 | 84 | #- run: | 85 | # make bootstrap 86 | # make release 87 | 88 | - name: Perform CodeQL Analysis 89 | uses: github/codeql-action/analyze@v2 90 | 91 | # test: 92 | # runs-on: ubuntu-latest 93 | # strategy: 94 | # fail-fast: false 95 | # matrix: 96 | # python-version: [3.8] 97 | # steps: 98 | # - name: Checkout repository 99 | # uses: actions/checkout@v3 100 | 101 | # - name: Set up Python 3.8 102 | # uses: actions/setup-python@v3 103 | # with: 104 | # python-version: ${{ matrix.python-version }} 105 | # # Optional - x64 or x86 architecture, defaults to x64 106 | # architecture: "x64" 107 | 108 | # # You can test your matrix by printing the current Python version 109 | # - name: Display Python version 110 | # run: python -c "import sys; print(sys.version)" 111 | 112 | # - name: test 113 | # run: | 114 | # apt-get update --quiet 115 | # apt-get upgrade -y --quiet 116 | # apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 117 | # pip install --upgrade pip --quiet 118 | # pip install -r requirements.txt --quiet 119 | # pip install pytest --quiet 120 | # pip install . --quiet 121 | # python -m pytest 122 | 123 | build: 124 | runs-on: ubuntu-latest 125 | strategy: 126 | fail-fast: false 127 | matrix: 128 | python-version: [3.8] 129 | steps: 130 | - name: Checkout repository 131 | uses: actions/checkout@v3 132 | 133 | - name: Set up Python 3.8 134 | uses: actions/setup-python@v3 135 | with: 136 | python-version: ${{ matrix.python-version }} 137 | # Optional - x64 or x86 architecture, defaults to x64 138 | architecture: "x64" 139 | 140 | # You can test your matrix by printing the current Python version 141 | - name: Display Python version 142 | run: python -c "import sys; print(sys.version)" 143 | 144 | - name: Sphinx Pages 145 | run: | 146 | pip install --upgrade pip --quiet 147 | pip install -r requirements.txt --quiet 148 | sphinx-build -b html docs/source public 149 | 150 | - name: Upload artifacts 151 | uses: actions/upload-artifact@v1 152 | with: 153 | name: html-docs 154 | path: public 155 | 156 | - name: Deploy 157 | uses: peaceiris/actions-gh-pages@v3 158 | if: github.ref == 'refs/heads/main' 159 | with: 160 | github_token: ${{ secrets.GITHUB_TOKEN }} 161 | publish_dir: public 162 | -------------------------------------------------------------------------------- /.gitlab/.gitlab-ci-py.yml: -------------------------------------------------------------------------------- 1 | default: 2 | image: python:3.7 3 | before_script: 4 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 5 | 6 | cache: 7 | paths: 8 | - .cache/node_modules 9 | # - .venv/ 10 | # - .conda/ 11 | # - .poetry/ 12 | only: 13 | - main 14 | 15 | variables: 16 | GIT_STRATEGY: clone 17 | PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" 18 | SAST_DEFAULT_ANALYZERS: semgrep,secrets,dependency-check 19 | SECRET_DETECTION_HISTORIC_SCAN: "true" 20 | 21 | stages: 22 | - lint 23 | - test 24 | - pages 25 | - release 26 | 27 | # Doc: https://pre-commit.com/ 28 | lint: 29 | stage: lint 30 | allow_failure: true 31 | before_script: 32 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 33 | - apt-get update && apt-get upgrade -y --quiet 34 | - apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 35 | - changed_files=$(git diff --no-commit-id --name-only -r $CI_COMMIT_SHA) 36 | - pip install --upgrade pip setuptools wheel pipenv virtualenv conda poetry -- quiet 37 | # Base 38 | # - pip install -r requirements.txt 39 | # # Virtualenv 40 | # - virtualenv venv 41 | # - source venv/bin/activate 42 | # # Pipenv 43 | # - pipenv install 44 | # # Conda Env 45 | # - conda env create -f environment.yml 46 | # - conda activate myenv 47 | # # Poetry 48 | # - poetry install 49 | script: 50 | - pip install --upgrade pip --quiet 51 | - pip install -r requirements.txt 52 | - pip install pre-commit==2.20.0 --quiet 53 | - pre-commit install 54 | - pre-commit run --all-files 55 | 56 | # Doc: https://docs.gitlab.com/ee/user/application_security/sast/ 57 | sast: 58 | stage: test 59 | before_script: 60 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 61 | allow_failure: true 62 | include: 63 | - template: Jobs/SAST.gitlab-ci.yml 64 | - template: Jobs/License-Scanning.gitlab-ci.yml 65 | - template: Jobs/Dependency-Scanning.gitlab-ci.yml 66 | - template: Jobs/Secret-Detection.gitlab-ci.yml 67 | 68 | # Doc: https://docs.python.org/3/library/doctest.html 69 | doctest: 70 | stage: test 71 | allow_failure: true 72 | script: 73 | - pip install -r requirements.txt 74 | - python -m doctest -v 75 | 76 | # Doc: https://docs.pytest.org/en/stable/ 77 | test: 78 | stage: test 79 | allow_failure: true 80 | before_script: 81 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 82 | - apt-get update && apt-get upgrade -y --quiet 83 | - apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 84 | - changed_files=$(git diff --no-commit-id --name-only -r $CI_COMMIT_SHA) 85 | - pip install --upgrade pip setuptools wheel pipenv virtualenv conda poetry -- quiet 86 | # Base 87 | # - pip install -r requirements.txt 88 | # # Virtualenv 89 | # - virtualenv venv 90 | # - source venv/bin/activate 91 | # # Pipenv 92 | # - pipenv install 93 | # # Conda Env 94 | # - conda env create -f environment.yml 95 | # - conda activate myenv 96 | # # Poetry 97 | # - poetry install 98 | script: 99 | - pip install -r requirements.txt 100 | - pip install pytest --quiet 101 | - python -m pytest -v --junitxml=pytest.xml 102 | artifacts: 103 | when: always 104 | reports: 105 | junit: pytest.xml 106 | 107 | # Doc: https://sphinx-doc.org/en/master/usage/index.html 108 | pages: 109 | stage: pages 110 | before_script: 111 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 112 | - apt-get update && apt-get upgrade -y --quiet 113 | - apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 114 | - changed_files=$(git diff --no-commit-id --name-only -r $CI_COMMIT_SHA) 115 | - pip install --upgrade pip setuptools wheel pipenv virtualenv conda poetry -- quiet 116 | # Base 117 | # - pip install -r requirements.txt 118 | # # Virtualenv 119 | # - virtualenv venv 120 | # - source venv/bin/activate 121 | # # Pipenv 122 | # - pipenv install 123 | # # Conda Env 124 | # - conda env create -f environment.yml 125 | # - conda activate myenv 126 | # # Poetry 127 | # - poetry install 128 | script: 129 | - pip install -U sphinx --quiet 130 | - pip install -U sphinx_rtd_theme --quiet 131 | - pip install -U recommonmark --quiet 132 | - pip install -U pyyaml --quiet 133 | - pip install -r requirements.txt --quiet 134 | - sphinx-build -b html docs/source public 135 | artifacts: 136 | paths: 137 | - public 138 | only: 139 | - main 140 | 141 | # Doc: https://twine.readthedocs.io/en/stable/ 142 | release: 143 | stage: release 144 | allow_failure: true 145 | before_script: 146 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 147 | - apt-get update && apt-get upgrade -y --quiet 148 | - apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 149 | - changed_files=$(git diff --no-commit-id --name-only -r $CI_COMMIT_SHA) 150 | - pip install --upgrade pip setuptools wheel pipenv virtualenv conda poetry -- quiet 151 | # Base 152 | # - pip install -r requirements.txt 153 | # # Virtualenv 154 | # - virtualenv venv 155 | # - source venv/bin/activate 156 | # # Pipenv 157 | # - pipenv install 158 | # # Conda Env 159 | # - conda env create -f environment.yml 160 | # - conda activate myenv 161 | # # Poetry 162 | # - poetry install 163 | script: 164 | - pip install --upgrade pip --quiet 165 | - pip install -r requirements.txt 166 | - pip install -U twine --quiet 167 | - python setup.py sdist bdist_wheel 168 | - python -m twine upload 169 | --repository-url https://gitlab.com/api/v4/projects/$CI_PROJECT_ID/packages/pypi 170 | --username "gitlab-ci-token" 171 | --password $CI_JOB_TOKEN 172 | --verbose 173 | dist/* 174 | only: tags 175 | 176 | check-merge-request: 177 | stage: test 178 | before_script: 179 | - echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" > ~/.netrc 180 | - apt-get update && apt-get upgrade -y --quiet 181 | - apt-get install build-essential libssl-dev libffi-dev python3-dev -y --quiet 182 | - changed_files=$(git diff --no-commit-id --name-only -r $CI_COMMIT_SHA) 183 | - pip install --upgrade pip setuptools wheel pipenv virtualenv conda poetry -- quiet 184 | # Base 185 | # - pip install -r requirements.txt 186 | # # Virtualenv 187 | # - virtualenv venv 188 | # - source venv/bin/activate 189 | # # Pipenv 190 | # - pipenv install 191 | # # Conda Env 192 | # - conda env create -f environment.yml 193 | # - conda activate myenv 194 | # # Poetry 195 | # - poetry install 196 | allow_failure: true 197 | script: 198 | - pip install pre-commit pytest --quiet 199 | - pre-commit install 200 | - git fetch 201 | - git checkout $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME 202 | - git checkout $CI_MERGE_REQUEST_TARGET_BRANCH_NAME 203 | # Identify cumulative additions/changes in merge request 204 | - changed_files_in_MR=$(git diff --no-commit-id --name-only -r $CI_MERGE_REQUEST_TARGET_BRANCH_NAME) 205 | - echo "Changed files in MR= $changed_files_in_MR" 206 | - echo "Running pre-commit and pytest on changed files in MR" 207 | - pre-commit run --files $changed_files_in_MR 208 | # - python -m pytest -v --junitxml=pytest.xml 209 | # artifacts: 210 | # when: always 211 | # reports: 212 | # junit: pytest.xml 213 | rules: 214 | - if: "$CI_MERGE_REQUEST_ID" 215 | -------------------------------------------------------------------------------- /xpath_params.yaml: -------------------------------------------------------------------------------- 1 | GENERAL_PARAMS: 2 | 3 | DEFAULT_HEADERS: | 4 | { 5 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", 6 | "accept-encoding": "gzip, deflate, br", 7 | "accept-language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7", 8 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36", 9 | "origin": "https://www.facebook.com", 10 | } 11 | 12 | FB_PARAMS: 13 | 14 | FB_LOGIN: "https://www.facebook.com/login/device-based/regular/login/?login_attempt=1" 15 | 16 | FB_LISTING: "https://www.facebook.com/marketplace/create/item" 17 | 18 | FB_IMAGE_UPLOAD_XPATH: "//input[@type='file']" 19 | 20 | FB_TITLE_XPATH: "//*[@aria-label='Title']" 21 | 22 | FB_AVAIL_QTY_XPATH: "//*[@aria-label='Available Quantity']" 23 | 24 | FB_PRICE_XPATH: "//*[@aria-label='Price per item']" 25 | 26 | FB_PRICE_ONE_XPATH: "//*[@aria-label='Price']" 27 | 28 | FB_CAT_XPATH: "//*[@aria-label='Category']" 29 | 30 | FB_CAT_MISC_XPATH: "//*[@role='radio']" 31 | 32 | FB_COND_XPATH: "//*[@aria-label='Condition']" 33 | 34 | FB_COND_NEW_XPATH: "//*[@role='menuitemradio']" 35 | 36 | FB_DESCRIPTION_XPATH: "//*[@aria-label='Description']" 37 | 38 | FB_TAGS_XPATH: "//*[@aria-label='Product tags']" 39 | 40 | FB_NEXT_PAGE_SHIPPING_XPATH: "//*[@aria-label='Next']" 41 | 42 | FB_SHIPPING_TYPE_XPATH: "//*[@aria-label='Delivery method']" 43 | 44 | FB_SHIPPING_ONLY_XPATH: "//div[@aria-checked='false'][@role='menuitemradio']" 45 | 46 | FB_SHIPPING_OPT_ACT_XPATH: "//*[@aria-label='Shipping option']" 47 | 48 | FB_SHIPPING_OPT_OWN_XPATH: "//div[@aria-checked='false'][@role='menuitemradio']" 49 | 50 | FB_SHIPPING_RATE_CLICK_INPUT_XPATH: "//*[@aria-label='Shipping rate per item']" 51 | 52 | FB_SHIPPING_RATE_CLICK_INPUT_SINGLE_XPATH: "//*[@aria-label='Shipping rate']" 53 | 54 | FB_SHIPPING_FREE_XPATH: "//input[@name='offer_free_shipping_checkbox']" 55 | 56 | FB_NEXT_PAGE_OFFER_XPATH: "//*[@aria-label='Next']" 57 | 58 | FB_NEXT_PAGE_PUBLISH_XPATH: "//*[@aria-label='Next']" 59 | 60 | FB_PUBLISH_BUTTON: "//*[@aria-label='Publish']" 61 | 62 | EBAY_PARAMS: 63 | 64 | EBAY_LOGIN: "https://signin.ebay.com/ws/eBayISAPI.dll?SignIn" 65 | 66 | EBAY_USERNAME_XPATH: "//input[@name='email']" 67 | 68 | EBAY_PASSWORD_XPATH: "//input[@name='password']" 69 | 70 | EBAY_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 71 | 72 | EBAY_LISTING: 'https://www.ebay.com/sl/prelist/suggest' 73 | 74 | EBAY_START_LISTING: "//input[@aria-label='Enter brand, model, or other details (ISBN, MPN, VIN)']" 75 | 76 | EBAY_IMAGE_UPLOAD_XPATH: "//form[@id='listing-photo-uploader']" 77 | 78 | # Two different instances 79 | 80 | EBAY_TITLE_XPATH: "//input[@name='title']" 81 | 82 | EBAY_WHO_MADE: "//select[@id='who_made']" 83 | 84 | EBAY_WHO_MADE_SOMEONE_ELSE: "//option[@value='someone_else']" 85 | 86 | EBAY_WHAT_IS_IT: "//select[@id='is_supply']" 87 | 88 | EBAY_WHAT_IS_IT_PRODUCT: "//option[@value='0']" 89 | 90 | EBAY_WHEN_MADE: "//select[@id='when_made']" 91 | 92 | EBAY_WHEN_MADE_TO_ORDER: "//option[@value='made_to_order']" 93 | 94 | EBAY_CTGRY_XPATH: "//input[@id='taxonomy-search']" 95 | 96 | EBAY_RENEW_LISTING: "/html/body/div[3]/section/div/div[4]/div[1]/div/div/div[2]/div/div/div/div[5]/div[12]/div/fieldset/div[2]/div[1]/div[2]/label/span" 97 | 98 | EBAY_DES_XPATH: "//label[@for='description']" 99 | 100 | EBAY_DES1_XPATH: "//textarea[@name='description']" 101 | 102 | EBAY_PRICE_XPATH: "//input[@name='price-input']" 103 | 104 | EBAY_QTY_XPATH: "//input[@name='quantity-input']" 105 | 106 | EBAY_SHIP_XPATH: "//label[@for='125349542613']" 107 | 108 | EBAY_PUBLISH_XPATH: "//button[@class='btn btn-primary']" 109 | 110 | EBAY_CONFIRM_XPATH: "//button[@data-ui='confirm']" 111 | 112 | ETSY_PARAMS: 113 | 114 | ETSY_LOGIN: 'https://www.etsy.com/signin' 115 | 116 | ETSY_USERNAME_XPATH: "//input[@name='email']" 117 | 118 | ETSY_PASSWORD_XPATH: "//input[@name='password']" 119 | 120 | ETSY_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 121 | 122 | ETSY_IMAGE_UPLOAD_XPATH: "//form[@id='listing-photo-uploader']" 123 | 124 | ETSY_LISTING: 'https://www.etsy.com/your/shops/VentureStoreBoutique/tools/listings/create' 125 | 126 | ETSY_TITLE_XPATH: "//input[@name='title']" 127 | 128 | ETSY_WHO_MADE: "//select[@id='who_made']" 129 | 130 | ETSY_WHO_MADE_SOMEONE_ELSE: "//option[@value='someone_else']" 131 | 132 | ETSY_WHAT_IS_IT: "//select[@id='is_supply']" 133 | 134 | ETSY_WHAT_IS_IT_PRODUCT: "//option[@value='0']" 135 | 136 | ETSY_WHEN_MADE: "//select[@id='when_made']" 137 | 138 | ETSY_WHEN_MADE_TO_ORDER: "//option[@value='made_to_order']" 139 | 140 | ETSY_CTGRY_XPATH: "//input[@id='taxonomy-search']" 141 | 142 | ETSY_RENEW_LISTING: "/html/body/div[3]/section/div/div[4]/div[1]/div/div/div[2]/div/div/div/div[5]/div[12]/div/fieldset/div[2]/div[1]/div[2]/label/span" 143 | 144 | ETSY_DES_XPATH: "//label[@for='description']" 145 | 146 | ETSY_DES1_XPATH: "//textarea[@name='description']" 147 | 148 | ETSY_PRICE_XPATH: "//input[@name='price-input']" 149 | 150 | ETSY_QTY_XPATH: "//input[@name='quantity-input']" 151 | 152 | ETSY_SHIP_XPATH: "//label[@for='125349542613']" 153 | 154 | ETSY_PUBLISH_XPATH: "//button[@class='btn btn-primary']" 155 | 156 | ETSY_CONFIRM_XPATH: "//button[@data-ui='confirm']" 157 | 158 | ALI_WEBSCRAPER_PARAMS: 159 | 160 | PAGES: 3 161 | 162 | ALI_LOGIN: "https://login.aliexpress.com/" 163 | 164 | ALI_USERNAME_XPATH: "//input[@name='fm-login-id']" 165 | 166 | ALI_PASSWORD_XPATH: "//input[@name='fm-login-password']" 167 | 168 | ALI_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 169 | 170 | ALI_US_SHIP: "//span[text()='United States']" 171 | 172 | ALI_SOUP_TITLE_EXT: '"h1", attrs={"class": "product-title-text"}' 173 | 174 | ALI_SOUP_DES_EXT: '"meta", attrs={"name": "description"}' 175 | 176 | ALI_SOUP_PRICE_EXT: '"span", attrs={"itemprop": "price"}' 177 | 178 | ALI_SOUP_SHIPPRICE_EXT: '"div", attrs={"class": "product-shipping-price"}' 179 | 180 | ALI_SOUP_SHIPHOW_EXT: '"span", attrs={"class": "product-shipping-info black-link"}' 181 | 182 | ALI_XPATH_WISHLIST_PAGE: "https://my.aliexpress.com/wishlist/wish_list_product_list.htm?page=" 183 | 184 | ALI_IMAGEPATHLIST_HTML: r'("imagePathList":\[.*?)\]' 185 | 186 | CHEWY_WEBSCRAPER_PARAMS: 187 | 188 | PAGES: 3 189 | 190 | CHEWY_LOGIN: "https://www.chewy.com/app/login?" 191 | 192 | CHEWY_USERNAME_XPATH: "//input[@id='username']" 193 | 194 | CHEWY_PASSWORD_XPATH: "//input[@id='password']" 195 | 196 | CHEWY_LOGIN_BUTTON_XPATH: "//input[@name='submitForm']" 197 | 198 | CHEWY_LISTS: "https://www.chewy.com/app/account/favorites" 199 | 200 | CHEWY_SOUP_SHIPHOW_EXT: '"span", attrs={"class": "free-shipping js-edd-conditional-display"}' 201 | 202 | CHEWY_SOUP_AUTOSHIPPRICE_EXT: '"div", attrs={"id": "autoship-pricing"}' 203 | 204 | CHEWY_SOUP_COLORLIST_EXT: '"div", attrs={"id": "variation-Color"}' 205 | 206 | CHEWY_SOUP_COLORLIST_FINDALL_EXT: '"span", attrs={"class": "cw-visually-hidden"}' 207 | 208 | CHEWY_SOUP_SIZELIST_EXT: '"div", attrs={"id": "variation-Size"}' 209 | 210 | CHEWY_SOUP_SIZELIST_FINDALL_EXT: '"span", attrs={"class": "cw-visually-hidden"}' 211 | 212 | CHEWY_XPATH_CLICKMYFAV: "//div[@class='ListCard-name'][text()='My Favorites']" 213 | 214 | CHEWY_SOUP_IMAGEFILE_EXT: '"img", attrs={"class": "pl-FluidImage-image"}' 215 | 216 | EBAY_WEBSCRAPER_PARAMS: 217 | 218 | PAGES: 3 219 | 220 | EBAY_LOGIN: "https://www.ebay.com/signin/" 221 | 222 | EBAY_USERNAME_XPATH: "//input[@name='email']" 223 | 224 | EBAY_PASSWORD_XPATH: "//input[@name='password']" 225 | 226 | EBAY_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 227 | 228 | EBAY_SOUP_LINK_FINDALL_EXT: '"a", attrs={"class": "display-inline-block listing-link"}' 229 | 230 | EBAY_SOUP_TITLE_EXT: "h1.it-ttl" 231 | 232 | EBAY_SOUP_DES_EXT: 'id="desc_ifr"' 233 | 234 | EBAY_SOUP_PRICE_EXT: '"span", attrs={"itemprop": "price"}' 235 | 236 | EBAY_SOUP_SHIPPRICE_EXT: '"span", attrs={"id": "fshippingCost"}' 237 | 238 | EBAY_SOUP_SHIPHOW_EXT: '"span", attrs={"itemprop": "availableAtOrFrom"}' 239 | 240 | EBAY_LINK_LIST: "https://www.ebay.com/mye/myebay/watchlist?custom_list_id=WATCH_LIST&page_number=" 241 | 242 | EBAY_SOUP_IMAGELINK_96_EXT: r"(http:|https:)(\/\/i.ebayimg.com[^\"\']*)(96.png|96.jpg|96.jpeg|96.gif|96.png|96.svg|96.webp)" 243 | 244 | EBAY_SOUP_IMAGELINK_300_EXT: r"(http:|https:)(\/\/i.ebayimg.com[^\"\']*)(300.png|300.jpg|300.jpeg|300.gif|300.png|300.svg|300.webp)" 245 | 246 | EBAY_SOUP_IMAGELINK_64_EXT: r"(http:|https:)(\/\/i.ebayimg.com[^\"\']*)(64.png|64.jpg|64.jpeg|64.gif|64.png|64.svg|64.webp)" 247 | 248 | ETSY_WEBSCRAPER_PARAMS: 249 | 250 | PAGES: 5 251 | 252 | ETSY_LOGIN: "https://www.etsy.com/signin" 253 | 254 | ETSY_USERNAME_XPATH: "//input[@name='email']" 255 | 256 | ETSY_PASSWORD_XPATH: "//input[@name='password']" 257 | 258 | ETSY_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 259 | 260 | ETSY_SOUP_SHIPPRICE_EXT: '"p", attrs={"class": "wt-text-body-03 wt-mt-xs-1 wt-line-height-tight"},' 261 | 262 | ETSY_SOUP_SHIPHOW_EXT: '"div", attrs={"class": "wt-grid__item-xs-12 wt-text-black wt-text-caption"},' 263 | 264 | ETSY_LINK_LIST: 'https://www.etsy.com/people/ehgp?ref=hdr_user_menu-profile&page=' 265 | 266 | ETSY_SOUP_IMAGELINK_EXT: r"(http:|https:)(\/\/[^\"\']*\.(?:png|jpg|jpeg|gif|png|svg|webp))" 267 | 268 | WAYFAIR_WEBSCRAPER_PARAMS: 269 | 270 | PAGES: 3 271 | 272 | WAYFAIR_LOGIN: "https://www.wayfair.com/v/account/authentication/login?" 273 | 274 | WAYFAIR_USERNAME_XPATH: "//input[@id='textInput-0']" 275 | 276 | WAYFAIR_PASSWORD_XPATH: "//input[@id='textInput-1']" 277 | 278 | WAYFAIR_LOGIN_BUTTON_XPATH: "//button[@type='submit']" 279 | 280 | WAYFAIR_LISTS: "https://www.wayfair.com/lists" 281 | 282 | WAYFAIR_WT_DIM_XPATH: "//div[text()='Weights & Dimensions']" 283 | 284 | WAYFAIR_SPEC_XPATH: "//div[text()='Specifications']" 285 | 286 | WAYFAIR_SOUP_SHIPPRICE_EXT: '"span", attrs={"class": "ShippingHeadline-text"}' 287 | 288 | WAYFAIR_SOUP_SHIPHOW_EXT: '"div", attrs={"data-enzyme-id": "FullWidthShippingOptions"}' 289 | 290 | WAYFAIR_FAVLIST_XPATH: "//div[@class='ListCard-name'][text()='My Favorites']" 291 | 292 | WAYFAIR_SOUP_LINK_EXT: '"a", attrs={"class": "pl-ProductCard-productCardElement"}' 293 | 294 | WAYFAIR_SOUP_IMGLINK_EXT: '"img", attrs={"class": "pl-FluidImage-image"}' 295 | 296 | -------------------------------------------------------------------------------- /ebay_desktop_sel_list.py: -------------------------------------------------------------------------------- 1 | """Selenium EBAY Marketplace Dropshipping Automator (SEMPDA). 2 | 3 | Allows user to leverage an excel sheet to automatically add products to EBAY 4 | 5 | Author: ehgp 6 | """ 7 | import datetime as dt 8 | import logging 9 | import logging.config 10 | import os 11 | import random 12 | import re 13 | import shutil 14 | import string 15 | import time 16 | from getpass import getuser 17 | from os import listdir 18 | from os.path import isfile, join 19 | from pathlib import Path 20 | 21 | import chromedriver_autoinstaller 22 | import keyring 23 | import pandas as pd 24 | import pyautogui 25 | import yaml 26 | from selenium import webdriver 27 | from selenium.webdriver.chrome.options import Options 28 | from selenium.webdriver.common.by import By 29 | from selenium.webdriver.common.keys import Keys 30 | from selenium.webdriver.support import expected_conditions as EC 31 | from selenium.webdriver.support.ui import WebDriverWait 32 | 33 | from webscraperali import imagewebscraperali 34 | from webscraperchewy import imagewebscraperchewy 35 | from webscraperebay import imagewebscraperebay 36 | from webscraperetsy import imagewebscraperetsy 37 | from webscraperwayfair import imagewebscraperwayfair 38 | 39 | 40 | def format_filename(s): 41 | """Take a string and return a valid filename constructed from the string. 42 | 43 | Uses a whitelist approach: any characters not present in valid_chars are 44 | removed. 45 | 46 | Note: this method may produce invalid filenames such as ``, `.` or `..` 47 | When I use this method I prepend a date string like '2009_01_15_19_46_32_' 48 | and append a file extension like '.txt', so I avoid the potential of using 49 | an invalid filename. 50 | 51 | """ 52 | valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) 53 | filename = "".join(re.sub("[^A-Za-z0-9]+", " ", c) for c in s if c in valid_chars) 54 | filename = " ".join(filename.split()) 55 | return filename 56 | 57 | 58 | def _load_config(): 59 | """Load the configuration yaml and return dictionary of setttings. 60 | 61 | Returns: 62 | yaml as a dictionary. 63 | """ 64 | config_path = os.path.dirname(os.path.realpath(__file__)) 65 | config_path = os.path.join(config_path, "xpath_params.yaml") 66 | with open(config_path, "r") as config_file: 67 | config_defs = yaml.safe_load(config_file.read()) 68 | 69 | if config_defs.values() is None: 70 | raise ValueError("parameters yaml file incomplete") 71 | 72 | return config_defs 73 | 74 | 75 | # from utils.encryption import create_encrypted_config, load_encrypted_config 76 | # os.makedirs(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET', exist_ok = True) 77 | # os.rmdir(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET') 78 | 79 | # Paths 80 | path = Path(os.getcwd()) 81 | # binary_path = Path(path, "chromedriver.exe") 82 | chromedriver_autoinstaller.install() 83 | dropship_sh_path = Path(path, "Dropshipping Items", "DROPSHIPPING_SPREADSHEET.xlsx") 84 | dropship_path = Path(path, "Dropshipping Items") 85 | 86 | # Logging 87 | Path("log").mkdir(parents=True, exist_ok=True) 88 | log_config = Path(path, "log_config.yaml") 89 | timestamp = "{:%Y_%m_%d_%H_%M_%S}".format(dt.datetime.now()) 90 | with open(log_config, "r") as log_file: 91 | config_dict = yaml.safe_load(log_file.read()) 92 | # Append date stamp to the file name 93 | log_filename = config_dict["handlers"]["file"]["filename"] 94 | base, extension = os.path.splitext(log_filename) 95 | base2 = "_" + os.path.splitext(os.path.basename(__file__))[0] + "_" 96 | log_filename = "{}{}{}{}".format(base, base2, timestamp, extension) 97 | config_dict["handlers"]["file"]["filename"] = log_filename 98 | logging.config.dictConfig(config_dict) 99 | logger = logging.getLogger(__name__) 100 | 101 | logger.info("Get Credentials") 102 | user = getuser() 103 | ebay_email = keyring.get_password("EBAY_EMAIL", user) 104 | ebay_pass = keyring.get_password("EBAY_PASSWORD", user) 105 | if (ebay_email or ebay_pass) is None: 106 | logger.info("Incomplete credentials") 107 | exit() 108 | 109 | cf = _load_config() 110 | 111 | DEFAULT_HEADERS = cf["GENERAL_PARAMS"]["DEFAULT_HEADERS"] 112 | 113 | EBAY_LOGIN = cf["EBAY_PARAMS"]["EBAY_LOGIN"] 114 | 115 | EBAY_USERNAME_XPATH = cf["EBAY_PARAMS"]["EBAY_USERNAME_XPATH"] 116 | 117 | EBAY_PASSWORD_XPATH = cf["EBAY_PARAMS"]["EBAY_PASSWORD_XPATH"] 118 | 119 | EBAY_LOGIN_BUTTON_XPATH = cf["EBAY_PARAMS"]["EBAY_LOGIN_BUTTON_XPATH"] 120 | 121 | EBAY_LISTING = cf["EBAY_PARAMS"]["EBAY_LISTING"] 122 | 123 | EBAY_START_LISTING = cf["EBAY_PARAMS"]["EBAY_START_LISTING"] 124 | 125 | EBAY_IMAGE_UPLOAD_XPATH = cf["EBAY_PARAMS"]["EBAY_IMAGE_UPLOAD_XPATH"] 126 | 127 | EBAY_TITLE_XPATH = cf["EBAY_PARAMS"]["EBAY_TITLE_XPATH"] 128 | 129 | EBAY_WHO_MADE = cf["EBAY_PARAMS"]["EBAY_WHO_MADE"] 130 | 131 | EBAY_WHO_MADE_SOMEONE_ELSE = cf["EBAY_PARAMS"]["EBAY_WHO_MADE_SOMEONE_ELSE"] 132 | 133 | EBAY_WHAT_IS_IT = cf["EBAY_PARAMS"]["EBAY_WHAT_IS_IT"] 134 | 135 | EBAY_WHAT_IS_IT_PRODUCT = cf["EBAY_PARAMS"]["EBAY_WHAT_IS_IT_PRODUCT"] 136 | 137 | EBAY_WHEN_MADE = cf["EBAY_PARAMS"]["EBAY_WHEN_MADE"] 138 | 139 | EBAY_WHEN_MADE_TO_ORDER = cf["EBAY_PARAMS"]["EBAY_WHEN_MADE_TO_ORDER"] 140 | 141 | EBAY_CTGRY_XPATH = cf["EBAY_PARAMS"]["EBAY_CTGRY_XPATH"] 142 | 143 | EBAY_RENEW_LISTING = cf["EBAY_PARAMS"]["EBAY_RENEW_LISTING"] 144 | 145 | EBAY_DES_XPATH = cf["EBAY_PARAMS"]["EBAY_DES_XPATH"] 146 | 147 | EBAY_DES1_XPATH = cf["EBAY_PARAMS"]["EBAY_DES1_XPATH"] 148 | 149 | EBAY_PRICE_XPATH = cf["EBAY_PARAMS"]["EBAY_PRICE_XPATH"] 150 | 151 | EBAY_QTY_XPATH = cf["EBAY_PARAMS"]["EBAY_QTY_XPATH"] 152 | 153 | EBAY_SHIP_XPATH = cf["EBAY_PARAMS"]["EBAY_SHIP_XPATH"] 154 | 155 | EBAY_PUBLISH_XPATH = cf["EBAY_PARAMS"]["EBAY_PUBLISH_XPATH"] 156 | 157 | EBAY_CONFIRM_XPATH = cf["EBAY_PARAMS"]["EBAY_CONFIRM_XPATH"] 158 | 159 | logger.info("Configure ChromeOptions") 160 | options = Options() 161 | options.page_load_strategy = "eager" 162 | options.add_experimental_option("excludeSwitches", ["enable-automation"]) 163 | options.add_experimental_option("useAutomationExtension", False) 164 | options.add_argument("--no-sandbox") # Bypass OS security model 165 | # prefs = {"profile.managed_default_content_settings.images": 2} 166 | # options.add_experimental_option("prefs", prefs) 167 | options.add_argument("user-data-dir=.profile-EBAY") 168 | # options.add_argument('--proxy-server=https://'+ self.proxies[0]) 169 | # options.add_argument('--proxy-server=http://'+ self.proxies[0]) 170 | # options.add_argument('--proxy-server=socks5://'+ self.proxies[0]) 171 | options.add_argument("--disable-notifications") 172 | options.add_argument("--ignore-certificate-errors") 173 | options.add_argument("--ignore-ssl-errors") 174 | # options.add_argument('user-agent = Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36') 175 | # options.add_argument('--headless') 176 | # options.add_argument('--window-size=1910x1080') 177 | # options.add_argument('--proxy-server=http://'+ proxies[0])) 178 | options.add_argument("--disable-infobars") # disabling infobars 179 | options.add_argument("--disable-extensions") # disabling extensions 180 | options.add_argument("--disable-gpu") # applicable to windows os only 181 | options.add_argument("--disable-dev-shm-usage") # overcome limited resource problems 182 | options.add_argument("--remote-debugging-port=9222") 183 | 184 | product_list = pd.read_excel(dropship_sh_path, sheet_name="PRODUCT_LIST") 185 | items_to_list = product_list[ 186 | (product_list["UNLISTED"] == "T") & (product_list["SUPPLIER"] != "EBAY") 187 | ].reset_index(drop=True) 188 | 189 | logger.info("Open Browser") 190 | if len(items_to_list) == 0: 191 | logger.info("no products to list") 192 | exit() 193 | 194 | for i in range(0, len(items_to_list)): 195 | 196 | all_image_files_list = [] 197 | all_images = [] 198 | pricebought = "" 199 | priceship = "" 200 | 201 | # if os.path.exists(Path(dropship_path, items_to_list["Title"][i])) is False: 202 | 203 | # if items_to_list["SUPPLIER"][i].upper() == "ALIEXPRESS": 204 | # imagewebscraperali(items_to_list["Link"][i], items_to_list["Title"][i]) 205 | 206 | # elif items_to_list["SUPPLIER"][i].upper() == "EBAY": 207 | # imagewebscraperebay(items_to_list["Link"][i], items_to_list["Title"][i]) 208 | 209 | # elif items_to_list["SUPPLIER"][i].upper() == "ETSY": 210 | # imagewebscraperetsy(items_to_list["Link"][i], items_to_list["Title"][i]) 211 | 212 | # elif items_to_list["SUPPLIER"][i].upper() == "WAYFAIR": 213 | # imagewebscraperwayfair(items_to_list["Link"][i], items_to_list["Title"][i]) 214 | 215 | # elif items_to_list["SUPPLIER"][i].upper() == "CHEWY": 216 | # imagewebscraperchewy(items_to_list["Link"][i], items_to_list["Title"][i]) 217 | # else: 218 | # logger.info("No Supplier Matched") 219 | 220 | # else: 221 | 222 | # all_images = [ 223 | # f 224 | # for f in listdir(Path(dropship_path, items_to_list["Title"][i])) 225 | # if isfile(join(Path(dropship_path, items_to_list["Title"][i]), f)) 226 | # ] 227 | 228 | # all_image_files_list = [] 229 | # for idx, imagename in enumerate(all_images): 230 | # all_image_files_list.append( 231 | # Path(dropship_path, items_to_list["Title"][i], imagename) 232 | # ) 233 | 234 | with webdriver.Chrome( 235 | # executable_path=binary_path, 236 | options=options 237 | ) as driver: 238 | try: 239 | logger.info("Log In") 240 | time.sleep(random.randint(3, 5)) 241 | driver.get(EBAY_LOGIN) 242 | time.sleep(random.randint(3, 5)) 243 | email = driver.find_element_by_id("userid").send_keys(ebay_email) 244 | time.sleep(random.uniform(0.15, 0.4)) 245 | continue_btn = driver.find_element_by_id("signin-continue-btn").click() 246 | time.sleep(random.randint(3, 5)) 247 | password = driver.find_element_by_id("pass").send_keys(ebay_pass) 248 | time.sleep(random.uniform(0.15, 0.4)) 249 | login = driver.find_element_by_id("sgnBt").click() 250 | 251 | logger.info("Redirect to EBAY LISTING CREATE") 252 | time.sleep(random.randint(3, 5)) 253 | driver.get(EBAY_LISTING) 254 | time.sleep(random.randint(3, 5)) 255 | start_listing = ( 256 | WebDriverWait(driver, 10) 257 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_START_LISTING))) 258 | .click() 259 | ) 260 | time.sleep(random.uniform(0.15, 0.4)) 261 | start_listing = ( 262 | WebDriverWait(driver, 10) 263 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_START_LISTING))) 264 | .click() 265 | ) 266 | 267 | logger.info("enter images") 268 | time.sleep(random.randint(3, 5)) 269 | fileInput = driver.find_element_by_xpath(EBAY_IMAGE_UPLOAD_XPATH).click() 270 | time.sleep(random.randint(3, 5)) 271 | pyautogui.hotkey("alt", "d") 272 | time.sleep(random.uniform(0.15, 0.4)) 273 | pyautogui.write(Path(dropship_path, items_to_list["Title"][i])) 274 | time.sleep(random.uniform(0.15, 0.4)) 275 | pyautogui.press("enter") 276 | for x in range(4): 277 | time.sleep(random.uniform(0.15, 0.4)) 278 | pyautogui.press("tab") 279 | time.sleep(random.uniform(0.15, 0.4)) 280 | pyautogui.hotkey("ctrl", "a") 281 | time.sleep(random.uniform(0.15, 0.4)) 282 | pyautogui.press("enter") 283 | 284 | logger.info("enter title") 285 | time.sleep(random.randint(3, 5)) 286 | title = ( 287 | WebDriverWait(driver, 10) 288 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_TITLE_XPATH))) 289 | .send_keys(items_to_list["Title"][i]) 290 | ) 291 | logger.info("RENEW LISTING MANUAL") 292 | renew_manual = ( 293 | WebDriverWait(driver, 10) 294 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_RENEW_LISTING))) 295 | .click() 296 | ) 297 | logger.info("ABOUT THIS LISTING") 298 | time.sleep(random.randint(3, 5)) 299 | who_made = ( 300 | WebDriverWait(driver, 10) 301 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_WHO_MADE))) 302 | .click() 303 | ) 304 | time.sleep(random.uniform(0.15, 0.4)) 305 | someone_else_made = ( 306 | WebDriverWait(driver, 10) 307 | .until( 308 | EC.element_to_be_clickable((By.XPATH, EBAY_WHO_MADE_SOMEONE_ELSE)) 309 | ) 310 | .click() 311 | ) 312 | time.sleep(random.uniform(0.15, 0.4)) 313 | what_is_it = ( 314 | WebDriverWait(driver, 10) 315 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_WHAT_IS_IT))) 316 | .click() 317 | ) 318 | time.sleep(random.uniform(0.15, 0.4)) 319 | what_is_it_product = ( 320 | WebDriverWait(driver, 10) 321 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_WHAT_IS_IT_PRODUCT))) 322 | .click() 323 | ) 324 | time.sleep(random.uniform(0.15, 0.4)) 325 | when_made = ( 326 | WebDriverWait(driver, 10) 327 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_WHEN_MADE))) 328 | .click() 329 | ) 330 | time.sleep(random.uniform(0.15, 0.4)) 331 | when_made_to_order = ( 332 | WebDriverWait(driver, 10) 333 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_WHEN_MADE_TO_ORDER))) 334 | .click() 335 | ) 336 | time.sleep(random.uniform(0.15, 0.4)) 337 | logger.info("CATEGORY") 338 | ctgry = driver.find_element_by_xpath(EBAY_CTGRY_XPATH).send_keys( 339 | items_to_list["Title"][i] 340 | ) 341 | ctgry = driver.find_element_by_xpath(EBAY_CTGRY_XPATH).send_keys(Keys.ENTER) 342 | time.sleep(random.randint(3, 5)) 343 | logger.info("DESCRIPTION") 344 | description = driver.find_element_by_xpath(EBAY_DES_XPATH).send_keys( 345 | Keys.END 346 | ) 347 | time.sleep(random.uniform(0.15, 0.4)) 348 | description = driver.find_element_by_xpath(EBAY_DES1_XPATH).send_keys( 349 | Keys.CLEAR 350 | ) 351 | time.sleep(random.uniform(0.15, 0.4)) 352 | description = driver.find_element_by_xpath(EBAY_DES1_XPATH).send_keys( 353 | items_to_list["Description"][i] 354 | ) 355 | time.sleep(random.randint(3, 5)) 356 | time.sleep(random.randint(10, 15)) 357 | logger.info("PRICE") 358 | price = driver.find_element_by_xpath(EBAY_PRICE_XPATH).send_keys( 359 | Keys.BACKSPACE 360 | ) 361 | price = driver.find_element_by_xpath(EBAY_PRICE_XPATH).send_keys( 362 | int(items_to_list["Price"][i]) 363 | ) 364 | time.sleep(random.uniform(0.15, 0.4)) 365 | logger.info("QUANTITY") 366 | qty = driver.find_element_by_xpath(EBAY_QTY_XPATH).send_keys(Keys.BACKSPACE) 367 | qty = driver.find_element_by_xpath(EBAY_QTY_XPATH).send_keys( 368 | int(items_to_list["Quantity"][i]) 369 | ) 370 | logger.info("SHIPPING DEFAULT") 371 | time.sleep(random.uniform(0.15, 0.4)) 372 | ship_default = driver.find_element_by_xpath(EBAY_SHIP_XPATH).send_keys( 373 | Keys.END 374 | ) 375 | time.sleep(random.uniform(0.15, 0.4)) 376 | ship_default = driver.find_element_by_xpath(EBAY_SHIP_XPATH).click() 377 | time.sleep(random.uniform(0.15, 0.4)) 378 | # PUBLISH 379 | publish = ( 380 | WebDriverWait(driver, 10) 381 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_PUBLISH_XPATH))) 382 | .click() 383 | ) 384 | time.sleep(random.uniform(0.15, 0.4)) 385 | logger.info("CONFIRM") 386 | confirm = ( 387 | WebDriverWait(driver, 10) 388 | .until(EC.element_to_be_clickable((By.XPATH, EBAY_CONFIRM_XPATH))) 389 | .click() 390 | ) 391 | time.sleep(random.randint(3, 5)) 392 | driver.quit() 393 | shutil.rmtree(".profile-EBAY") 394 | except Exception as e: 395 | driver.quit() 396 | logger.info(e) 397 | shutil.rmtree(".profile-EBAY") 398 | -------------------------------------------------------------------------------- /etsy_desktop_sel_list.py: -------------------------------------------------------------------------------- 1 | """Selenium ETSY Marketplace Dropshipping Automator (SEMPDA). 2 | 3 | Allows user to leverage an excel sheet to automatically add products to ETSY 4 | 5 | Author: ehgp 6 | """ 7 | import datetime as dt 8 | import logging 9 | import logging.config 10 | import os 11 | import random 12 | import re 13 | import shutil 14 | import string 15 | import time 16 | from getpass import getuser 17 | from os import listdir 18 | from os.path import isfile, join 19 | from pathlib import Path 20 | 21 | import chromedriver_autoinstaller 22 | import keyring 23 | import pandas as pd 24 | import pyautogui 25 | import yaml 26 | from selenium import webdriver 27 | from selenium.webdriver.chrome.options import Options 28 | from selenium.webdriver.common.by import By 29 | from selenium.webdriver.common.keys import Keys 30 | from selenium.webdriver.support import expected_conditions as EC 31 | from selenium.webdriver.support.ui import WebDriverWait 32 | 33 | from webscraperali import imagewebscraperali 34 | from webscraperchewy import imagewebscraperchewy 35 | from webscraperebay import imagewebscraperebay 36 | from webscraperetsy import imagewebscraperetsy 37 | from webscraperwayfair import imagewebscraperwayfair 38 | 39 | 40 | def format_filename(s): 41 | """Take a string and return a valid filename constructed from the string. 42 | 43 | Uses a whitelist approach: any characters not present in valid_chars are 44 | removed. 45 | 46 | Note: this method may produce invalid filenames such as ``, `.` or `..` 47 | When I use this method I prepend a date string like '2009_01_15_19_46_32_' 48 | and append a file extension like '.txt', so I avoid the potential of using 49 | an invalid filename. 50 | 51 | """ 52 | valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) 53 | filename = "".join(re.sub("[^A-Za-z0-9]+", " ", c) for c in s if c in valid_chars) 54 | filename = " ".join(filename.split()) 55 | return filename 56 | 57 | 58 | def _load_config(): 59 | """Load the configuration yaml and return dictionary of setttings. 60 | 61 | Returns: 62 | yaml as a dictionary. 63 | """ 64 | config_path = os.path.dirname(os.path.realpath(__file__)) 65 | config_path = os.path.join(config_path, "xpath_params.yaml") 66 | with open(config_path, "r") as config_file: 67 | config_defs = yaml.safe_load(config_file.read()) 68 | 69 | if config_defs.values() is None: 70 | raise ValueError("parameters yaml file incomplete") 71 | 72 | return config_defs 73 | 74 | 75 | # from utils.encryption import create_encrypted_config, load_encrypted_config 76 | # os.makedirs(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET', exist_ok = True) 77 | # os.rmdir(str(path) + "\\Dropshipping Items\\"+ 'TESTESTSETESTSET') 78 | 79 | # Paths 80 | path = Path(os.getcwd()) 81 | # binary_path = Path(path, "chromedriver.exe") 82 | chromedriver_autoinstaller.install() 83 | dropship_sh_path = Path(path, "Dropshipping Items", "DROPSHIPPING_SPREADSHEET.xlsx") 84 | dropship_path = Path(path, "Dropshipping Items") 85 | 86 | # Logging 87 | Path("log").mkdir(parents=True, exist_ok=True) 88 | log_config = Path(path, "log_config.yaml") 89 | timestamp = "{:%Y_%m_%d_%H_%M_%S}".format(dt.datetime.now()) 90 | with open(log_config, "r") as log_file: 91 | config_dict = yaml.safe_load(log_file.read()) 92 | # Append date stamp to the file name 93 | log_filename = config_dict["handlers"]["file"]["filename"] 94 | base, extension = os.path.splitext(log_filename) 95 | base2 = "_" + os.path.splitext(os.path.basename(__file__))[0] + "_" 96 | log_filename = "{}{}{}{}".format(base, base2, timestamp, extension) 97 | config_dict["handlers"]["file"]["filename"] = log_filename 98 | logging.config.dictConfig(config_dict) 99 | logger = logging.getLogger(__name__) 100 | 101 | logger.info("Get Credentials") 102 | user = getuser() 103 | etsy_email = keyring.get_password("ETSY_EMAIL", user) 104 | etsy_pass = keyring.get_password("ETSY_PASSWORD", user) 105 | if (etsy_email or etsy_pass) is None: 106 | logger.info("Incomplete credentials") 107 | exit() 108 | 109 | cf = _load_config() 110 | 111 | DEFAULT_HEADERS = cf["GENERAL_PARAMS"]["DEFAULT_HEADERS"] 112 | 113 | ETSY_LOGIN = cf["ETSY_PARAMS"]["ETSY_LOGIN"] 114 | 115 | ETSY_USERNAME_XPATH = cf["ETSY_PARAMS"]["ETSY_USERNAME_XPATH"] 116 | 117 | ETSY_PASSWORD_XPATH = cf["ETSY_PARAMS"]["ETSY_PASSWORD_XPATH"] 118 | 119 | ETSY_LOGIN_BUTTON_XPATH = cf["ETSY_PARAMS"]["ETSY_LOGIN_BUTTON_XPATH"] 120 | 121 | ETSY_IMAGE_UPLOAD_XPATH = cf["ETSY_PARAMS"]["ETSY_IMAGE_UPLOAD_XPATH"] 122 | 123 | ETSY_LISTING = cf["ETSY_PARAMS"]["ETSY_LISTING"] 124 | 125 | ETSY_TITLE_XPATH = cf["ETSY_PARAMS"]["ETSY_TITLE_XPATH"] 126 | 127 | ETSY_WHO_MADE = cf["ETSY_PARAMS"]["ETSY_WHO_MADE"] 128 | 129 | ETSY_WHO_MADE_SOMEONE_ELSE = cf["ETSY_PARAMS"]["ETSY_WHO_MADE_SOMEONE_ELSE"] 130 | 131 | ETSY_WHAT_IS_IT = cf["ETSY_PARAMS"]["ETSY_WHAT_IS_IT"] 132 | 133 | ETSY_WHAT_IS_IT_PRODUCT = cf["ETSY_PARAMS"]["ETSY_WHAT_IS_IT_PRODUCT"] 134 | 135 | ETSY_WHEN_MADE = cf["ETSY_PARAMS"]["ETSY_WHEN_MADE"] 136 | 137 | ETSY_WHEN_MADE_TO_ORDER = cf["ETSY_PARAMS"]["ETSY_WHEN_MADE_TO_ORDER"] 138 | 139 | ETSY_CTGRY_XPATH = cf["ETSY_PARAMS"]["ETSY_CTGRY_XPATH"] 140 | 141 | ETSY_RENEW_LISTING = cf["ETSY_PARAMS"]["ETSY_RENEW_LISTING"] 142 | 143 | ETSY_DES_XPATH = cf["ETSY_PARAMS"]["ETSY_DES_XPATH"] 144 | 145 | ETSY_DES1_XPATH = cf["ETSY_PARAMS"]["ETSY_DES1_XPATH"] 146 | 147 | ETSY_PRICE_XPATH = cf["ETSY_PARAMS"]["ETSY_PRICE_XPATH"] 148 | 149 | ETSY_QTY_XPATH = cf["ETSY_PARAMS"]["ETSY_QTY_XPATH"] 150 | 151 | ETSY_SHIP_XPATH = cf["ETSY_PARAMS"]["ETSY_SHIP_XPATH"] 152 | 153 | ETSY_PUBLISH_XPATH = cf["ETSY_PARAMS"]["ETSY_PUBLISH_XPATH"] 154 | 155 | ETSY_CONFIRM_XPATH = cf["ETSY_PARAMS"]["ETSY_CONFIRM_XPATH"] 156 | 157 | logger.info("Configure ChromeOptions") 158 | options = Options() 159 | options.page_load_strategy = "eager" 160 | options.add_experimental_option("excludeSwitches", ["enable-automation"]) 161 | options.add_experimental_option("useAutomationExtension", False) 162 | # prefs = {"profile.managed_default_content_settings.images": 2} 163 | # options.add_experimental_option("prefs", prefs) 164 | options.add_argument("user-data-dir=.profile-ETSY") 165 | # options.add_argument('--proxy-server=https://'+ self.proxies[0]) 166 | # options.add_argument('--proxy-server=http://'+ self.proxies[0]) 167 | # options.add_argument('--proxy-server=socks5://'+ self.proxies[0]) 168 | options.add_argument("--disable-notifications") 169 | options.add_argument("--ignore-certificate-errors") 170 | options.add_argument("--ignore-ssl-errors") 171 | # options.add_argument('user-agent = Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36') 172 | # options.add_argument('--headless') 173 | # options.add_argument('--window-size=1910x1080') 174 | # options.add_argument('--proxy-server=http://'+ proxies[0])) 175 | options.add_argument("--disable-infobars") # disabling infobars 176 | options.add_argument("--disable-extensions") # disabling extensions 177 | options.add_argument("--disable-gpu") # applicable to windows os only 178 | options.add_argument("--no-sandbox") # Bypass OS security model 179 | options.add_argument("--disable-dev-shm-usage") # overcome limited resource problems 180 | options.add_argument("--remote-debugging-port=9222") 181 | 182 | product_list = pd.read_excel(dropship_sh_path, sheet_name="PRODUCT_LIST") 183 | items_to_list = product_list[ 184 | (product_list["UNLISTED"] == "T") & (product_list["SUPPLIER"] != "ETSY") 185 | ].reset_index(drop=True) 186 | 187 | logger.info("Open Browser") 188 | if len(items_to_list) == 0: 189 | logger.info("no products to list") 190 | exit() 191 | 192 | for i in range(0, len(items_to_list)): 193 | 194 | all_image_files_list = [] 195 | all_images = [] 196 | pricebought = "" 197 | priceship = "" 198 | 199 | if os.path.exists(Path(dropship_path, items_to_list["Title"][i])) is False: 200 | 201 | if items_to_list["SUPPLIER"][i].upper() == "ALIEXPRESS": 202 | imagewebscraperali(items_to_list["Link"][i], items_to_list["Title"][i]) 203 | 204 | elif items_to_list["SUPPLIER"][i].upper() == "EBAY": 205 | imagewebscraperebay(items_to_list["Link"][i], items_to_list["Title"][i]) 206 | 207 | elif items_to_list["SUPPLIER"][i].upper() == "ETSY": 208 | imagewebscraperetsy(items_to_list["Link"][i], items_to_list["Title"][i]) 209 | 210 | elif items_to_list["SUPPLIER"][i].upper() == "WAYFAIR": 211 | imagewebscraperwayfair(items_to_list["Link"][i], items_to_list["Title"][i]) 212 | 213 | elif items_to_list["SUPPLIER"][i].upper() == "CHEWY": 214 | imagewebscraperchewy(items_to_list["Link"][i], items_to_list["Title"][i]) 215 | else: 216 | logger.info("No Supplier Matched") 217 | 218 | else: 219 | 220 | all_images = [ 221 | f 222 | for f in listdir(Path(dropship_path, items_to_list["Title"][i])) 223 | if isfile(join(Path(dropship_path, items_to_list["Title"][i]), f)) 224 | ] 225 | 226 | all_image_files_list = [] 227 | for idx, imagename in enumerate(all_images): 228 | all_image_files_list.append( 229 | Path(dropship_path, items_to_list["Title"][i], imagename) 230 | ) 231 | 232 | with webdriver.Chrome( 233 | # executable_path=binary_path, 234 | options=options 235 | ) as driver: 236 | try: 237 | driver.get(ETSY_LOGIN) 238 | logger.info("Log In") 239 | time.sleep(random.uniform(0.15, 0.4)) 240 | username = ( 241 | WebDriverWait(driver, 10) 242 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_USERNAME_XPATH))) 243 | .send_keys(etsy_email) 244 | ) 245 | time.sleep(random.uniform(0.15, 0.4)) 246 | password = ( 247 | WebDriverWait(driver, 10) 248 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_PASSWORD_XPATH))) 249 | .send_keys(etsy_pass) 250 | ) 251 | # time.sleep(random.randint(30,40)) 252 | time.sleep(random.uniform(0.15, 0.4)) 253 | login = ( 254 | WebDriverWait(driver, 10) 255 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_LOGIN_BUTTON_XPATH))) 256 | .click() 257 | ) 258 | 259 | logger.info("Redirect to ETSY LISTING CREATE") 260 | time.sleep(random.randint(3, 5)) 261 | driver.get(ETSY_LISTING) 262 | 263 | logger.info("enter images") 264 | time.sleep(random.randint(3, 5)) 265 | fileInput = driver.find_element_by_xpath(ETSY_IMAGE_UPLOAD_XPATH).click() 266 | time.sleep(random.randint(3, 5)) 267 | pyautogui.hotkey("alt", "d") 268 | time.sleep(random.uniform(0.15, 0.4)) 269 | pyautogui.write(items_to_list["Pic_Directory"][i]) 270 | time.sleep(random.uniform(0.15, 0.4)) 271 | pyautogui.press("enter") 272 | for x in range(4): 273 | time.sleep(random.uniform(0.15, 0.4)) 274 | pyautogui.press("tab") 275 | time.sleep(random.uniform(0.15, 0.4)) 276 | pyautogui.hotkey("ctrl", "a") 277 | time.sleep(random.uniform(0.15, 0.4)) 278 | pyautogui.press("enter") 279 | 280 | logger.info("enter title") 281 | time.sleep(random.randint(3, 5)) 282 | title = ( 283 | WebDriverWait(driver, 10) 284 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_TITLE_XPATH))) 285 | .send_keys(items_to_list["Title"][i]) 286 | ) 287 | logger.info("RENEW LISTING MANUAL") 288 | renew_manual = ( 289 | WebDriverWait(driver, 10) 290 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_RENEW_LISTING))) 291 | .click() 292 | ) 293 | logger.info("ABOUT THIS LISTING") 294 | time.sleep(random.randint(3, 5)) 295 | who_made = ( 296 | WebDriverWait(driver, 10) 297 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_WHO_MADE))) 298 | .click() 299 | ) 300 | time.sleep(random.uniform(0.15, 0.4)) 301 | someone_else_made = ( 302 | WebDriverWait(driver, 10) 303 | .until( 304 | EC.element_to_be_clickable((By.XPATH, ETSY_WHO_MADE_SOMEONE_ELSE)) 305 | ) 306 | .click() 307 | ) 308 | time.sleep(random.uniform(0.15, 0.4)) 309 | what_is_it = ( 310 | WebDriverWait(driver, 10) 311 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_WHAT_IS_IT))) 312 | .click() 313 | ) 314 | time.sleep(random.uniform(0.15, 0.4)) 315 | what_is_it_product = ( 316 | WebDriverWait(driver, 10) 317 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_WHAT_IS_IT_PRODUCT))) 318 | .click() 319 | ) 320 | time.sleep(random.uniform(0.15, 0.4)) 321 | when_made = ( 322 | WebDriverWait(driver, 10) 323 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_WHEN_MADE))) 324 | .click() 325 | ) 326 | time.sleep(random.uniform(0.15, 0.4)) 327 | when_made_to_order = ( 328 | WebDriverWait(driver, 10) 329 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_WHEN_MADE_TO_ORDER))) 330 | .click() 331 | ) 332 | time.sleep(random.uniform(0.15, 0.4)) 333 | logger.info("CATEGORY") 334 | ctgry = driver.find_element_by_xpath(ETSY_CTGRY_XPATH).send_keys( 335 | items_to_list["Title"][i] 336 | ) 337 | ctgry = driver.find_element_by_xpath(ETSY_CTGRY_XPATH).send_keys(Keys.ENTER) 338 | time.sleep(random.randint(3, 5)) 339 | logger.info("DESCRIPTION") 340 | description = driver.find_element_by_xpath(ETSY_DES_XPATH).send_keys( 341 | Keys.END 342 | ) 343 | time.sleep(random.uniform(0.15, 0.4)) 344 | description = driver.find_element_by_xpath(ETSY_DES1_XPATH).send_keys( 345 | Keys.CLEAR 346 | ) 347 | time.sleep(random.uniform(0.15, 0.4)) 348 | description = driver.find_element_by_xpath(ETSY_DES1_XPATH).send_keys( 349 | ( 350 | "✅ New Product ✅ Price for each! Order " 351 | + items_to_list["Title"][i] 352 | + "! " 353 | + items_to_list["Description"][i] 354 | + """ Free Shipping Available 24/7 No Returns. \ 355 | USA Only I accept square, paypal, venmo, zelle, \ 356 | facebook pay, bitcoin/ethereum, cash app""" 357 | ) 358 | ) 359 | time.sleep(random.randint(3, 5)) 360 | time.sleep(random.randint(10, 15)) 361 | logger.info("PRICE") 362 | price = driver.find_element_by_xpath(ETSY_PRICE_XPATH).send_keys( 363 | Keys.BACKSPACE 364 | ) 365 | price = driver.find_element_by_xpath(ETSY_PRICE_XPATH).send_keys( 366 | int(items_to_list["Price"][i]) 367 | ) 368 | time.sleep(random.uniform(0.15, 0.4)) 369 | logger.info("QUANTITY") 370 | qty = driver.find_element_by_xpath(ETSY_QTY_XPATH).send_keys(Keys.BACKSPACE) 371 | qty = driver.find_element_by_xpath(ETSY_QTY_XPATH).send_keys( 372 | int(items_to_list["Quantity"][i]) 373 | ) 374 | logger.info("SHIPPING DEFAULT") 375 | time.sleep(random.uniform(0.15, 0.4)) 376 | ship_default = driver.find_element_by_xpath(ETSY_SHIP_XPATH).send_keys( 377 | Keys.END 378 | ) 379 | time.sleep(random.uniform(0.15, 0.4)) 380 | ship_default = driver.find_element_by_xpath(ETSY_SHIP_XPATH).click() 381 | time.sleep(random.uniform(0.15, 0.4)) 382 | logger.info("PUBLISH") 383 | publish = ( 384 | WebDriverWait(driver, 10) 385 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_PUBLISH_XPATH))) 386 | .click() 387 | ) 388 | time.sleep(random.uniform(0.15, 0.4)) 389 | logger.info("CONFIRM") 390 | confirm = ( 391 | WebDriverWait(driver, 10) 392 | .until(EC.element_to_be_clickable((By.XPATH, ETSY_CONFIRM_XPATH))) 393 | .click() 394 | ) 395 | time.sleep(random.randint(3, 5)) 396 | driver.quit() 397 | shutil.rmtree(".profile-ETSY") 398 | except Exception as e: 399 | driver.quit() 400 | logger.info(e) 401 | shutil.rmtree(".profile-ETSY") 402 | -------------------------------------------------------------------------------- /webscraperetsy.py: -------------------------------------------------------------------------------- 1 | """ETSY IMAGE SCRAPER. 2 | 3 | Scrapes images from your list on ETSY 4 | 5 | Author: ehgp 6 | """ 7 | import datetime as dt 8 | import json 9 | import logging 10 | import logging.config 11 | import os 12 | import pickle 13 | import random 14 | import re 15 | import string 16 | import sys 17 | import time 18 | from getpass import getuser 19 | from os import listdir 20 | from os.path import isfile, join 21 | from pathlib import Path 22 | 23 | import chromedriver_autoinstaller 24 | import keyring 25 | import pandas as pd 26 | import requests 27 | import yaml 28 | from bs4 import BeautifulSoup 29 | from selenium import webdriver 30 | from selenium.webdriver.chrome.options import Options 31 | from selenium.webdriver.common.by import By 32 | from selenium.webdriver.common.keys import Keys 33 | from selenium.webdriver.support import expected_conditions as EC 34 | from selenium.webdriver.support.ui import WebDriverWait 35 | 36 | 37 | def format_filename(s): 38 | """Take a string and return a valid filename constructed from the string. 39 | 40 | Uses a whitelist approach: any characters not present in valid_chars are 41 | removed. 42 | 43 | Note: this method may produce invalid filenames such as ``, `.` or `..` 44 | When I use this method I prepend a date string like '2009_01_15_19_46_32_' 45 | and append a file extension like '.txt', so I avoid the potential of using 46 | an invalid filename. 47 | 48 | """ 49 | valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) 50 | filename = "".join(re.sub("[^A-Za-z0-9]+", " ", c) for c in s if c in valid_chars) 51 | filename = " ".join(filename.split()) 52 | return filename 53 | 54 | 55 | def _load_config(): 56 | """Load the configuration yaml and return dictionary of setttings. 57 | 58 | Returns: 59 | yaml as a dictionary. 60 | """ 61 | config_path = os.path.dirname(os.path.realpath(__file__)) 62 | config_path = os.path.join(config_path, "xpath_params.yaml") 63 | with open(config_path, "r") as config_file: 64 | config_defs = yaml.safe_load(config_file.read()) 65 | 66 | if config_defs.values() is None: 67 | raise ValueError("parameters yaml file incomplete") 68 | 69 | return config_defs 70 | 71 | 72 | # Creds 73 | user = getuser() 74 | etsy_email = keyring.get_password("ETSY_EMAIL", user) 75 | etsy_pass = keyring.get_password("ETSY_PASSWORD", user) 76 | 77 | # Paths 78 | path = Path(os.getcwd()) 79 | # binary_path = Path(path, "chromedriver.exe") 80 | chromedriver_autoinstaller.install() 81 | dropship_sh_path = Path(path, "Dropshipping Items", "DROPSHIPPING_SPREADSHEET.xlsx") 82 | dropship_path = Path(path, "Dropshipping Items") 83 | 84 | # Logging 85 | Path("log").mkdir(parents=True, exist_ok=True) 86 | log_config = Path(path, "log_config.yaml") 87 | timestamp = "{:%Y_%m_%d_%H_%M_%S}".format(dt.datetime.now()) 88 | with open(log_config, "r") as log_file: 89 | config_dict = yaml.safe_load(log_file.read()) 90 | # Append date stamp to the file name 91 | log_filename = config_dict["handlers"]["file"]["filename"] 92 | base, extension = os.path.splitext(log_filename) 93 | base2 = "_" + os.path.splitext(os.path.basename(__file__))[0] + "_" 94 | log_filename = "{}{}{}{}".format(base, base2, timestamp, extension) 95 | config_dict["handlers"]["file"]["filename"] = log_filename 96 | logging.config.dictConfig(config_dict) 97 | logger = logging.getLogger(__name__) 98 | 99 | cf = _load_config() 100 | 101 | DEFAULT_HEADERS = cf["GENERAL_PARAMS"]["DEFAULT_HEADERS"] 102 | 103 | PAGES = cf["ETSY_WEBSCRAPER_PARAMS"]["PAGES"] 104 | 105 | ETSY_LOGIN = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_LOGIN"] 106 | 107 | ETSY_USERNAME_XPATH = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_USERNAME_XPATH"] 108 | 109 | ETSY_PASSWORD_XPATH = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_PASSWORD_XPATH"] 110 | 111 | ETSY_LOGIN_BUTTON_XPATH = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_LOGIN_BUTTON_XPATH"] 112 | 113 | ETSY_SOUP_SHIPPRICE_EXT = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_SOUP_SHIPPRICE_EXT"] 114 | 115 | ETSY_SOUP_SHIPHOW_EXT = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_SOUP_SHIPHOW_EXT"] 116 | 117 | ETSY_LINK_LIST = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_LINK_LIST"] 118 | 119 | ETSY_SOUP_IMAGELINK_EXT = cf["ETSY_WEBSCRAPER_PARAMS"]["ETSY_SOUP_IMAGELINK_EXT"] 120 | 121 | # Configure ChromeOptions 122 | options = Options() 123 | options.page_load_strategy = "eager" 124 | options.add_experimental_option("excludeSwitches", ["enable-automation"]) 125 | options.add_experimental_option("useAutomationExtension", False) 126 | # prefs = {"profile.managed_default_content_settings.images": 2} 127 | # options.add_experimental_option("prefs", prefs) 128 | options.add_argument("user-data-dir=.profile-ETSY") 129 | # options.add_argument('--proxy-server=https://'+ self.proxies[0]) 130 | # options.add_argument('--proxy-server=http://'+ self.proxies[0]) 131 | # options.add_argument('--proxy-server=socks5://'+ self.proxies[0]) 132 | options.add_argument("--disable-notifications") 133 | options.add_argument("--ignore-certificate-errors") 134 | options.add_argument("--ignore-ssl-errors") 135 | # options.add_argument('user-agent = Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36') 136 | # options.add_argument('--headless') 137 | # options.add_argument('--window-size=1910x1080') 138 | # options.add_argument('--proxy-server=http://'+ proxies[0])) 139 | options.add_argument("--disable-infobars") # disabling infobars 140 | options.add_argument("--disable-extensions") # disabling extensions 141 | options.add_argument("--disable-gpu") # applicable to windows os only 142 | options.add_argument("--disable-dev-shm-usage") # overcome limited resource problems 143 | options.add_argument("--remote-debugging-port=9222") 144 | 145 | 146 | def save_to_file_wishlist(response, i): 147 | """Save response to response.html.""" 148 | with open( 149 | Path(path, "etsy responses", f"responseetsywishlist{i}.html"), 150 | "w", 151 | encoding="utf-8", 152 | ) as fp: 153 | fp.write(response) 154 | 155 | 156 | def save_html_to_file(response, title): 157 | """Save response to response.html.""" 158 | with open( 159 | Path(path, "etsy responses", f"responseetsy{title}.html"), 160 | "w", 161 | encoding="utf-8", 162 | ) as fp: 163 | fp.write(response) 164 | 165 | 166 | def fix_cookies_and_load_to_requests(cookie_list, request_session): 167 | """Fix cookie values and add cookies to request_session.""" 168 | for index in range(len(cookie_list)): 169 | for item in cookie_list[index]: 170 | if type(cookie_list[index][item]) != str: 171 | cookie_list[index][item] = str(cookie_list[index][item]) 172 | cookies = requests.utils.cookiejar_from_dict(cookie_list[index]) 173 | request_session.cookies.update(cookies) 174 | return request_session 175 | 176 | 177 | # you function to get the cookies from the file. 178 | def load_cookies(filename): 179 | """Load cookies from file.""" 180 | with open(filename, "rb") as f: 181 | return pickle.load(f) 182 | 183 | 184 | new_descriptions = [] 185 | new_titles = [] 186 | extracted_links = [] 187 | new_prices = [] 188 | new_ship_prices = [] 189 | new_ship_how = [] 190 | 191 | 192 | def listingscraper(new_items): 193 | """Scrape listing title, description, price, ship price.""" 194 | for idx, link in enumerate(new_items): 195 | 196 | page = requests.get(link) 197 | 198 | save_html_to_file(page.text, idx) 199 | 200 | soup = BeautifulSoup(page.text, "lxml") 201 | json_content = soup.find_all(type="application/ld+json") 202 | content = json.loads(json_content[0].contents[0]) 203 | try: 204 | title = ( 205 | content["name"] 206 | .replace("Details about \xa0", "") 207 | .replace("from China", "") 208 | .replace("Worldwide", "") 209 | .replace("Etsy", "") 210 | .replace("etsy", "") 211 | .replace("ETSY", "") 212 | .replace("eBay", "") 213 | .replace("ebay", "") 214 | .replace("EBAY", "") 215 | .replace("AliExpress", "") 216 | .replace("aliexpress", "") 217 | .replace("ALIEXPRESS", "") 218 | .replace("LIFETIME WARRANTY", "") 219 | .replace("WARRANTY", "") 220 | .replace("lifetime warranty", "") 221 | .replace("|", "") 222 | .replace("\n", " ") 223 | .replace("\xa0", "") 224 | .replace(" Store Categories Store Categories ", "") 225 | .replace("US $", "") 226 | .replace("Return", "") 227 | .replace("return", "") 228 | .replace("Refund", "") 229 | .replace("refund", "") 230 | ) 231 | except Exception as e: 232 | print(e) 233 | title = "" 234 | pass 235 | 236 | new_titles.append(title) 237 | 238 | try: 239 | description = ( 240 | content["description"] 241 | .replace("Details about \xa0", "") 242 | .replace("from China", "") 243 | .replace("Worldwide", "") 244 | .replace("Etsy", "") 245 | .replace("etsy", "") 246 | .replace("ETSY", "") 247 | .replace("eBay", "") 248 | .replace("ebay", "") 249 | .replace("EBAY", "") 250 | .replace("AliExpress", "") 251 | .replace("aliexpress", "") 252 | .replace("ALIEXPRESS", "") 253 | .replace("LIFETIME WARRANTY", "") 254 | .replace("WARRANTY", "") 255 | .replace("lifetime warranty", "") 256 | .replace("|", "") 257 | .replace("\n", " ") 258 | .replace("\xa0", "") 259 | .replace(" Store Categories Store Categories ", "") 260 | .replace("US $", "") 261 | .replace("Return", "") 262 | .replace("return", "") 263 | .replace("Refund", "") 264 | .replace("refund", "") 265 | ) 266 | # product-description > div > div.detailmodule_html > div > div > div > div > div > div > div > div > div:nth-child(4) > div 267 | except Exception as e: 268 | print(e) 269 | description = "" 270 | pass 271 | 272 | new_descriptions.append(description) 273 | 274 | try: 275 | price = ( 276 | content["offers"]["highPrice"] 277 | .replace("Details about \xa0", "") 278 | .replace("from China", "") 279 | .replace("Worldwide", "") 280 | .replace("Etsy", "") 281 | .replace("etsy", "") 282 | .replace("ETSY", "") 283 | .replace("eBay", "") 284 | .replace("ebay", "") 285 | .replace("EBAY", "") 286 | .replace("AliExpress", "") 287 | .replace("aliexpress", "") 288 | .replace("ALIEXPRESS", "") 289 | .replace("LIFETIME WARRANTY", "") 290 | .replace("WARRANTY", "") 291 | .replace("lifetime warranty", "") 292 | .replace("|", "") 293 | .replace("\n", " ") 294 | .replace("\xa0", "") 295 | .replace(" Store Categories Store Categories ", "") 296 | .replace("US $", "") 297 | .replace("Return", "") 298 | .replace("return", "") 299 | .replace("Refund", "") 300 | .replace("refund", "") 301 | ) 302 | except Exception as e: 303 | print(e) 304 | price = "" 305 | pass 306 | 307 | new_prices.append(price) 308 | 309 | try: 310 | ship_price = ( 311 | str(soup.find(ETSY_SOUP_SHIPPRICE_EXT).text) 312 | .replace("Details about \xa0", "") 313 | .replace("from China", "") 314 | .replace("Worldwide", "") 315 | .replace("Etsy", "") 316 | .replace("etsy", "") 317 | .replace("ETSY", "") 318 | .replace("eBay", "") 319 | .replace("ebay", "") 320 | .replace("EBAY", "") 321 | .replace("AliExpress", "") 322 | .replace("aliexpress", "") 323 | .replace("ALIEXPRESS", "") 324 | .replace("LIFETIME WARRANTY", "") 325 | .replace("WARRANTY", "") 326 | .replace("lifetime warranty", "") 327 | .replace("|", "") 328 | .replace("\n", " ") 329 | .replace("\xa0", "") 330 | .replace(" Store Categories Store Categories ", "") 331 | .replace("US $", "") 332 | .replace("Return", "") 333 | .replace("return", "") 334 | .replace("Refund", "") 335 | .replace("refund", "") 336 | ) 337 | except Exception as e: 338 | print(e) 339 | ship_price = "" 340 | pass 341 | 342 | new_ship_prices.append(ship_price) 343 | 344 | try: 345 | ship_how = str(soup.find(ETSY_SOUP_SHIPHOW_EXT).text) 346 | except Exception as e: 347 | print(e) 348 | ship_how = "" 349 | pass 350 | 351 | new_ship_how.append(ship_how) 352 | 353 | new_items_df = pd.DataFrame( 354 | { 355 | "new_title": new_titles, 356 | "new_description": new_descriptions, 357 | "new_link": new_items, 358 | "new_price": new_prices, 359 | "new_ship_price": new_ship_prices, 360 | "new_ship_how": new_ship_how, 361 | } 362 | ) 363 | 364 | new_items_df.to_csv(Path(path, "Dropshipping Items", "new_items_etsy.csv")) 365 | 366 | 367 | # title = 'Skateboard Complete Standard Street Cruiser 32 INCH BY 8 INCH Board' 368 | # page = requests.get('https://www.etsy.com/listing/869334327/skateboard-complete-standard-street?') 369 | # soup = BeautifulSoup(page.text, 'lxml') 370 | # save_html_to_file(page,0) 371 | #