├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── LICENCE ├── README.md ├── chromedriver.py ├── demo.gif ├── devianart.py ├── requirements.txt ├── wallpaper.ps1 └── wallpaper.sh /.gitignore: -------------------------------------------------------------------------------- 1 | chromedriver 2 | -gallery.txt 3 | *.pyc 4 | images -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Download gallery", 9 | "type": "debugpy", 10 | "request": "launch", 11 | "program": "devianart.py", 12 | "console": "integratedTerminal" 13 | }, 14 | { 15 | "name": "Download 3 images from gallery", 16 | "type": "debugpy", 17 | "request": "launch", 18 | "program": "devianart.py", 19 | "console": "integratedTerminal", 20 | "args": [ 21 | "--count", 22 | "3" 23 | ] 24 | }, 25 | { 26 | "name": "Download gallery custom url", 27 | "type": "debugpy", 28 | "request": "launch", 29 | "program": "devianart.py", 30 | "console": "integratedTerminal", 31 | "args": [ 32 | "--url", 33 | "https://www.deviantart.com/topic/photo-manipulation" 34 | ] 35 | }, 36 | { 37 | "name": "Download wallpaper", 38 | "type": "debugpy", 39 | "request": "launch", 40 | "program": "devianart.py", 41 | "console": "integratedTerminal", 42 | "args": [ 43 | "-f", 44 | "wallpaper", 45 | "-c", 46 | "1", 47 | "-r", 48 | "-u", 49 | "https://www.deviantart.com/topic/stock-images" 50 | ] 51 | }, 52 | { 53 | "name": "Download 1 random", 54 | "type": "debugpy", 55 | "request": "launch", 56 | "program": "devianart.py", 57 | "console": "integratedTerminal", 58 | "args": [ 59 | "--count", 60 | "1", 61 | "--random" 62 | ] 63 | } 64 | ] 65 | } 66 | 67 | -c 1 -r -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | // Workspace settings 2 | { 3 | // The following will hide in the editor 4 | "files.exclude": { 5 | "**/__pycache__": true, 6 | "**/*/*.zip": true, 7 | "chromedriver": true 8 | }, 9 | } -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Malvin Todorov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeviantArt Scraper 2 | 3 | The easy way to download the highest resolution DeviantArt Galleries. For more information check this [article](https://mlvnt.com/blog/tech/2018/05/scraping-deviantart/). 4 | 5 | ![Demo](demo.gif) 6 | 7 | ## Requirements 8 | 9 | **1. Install python dependencies:** 10 | 11 | ``` 12 | pip install -r requirements.txt 13 | sudo pip3 install selenium 14 | ``` 15 | 16 | **2. Install Chromium:** 17 | 18 | - Debian Based 19 | 20 | ``` 21 | sudo apt install chromium-browser 22 | ``` 23 | 24 | - Arch 25 | 26 | ``` 27 | sudo pacman -S chromium 28 | ``` 29 | 30 | **3. Download the Chrome Driver.** 31 | 32 | - Download [Chrome Driver](https://chromedriver.chromium.org/downloads). *Note, the version should match your version of Chrome installed.* 33 | - Copy `chromedriver` into the folder `deviantart-scraper`. 34 | 35 | 36 | ## Quick Start 37 | 38 | To begin downloading images, use the following steps. Images will be downloaded into a default folder `images`. Additionally, a metadata file `gallery.txt` will be generated, containing the names of the files downloaded. 39 | 40 | ```bash 41 | python3 devianart.py 42 | ``` 43 | 44 | ## Additional Options 45 | 46 | The following command-line arguments are supported. 47 | 48 | ```bash 49 | python3 devianart.py --help 50 | ``` 51 | 52 | ```text 53 | usage: devianart.py [-h] [-d DIR] [-f FILENAME] [-u URL] [-c COUNT] [-r] 54 | 55 | optional arguments: 56 | -h, --help show this help message and exit 57 | -d DIR, --dir DIR Directory to store images. Default: ./images 58 | -f FILENAME, --filename FILENAME 59 | Explicit base filename to use. Default: downloaded 60 | filename 61 | -u URL, --url URL DeviantArt gallery url to scrape images from. Default: 62 | deviantart.com 63 | -c COUNT, --count COUNT 64 | Maximum number of images to download. Default: 25 65 | -r, --random Download a random image. Default: False 66 | ``` 67 | 68 | ## Automatically Changing the Desktop Background Wallpaper 69 | 70 | You can automatically download and set the desktop background wallpaper by using the command-line arguments with a script or desktop background image changer utility. The following example demonstrates this for Windows [wallpaper.ps1](wallpaper.ps1) and Linux Mint using [wallpaper.sh](wallpaper.sh). 71 | 72 | ### Windows 73 | 74 | The following Windows PowerShell script will automatically download and set the desktop wallpaper. To run this automatically every day: 75 | 76 | 1. Open Task Scheduler by pressing **Win + R**, type **taskschd.msc**, and press Enter. 77 | 2. In the Task Scheduler window, click on **Create Task** in the right-hand pane. 78 | 3. For **Triggers**, set to run **Daily**. 79 | 4. For **Actions**, in the Program/script field, enter **powershell.exe**. 80 | 5. In the **Add arguments** (optional) field, enter **-File "C:\Path\To\Your\Script\wallpaper.ps1"** 81 | 6. Click **OK** to save the task. 82 | 83 | ```bash 84 | # Get the current logged-in user's profile path 85 | $userProfile = [System.Environment]::GetFolderPath('UserProfile') 86 | 87 | # Replace "\Documents\deviantart-scraper" with the path to deviantart-scraper. 88 | $deviantArtScraperFolder = "Documents\deviantart-scraper" 89 | 90 | # Define the folder containing the wallpapers. 91 | $wallpaperFolder = "$userProfile\$deviantArtScraperFolder\images" 92 | 93 | # Delete existing wallpaper files 94 | Remove-Item -Path "$wallpaperFolder\wallpaper.jpg" -ErrorAction SilentlyContinue 95 | Remove-Item -Path "$wallpaperFolder\wallpaper.png" -ErrorAction SilentlyContinue 96 | Remove-Item -Path "$wallpaperFolder\wallpaper.jpeg" -ErrorAction SilentlyContinue 97 | Remove-Item -Path "$wallpaperFolder\wallpaper.bmp" -ErrorAction SilentlyContinue 98 | 99 | $url = "https://www.deviantart.com/topic/random" 100 | 101 | # Download image 102 | python "$userProfile\$deviantArtScraperFolder\devianart.py" -f wallpaper -c 1 -r -u $url 103 | 104 | # Get all image files in the folder 105 | $wallpapers = Get-ChildItem -Path $wallpaperFolder -Recurse | Where-Object { $_.Extension -match "jpg|jpeg|png|bmp" } 106 | 107 | # Check if any wallpapers were found 108 | if ($null -eq $wallpapers -or $wallpapers.Count -eq 0) { 109 | Write-Host "No wallpapers found in the specified folder: $wallpaperFolder" 110 | exit 111 | } 112 | 113 | # Select a random wallpaper 114 | $randomWallpaper = Get-Random -InputObject $wallpapers 115 | 116 | # Output the selected file for debugging 117 | Write-Host "Selected file: "$randomWallpaper.FullName 118 | 119 | # Set the wallpaper using SystemParametersInfo 120 | $code = @" 121 | using System; 122 | using System.Runtime.InteropServices; 123 | 124 | public class Wallpaper { 125 | [DllImport("user32.dll", CharSet = CharSet.Auto)] 126 | public static extern int SystemParametersInfo(int uAction, int uParam, string lpvParam, int fuWinIni); 127 | 128 | public static void SetWallpaper(string path) { 129 | SystemParametersInfo(0x0014, 0, path, 0x01 | 0x02); 130 | } 131 | } 132 | "@ 133 | 134 | Add-Type -TypeDefinition $code 135 | [Wallpaper]::SetWallpaper($randomWallpaper.FullName) 136 | ``` 137 | 138 | ### Linux 139 | 140 | The following Linux bash script will automatically download and set the desktop wallpaper. 141 | 142 | ```bash 143 | #!/bin/bash 144 | 145 | USER=$(whoami) 146 | ORIGINAL_DIR=$(pwd) 147 | 148 | # Fix to allow cronjob to accurately set the desktop background. https://askubuntu.com/a/198508 149 | fl=$(find /proc -maxdepth 2 -user $USER -name environ -print -quit) 150 | while [ -z $(grep -z DBUS_SESSION_BUS_ADDRESS "$fl" | cut -d= -f2- | tr -d '\000' ) ] 151 | do 152 | fl=$(find /proc -maxdepth 2 -user $USER -name environ -newer "$fl" -print -quit) 153 | done 154 | export DBUS_SESSION_BUS_ADDRESS=$(grep -z DBUS_SESSION_BUS_ADDRESS "$fl" | cut -d= -f2-) 155 | echo $DBUS_SESSION_BUS_ADDRESS > /var/tmp/wallpaper.log 156 | 157 | # Delete cached wallpaper. 158 | rm -f /var/tmp/wallpaper.jpg /var/tmp/wallpaper.jpeg /var/tmp/wallpaper.gif /var/tmp/wallpaper.png 159 | 160 | # Download image. 161 | cd /home/$USER/Documents/deviantart-scraper/ 162 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r >> /var/tmp/wallpaper.log 163 | FILE_PATH=$(tail -n 1 /var/tmp/wallpaper.log) 164 | cd $ORIGINAL_DIR 165 | 166 | # Delete cached wallpaper. 167 | rm -f /home/$USER/.cache/wallpaper/* 168 | 169 | echo "Downloaded $FILE_PATH" >> /var/tmp/wallpaper.log 170 | 171 | # Set new wallpaper. 172 | gsettings set org.gnome.desktop.background picture-options "zoom" 173 | gsettings set org.gnome.desktop.background picture-uri file://$FILE_PATH 174 | ``` 175 | 176 | You can automatically run the above bash script via cron job with the following command. 177 | 178 | ```bash 179 | chmod +x wallpaper.sh 180 | crontab -e 181 | ``` 182 | 183 | Paste the following lines to the end of the cron file. 184 | 185 | ``` 186 | # Add a cron job to run this script every 15 minutes. 187 | */15 * * * * /home/YOUR_USER_NAME/Documents/deviantart-scraper/wallpaper.sh 188 | @reboot /home/YOUR_USER_NAME/Documents/deviantart-scraper/wallpaper.sh 189 | ``` 190 | 191 | ## License 192 | 193 | The code is licensed under the MIT License. 194 | 195 | > Disclaimer: All art you download using this script belongs to their rightful owners. Please support them by purchasing their art. 196 | -------------------------------------------------------------------------------- /chromedriver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from selenium import webdriver 3 | from selenium.webdriver.common.keys import Keys 4 | from selenium.common.exceptions import WebDriverException, SessionNotCreatedException 5 | from selenium.webdriver.chrome.service import Service 6 | import sys 7 | import os 8 | import shutil 9 | import pathlib 10 | import urllib.request 11 | import re 12 | import zipfile 13 | import stat 14 | from sys import platform 15 | 16 | def get_driver(): 17 | # Attempt to open the Selenium chromedriver. If it fails, download the latest chromedriver. 18 | driver = None 19 | retry = True 20 | major_version = None 21 | 22 | # Determine the version of Chrome installed. 23 | version = get_chrome_version() 24 | if version: 25 | parts = version.split('.') 26 | major_version = parts[0] if len(parts) > 0 else 0 27 | 28 | while retry: 29 | retry = False 30 | is_download = False 31 | 32 | try: 33 | options = webdriver.ChromeOptions() 34 | options.add_argument('--headless') 35 | executable_path="./chromedriver" + ('.exe' if 'win' in platform else '') 36 | service = Service(executable_path=executable_path) 37 | driver = webdriver.Chrome(options=options, service=service) 38 | except SessionNotCreatedException as e: 39 | if 'This version of ChromeDriver' in e.msg: 40 | is_download = True 41 | print('Warning: You may need to update the Chrome web browser to the latest version. Run Chrome, click Help->About.') 42 | except WebDriverException as e: 43 | if "wrong permissions" in e.msg: 44 | st = os.stat('./chromedriver') 45 | os.chmod('./chromedriver', st.st_mode | stat.S_IEXEC) 46 | retry = True 47 | elif "chromedriver' executable needs to be in PATH" in e.msg: 48 | is_download = True 49 | elif "error" in e.msg: 50 | print(e.msg) 51 | is_download = True 52 | 53 | retry = is_download and download_driver(major_version) 54 | 55 | return driver 56 | 57 | def download_driver(version=None): 58 | # Find the latest chromedriver, download, unzip, set permissions to executable. 59 | result = False 60 | url = 'https://googlechromelabs.github.io/chrome-for-testing' 61 | base_driver_url = 'https://storage.googleapis.com/chrome-for-testing-public' 62 | file_name = 'chromedriver-' + get_platform_filename() 63 | driver_file_name = 'chromedriver' + '.exe' if platform == "win32" else '' 64 | pattern = 'https://storage.googleapis.com/chrome-for-testing-public/(' + (version or '\d+') + '\.\d+\.\d+\.\d+)' 65 | 66 | # Download latest chromedriver. 67 | print('Finding latest chromedriver..') 68 | opener = urllib.request.FancyURLopener({}) 69 | stream = opener.open(url) 70 | content = stream.read().decode('utf8') 71 | 72 | # Parse the latest version. 73 | match = re.search(pattern, content) 74 | if match and match.groups(): 75 | # Url of download html page. 76 | url = match.group(0) 77 | # Version of latest driver. 78 | version = match.group(1) 79 | driver_url = f"{base_driver_url}/{version}/{get_platform_filename(False)}/{file_name}" 80 | 81 | # Download the file. 82 | print('Version ' + version) 83 | print('Downloading ' + driver_url) 84 | app_path = os.path.dirname(os.path.realpath(__file__)) 85 | chromedriver_path = app_path + '/' + driver_file_name 86 | file_path = app_path + '/' + file_name 87 | urllib.request.urlretrieve(driver_url, file_path) 88 | 89 | # Unzip the file. 90 | print('Unzipping ' + file_path) 91 | with zipfile.ZipFile(file_path, 'r') as zip_ref: 92 | zip_ref.extractall(app_path) 93 | 94 | # Copy exe to parent path. 95 | source_file_path = f"chromedriver-{get_platform_filename(False)}/chromedriver.exe" 96 | 97 | shutil.copyfile(source_file_path, chromedriver_path) 98 | print('Setting executable permission on ' + chromedriver_path) 99 | st = os.stat(chromedriver_path) 100 | os.chmod(chromedriver_path, st.st_mode | stat.S_IEXEC) 101 | 102 | # Cleanup. 103 | os.remove(file_path) 104 | 105 | result = True 106 | 107 | return result 108 | 109 | def get_platform_filename(isExtension=True): 110 | filename = '' 111 | 112 | is_64bits = sys.maxsize > 2**32 113 | 114 | if platform == "linux" or platform == "linux2": 115 | # linux 116 | filename += 'linux64' 117 | elif platform == "darwin": 118 | # OS X 119 | filename += 'mac-x64' 120 | elif platform == "win32": 121 | # Windows 122 | filename += 'win64' if is_64bits else 'win32' 123 | 124 | filename += '.zip' if isExtension else '' 125 | 126 | return filename 127 | 128 | def extract_version(output): 129 | try: 130 | google_version = '' 131 | for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]: 132 | if letter != '\n': 133 | google_version += letter 134 | else: 135 | break 136 | return(google_version.strip()) 137 | except TypeError: 138 | return 139 | 140 | def get_chrome_version(): 141 | version = None 142 | install_path = None 143 | 144 | try: 145 | if platform == "linux" or platform == "linux2": 146 | # linux 147 | install_path = "/usr/bin/google-chrome" 148 | elif platform == "darwin": 149 | # OS X 150 | install_path = "/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome" 151 | elif platform == "win32": 152 | # Windows... 153 | stream = os.popen('reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"') 154 | output = stream.read() 155 | version = extract_version(output) 156 | except Exception as ex: 157 | print(ex) 158 | 159 | version = os.popen(f"{install_path} --version").read().strip('Google Chrome ').strip() if install_path else version 160 | 161 | return version -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/primaryobjects/deviantart-scraper/27e6ee597e1e5ba4d44bd770cd7cc94731bdfd6e/demo.gif -------------------------------------------------------------------------------- /devianart.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from bs4 import BeautifulSoup 3 | from queue import Queue 4 | from threading import Thread, Lock 5 | import collections 6 | import datetime 7 | import time 8 | import os 9 | import pathlib 10 | import requests 11 | import subprocess 12 | import imghdr 13 | import argparse 14 | from random import randint 15 | from chromedriver import get_driver 16 | from selenium.common.exceptions import NoSuchElementException 17 | from selenium.webdriver.common.by import By 18 | 19 | #======================== INITIALIZE VARIABLES ================================= 20 | 21 | images = [] 22 | workers = 1 #20 23 | threads = [] 24 | tasks = Queue() 25 | lock = Lock() 26 | img_num = 0 27 | max_image_count = 0 # Set to 0 for all images. 28 | no_image_found_count = 0 29 | folder = '' 30 | file_name = '' 31 | first_image = '' 32 | is_random = '' 33 | url = '' 34 | 35 | #======================== WELCOME MESSAGE ====================================== 36 | 37 | def welcome_message(): 38 | now = datetime.datetime.now() 39 | today = now.strftime("%A • %B %e • %H:%M • %Y") 40 | print('\n DeviantArt Scraper') 41 | print('\n DATE: ' + today) 42 | 43 | #======================== GET USERNAME ========================================= 44 | 45 | def get_username(d): 46 | global username 47 | #html = d.page_source 48 | #soup = BeautifulSoup(html, 'html.parser') 49 | username = "" #soup.find(class_='gruserbadge').find('a').get_text() 50 | 51 | #======================== GET LINKS FROM TUMBNAILS ============================= 52 | 53 | def get_thumb_links(q): 54 | d = get_driver() 55 | # REPLACE username with your preferred artist 56 | print('Downloading ' + url) 57 | d.get(url) 58 | unique_img = scroll_page_down(d) 59 | time.sleep(0.5) 60 | for img in unique_img: 61 | q.put(img) 62 | global expected_img_num 63 | expected_img_num = str(len(unique_img)) 64 | get_username(d) 65 | print(' Unique images found = ' + expected_img_num) 66 | print(' Artist = ' + username + "\n") 67 | time.sleep(0.5) 68 | d.close() 69 | 70 | #======================== SCROLL DOWN ========================================== 71 | 72 | def scroll_page_down(d): 73 | SCROLL_PAUSE_TIME = 1.5 74 | 75 | global is_random 76 | r = -1 77 | if is_random: 78 | r = randint(0, 250) 79 | 80 | page = 1 81 | next = None 82 | last_height = d.execute_script("return document.body.scrollHeight") 83 | 84 | while ((r == -1 and len(images) < max_image_count) or (r > -1 and len(images) < r + 1)) or max_image_count == 0: 85 | if next: 86 | # Navigate to the next page of results. 87 | url = next[0].get_attribute("href") 88 | page += 1 89 | print('Moving to page ' + str(page) + ' at ' + url) 90 | d.get(url) 91 | 92 | # Get next page. 93 | try: 94 | # If a "Next" link exists, get the url for it. 95 | next = d.find_elements(By.XPATH, "//a[contains(text(), 'Next') and contains(@href,'?cursor=')]") 96 | except NoSuchElementException: 97 | print("Skipping next button and using auto-scrolling.") 98 | 99 | if not next: 100 | # If no "Next" link exists, scroll down to bottom of page. 101 | print('Auto-scrolling down page.') 102 | d.execute_script("window.scrollTo(0, document.body.scrollHeight);") 103 | time.sleep(SCROLL_PAUSE_TIME) # Wait to load page 104 | # Calculate new scroll height and compare with last scroll height 105 | new_height = d.execute_script("return document.body.scrollHeight") 106 | 107 | # Find all content links on the page. 108 | links = d.find_elements(By.XPATH, "//div[@data-testid='thumb']/..") 109 | 110 | # Queue unique links. 111 | for link in links: 112 | l = link.get_attribute('href') 113 | if l: 114 | if not l in images and (max_image_count == 0 or ((r == -1 and len(images) < max_image_count) or (r > -1 and len(images) < r + 1))): 115 | print('Queuing ' + l) 116 | images.append(l) 117 | else: 118 | print('Skipping duplicate ' + l) 119 | 120 | # Remove duplicate links. 121 | unique_img = list(set(images)) 122 | time.sleep(0.5) 123 | 124 | if not next: 125 | # Break when the end is reached while scrolling down. 126 | if new_height == last_height: 127 | break 128 | last_height = new_height 129 | 130 | if r > 0: 131 | # Select new index for random image. 132 | r = randint(0, len(images)) 133 | print("Selecting image " + str(r) + " of " + str(len(images))) 134 | selected_image = images[r] 135 | images.clear() 136 | images.append(selected_image) 137 | unique_img = list(set(images)) # Remove duplicates 138 | 139 | return unique_img 140 | 141 | #======================== GET FULL RESOLUTION IMAGES =========================== 142 | 143 | def get_full_image(l): 144 | s = requests.Session() 145 | h = {'User-Agent': 'Firefox'} 146 | soup = BeautifulSoup(s.get(l, headers=h).text, 'html.parser') 147 | title = '' 148 | link = '' 149 | it = None 150 | 151 | try: 152 | img = soup.find('img', {"property": "contentUrl"}) 153 | if img: 154 | link = img['src'] 155 | h1 = soup.find('h1') 156 | if h1: 157 | title = h1.text.replace(' ', '_').lower() 158 | except TypeError as e: 159 | print('Error downloading ' + l) 160 | print(e) 161 | 162 | if link: 163 | req = s.get(link, headers=h) 164 | time.sleep(0.1) 165 | download_now(req,title) 166 | url = req.url 167 | ITuple = collections.namedtuple('ITuple', ['u', 't']) 168 | it = ITuple(u=url, t=title) 169 | 170 | return it 171 | 172 | #======================== GET AGE-RESTRICTED IMAGES ============================ 173 | 174 | def age_restricted(l): 175 | d = get_driver() 176 | d.get(l) 177 | time.sleep(0.8) 178 | d.find_elements(By.CLASS_NAME, 'datefields') 179 | d.find_elements(By.CLASS_NAME, 'datefield') 180 | d.find_elements(By.ID, 'month').send_keys('01') 181 | d.find_elements(By.ID, 'day').send_keys('01') 182 | d.find_elements(By.ID, 'year').send_keys('1991') 183 | d.find_elements(By.CLASS_NAME, 'tos-label').click() 184 | d.find_elements(By.CLASS_NAME, 'submitbutton').click() 185 | time.sleep(1) 186 | img_lnk = d.find_elements(By.CLASS_NAME, 'dev-page-download') 187 | d.get(img_lnk.get_attribute('href')) 188 | time.sleep(0.5) 189 | link = d.current_url 190 | d.close() 191 | return link 192 | 193 | #======================== FILENAME FORMATTING ================================== 194 | 195 | def name_format(url,title): 196 | timestr = time.strftime("%Y-%m-%d-%H-%M-%S") 197 | name = title + '_' + timestr 198 | if title != '': 199 | name = title + '.jpg' 200 | return name 201 | 202 | #======================== DOWNLOAD USING REQUESTS ============================== 203 | 204 | def download_now(req,title): 205 | global file_name 206 | 207 | url = req.url 208 | name = file_name or name_format(url,title) 209 | name = name.replace('/', '_') 210 | 211 | global folder 212 | pathlib.Path('{}'.format(folder)).mkdir(parents=True, exist_ok=True) 213 | file_path = os.path.join('{}'.format(folder + name)) 214 | 215 | with open(file_path,'wb') as file: 216 | file.write(req.content) 217 | 218 | # Set image extension by detecting the type of image (jpg, gif, png). 219 | ext = imghdr.what(file_path) or 'jpg' 220 | if ext == 'jpeg': 221 | ext = 'jpg' 222 | base = os.path.splitext(file_path)[0] 223 | os.rename(file_path, base + '.' + ext) 224 | 225 | global first_image 226 | if first_image == '': 227 | first_image = base + '.' + ext 228 | #======================== SAVE IMAGE LINKS TO A FILE =========================== 229 | 230 | def save_img(url): 231 | try: 232 | with open('{}-gallery.txt'.format(username), 'a+') as file: 233 | file.write(url + '\n') 234 | except: 235 | print('An write error occurred.') 236 | pass 237 | 238 | #======================== WORKER THREAD ======================================== 239 | 240 | def worker_thread(q, lock): 241 | while True: 242 | link = q.get() 243 | if link is None: 244 | break 245 | p = get_full_image(link) 246 | if p: 247 | url = p.u 248 | title = p.t 249 | name = name_format(url, title) 250 | with lock: 251 | save_img(url) 252 | global img_num 253 | img_num = img_num + 1 254 | print('Image ' + str(img_num) + ' ' + name) 255 | else: 256 | global no_image_found_count 257 | no_image_found_count += 1 258 | print('No image found for ' + link) 259 | q.task_done() 260 | 261 | #======================== MAIN FUNCTION ======================================== 262 | 263 | def main(): 264 | global folder 265 | global file_name 266 | global max_image_count 267 | global is_random 268 | global url 269 | 270 | ap = argparse.ArgumentParser() 271 | 272 | # Add the arguments to the parser 273 | ap.add_argument("-d", "--dir", required=False, help="Directory to store images. Default: ./images", default="images") 274 | ap.add_argument("-f", "--filename", required=False, help="Explicit base filename to use. Default: downloaded filename", default="") 275 | ap.add_argument("-u", "--url", required=False, help="DeviantArt gallery url to scrape images from. Default: deviantart.com", default="https://www.deviantart.com") 276 | ap.add_argument("-c", "--count", required=False, help="Maximum number of images to download. Default: 25", type=int, default=25) 277 | ap.add_argument("-r", "--random", required=False, help="Download a random image. Default: False", action="store_true") 278 | 279 | # Parse command-line arguments. 280 | args = vars(ap.parse_args()) 281 | 282 | folder = os.path.join(args['dir'].lstrip(), '') 283 | file_name = args['filename'].lstrip() 284 | url = args['url'].lstrip() 285 | max_image_count = args['count'] 286 | is_random = args['random'] 287 | 288 | welcome_message() # Display Welcome Message 289 | start = time.time() 290 | get_thumb_links(tasks) # Fill the Queue 291 | 292 | # Start the Threads 293 | for i in range(workers): 294 | t = Thread(target = worker_thread, args = (tasks, lock)) 295 | t.start() 296 | threads.append(t) 297 | 298 | # When done close worker threads 299 | tasks.join() 300 | for _ in range(workers): 301 | tasks.put(None) 302 | for t in threads: 303 | t.join() 304 | 305 | # Print Stats 306 | if not '/tmp' in folder: 307 | try: 308 | folder_size = subprocess.check_output(['du','-shx', folder]).split()[0].decode('utf-8') 309 | print('\n Total Images: ' + str(img_num) + ' (' + str(folder_size) + ')') 310 | print(' Expected: ' + expected_img_num - no_image_found_count) 311 | end = time.time() 312 | print(' Elapsed Time: {:.4f}\n'.format(end-start)) 313 | except: 314 | pass 315 | 316 | if max_image_count == 1: 317 | global first_image 318 | print(first_image) 319 | 320 | if __name__ == '__main__': 321 | try: 322 | main() 323 | except KeyboardInterrupt: 324 | print() 325 | #=============================================================================== -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | ###### Required Packages ###### 2 | selenium 3 | bs4 4 | requests -------------------------------------------------------------------------------- /wallpaper.ps1: -------------------------------------------------------------------------------- 1 | # Get the current logged-in user's profile path 2 | $userProfile = [System.Environment]::GetFolderPath('UserProfile') 3 | 4 | ############# 5 | # IMPORTANT 6 | # Replace "\Documents\deviantart-scraper" with the path to deviantart-scraper. 7 | ############# 8 | $deviantArtScraperFolder = "Documents\deviantart-scraper" 9 | ############# 10 | 11 | # Define the folder containing the wallpapers. 12 | $wallpaperFolder = "$userProfile\$deviantArtScraperFolder\images" 13 | 14 | # Delete existing wallpaper files 15 | Remove-Item -Path "$wallpaperFolder\wallpaper.jpg" -ErrorAction SilentlyContinue 16 | Remove-Item -Path "$wallpaperFolder\wallpaper.png" -ErrorAction SilentlyContinue 17 | Remove-Item -Path "$wallpaperFolder\wallpaper.jpeg" -ErrorAction SilentlyContinue 18 | Remove-Item -Path "$wallpaperFolder\wallpaper.bmp" -ErrorAction SilentlyContinue 19 | 20 | # Get the current month 21 | $month = (Get-Date).Month 22 | 23 | # Define the URL based on the current month 24 | $url = switch ($month) { 25 | 9 { "https://www.deviantart.com/topic/creepy" } 26 | 10 { "https://www.deviantart.com/topic/horror" } 27 | 11 { "https://www.deviantart.com/topic/surreal" } 28 | 12 { "https://www.deviantart.com/topic/christmas" } 29 | 1 { "https://www.deviantart.com/topic/winter" } 30 | 2 { "https://www.deviantart.com/topic/cyberpunk" } 31 | 3 { "https://www.deviantart.com/topic/fantasy" } 32 | 4 { "https://www.deviantart.com/topic/science-fiction" } 33 | 5 { "https://www.deviantart.com/topic/photo-manipulation" } 34 | default { "https://www.deviantart.com/topic/random" } 35 | } 36 | 37 | # Download image 38 | python "$userProfile\$deviantArtScraperFolder\devianart.py" -f wallpaper -c 1 -r -u $url 39 | 40 | # Get all image files in the folder 41 | $wallpapers = Get-ChildItem -Path $wallpaperFolder -Recurse | Where-Object { $_.Extension -match "jpg|jpeg|png|bmp" } 42 | 43 | # Check if any wallpapers were found 44 | if ($null -eq $wallpapers -or $wallpapers.Count -eq 0) { 45 | Write-Host "No wallpapers found in the specified folder: $wallpaperFolder" 46 | exit 47 | } 48 | 49 | # Select a random wallpaper 50 | $randomWallpaper = Get-Random -InputObject $wallpapers 51 | 52 | # Output the selected file for debugging 53 | Write-Host "Selected file: "$randomWallpaper.FullName 54 | 55 | # Set the wallpaper using SystemParametersInfo 56 | $code = @" 57 | using System; 58 | using System.Runtime.InteropServices; 59 | 60 | public class Wallpaper { 61 | [DllImport("user32.dll", CharSet = CharSet.Auto)] 62 | public static extern int SystemParametersInfo(int uAction, int uParam, string lpvParam, int fuWinIni); 63 | 64 | public static void SetWallpaper(string path) { 65 | SystemParametersInfo(0x0014, 0, path, 0x01 | 0x02); 66 | } 67 | } 68 | "@ 69 | 70 | Add-Type -TypeDefinition $code 71 | [Wallpaper]::SetWallpaper($randomWallpaper.FullName) 72 | -------------------------------------------------------------------------------- /wallpaper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Add a cron job to run this script every 15 minutes. 5 | # crontab -e 6 | # */15 * * * * /home/YOUR_USER_NAME/Documents/wallpaper.sh 7 | # 8 | USER=$(whoami) 9 | ORIGINAL_DIR=$(pwd) 10 | LOG_DIR=/var/tmp/wallpaper.log 11 | 12 | START_DATE_TIME=$(date '+%m/%d/%Y %H:%M:%S') 13 | echo "$START_DATE_TIME Starting wallpaper change." > $LOG_DIR 14 | 15 | # Fix to allow cronjob to accurately set the desktop background. https://askubuntu.com/a/198508 16 | count=0 17 | fl=$(find /proc -maxdepth 2 -user $USER -name environ -print -quit) 18 | while [ -z $(grep -z DBUS_SESSION_BUS_ADDRESS "$fl" | cut -d= -f2- | tr -d '\000' ) ] 19 | do 20 | count=$((count+1)) 21 | if [ "$count" -gt 100 ];then 22 | DATE_TIME=$(date '+%m/%d/%Y %H:%M:%S') 23 | echo "$DATE_TIME Failed to find DBUS_SESSION after $((count-1)) attempts." >> $LOG_DIR 24 | break 25 | fi 26 | 27 | DATE_TIME=$(date '+%m/%d/%Y %H:%M:%S') 28 | echo "$DATE_TIME Searching for DBUS_SESSION ($count)." >> $LOG_DIR 29 | 30 | fl=$(find /proc -maxdepth 2 -user $USER -name environ -newer "$fl" -print -quit) 31 | done 32 | 33 | export DBUS_SESSION_BUS_ADDRESS=$(grep -z DBUS_SESSION_BUS_ADDRESS "$fl" | cut -d= -f2-) 34 | DATE_TIME=$(date '+%m/%d/%Y %H:%M:%S') 35 | echo "$DATE_TIME Found DBUS_SESSION at $DBUS_SESSION_BUS_ADDRESS" >> $LOG_DIR 36 | 37 | # Delete cached wallpaper. 38 | rm -f /var/tmp/wallpaper.jpg /var/tmp/wallpaper.jpeg /var/tmp/wallpaper.gif /var/tmp/wallpaper.png 39 | 40 | # Download image. 41 | cd /home/$USER/Documents/deviantart-scraper/ 42 | 43 | month=$(date +%m) 44 | if [ $month -eq "10" ]; then 45 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/horror" >> $LOG_DIR 46 | elif [ $month -eq "11" ]; then 47 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/artisan-crafts" >> $LOG_DIR 48 | elif [ $month -eq "12" ]; then 49 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/poetry" >> $LOG_DIR 50 | elif [ $month -eq "1" ]; then 51 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/photography" >> $LOG_DIR 52 | elif [ $month -eq "2" ]; then 53 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/game-art" >> $LOG_DIR 54 | elif [ $month -eq "3" ]; then 55 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/stock-images" >> $LOG_DIR 56 | elif [ $month -eq "4" ]; then 57 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/science-fiction" >> $LOG_DIR 58 | elif [ $month -eq "5" ]; then 59 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r -u "https://www.deviantart.com/topic/photo-manipulation" >> $LOG_DIR 60 | else 61 | python3 devianart.py -d /var/tmp -f wallpaper -c 1 -r >> $LOG_DIR 62 | fi 63 | 64 | FILE_PATH=$(tail -n 1 $LOG_DIR) 65 | cd $ORIGINAL_DIR 66 | 67 | # Delete cached wallpaper. 68 | rm -f /home/$USER/.cache/wallpaper/* 69 | 70 | # Set new wallpaper. 71 | gsettings set org.gnome.desktop.background picture-options "zoom" 72 | gsettings set org.gnome.desktop.background picture-uri file://$FILE_PATH_invalid 73 | gsettings set org.gnome.desktop.background picture-uri file://$FILE_PATH 74 | 75 | END_DATE_TIME=$(date '+%m/%d/%Y %H:%M:%S') 76 | echo "Downloaded $FILE_PATH" >> $LOG_DIR 77 | echo "Started at $START_DATE_TIME. Finished at $END_DATE_TIME." >> $LOG_DIR 78 | --------------------------------------------------------------------------------