├── .gitignore ├── LICENSE.md ├── README.md ├── featureList ├── imgurScraper.py ├── main.py └── requirements.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.png 2 | *.jpg 3 | *.gif 4 | *~ 5 | *.pyc 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Owen Anderson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MemeMachine 2 | 3 | The best in meme downloading technology, introducing the MemeMachine! Download thousands of memes with the click of a button! 4 | 5 | ## How to use 6 | 7 | Download (or clone) the program using the green button in the upper right hand corner of the page. 8 | If you dont want to clone it just use the download ZIP button. 9 | 10 | First make sure you have python installed. Get it [here] (https://wiki.python.org/moin/BeginnersGuide/Download) 11 | Follow [this] (https://github.com/BurntSushi/nfldb/wiki/Python-&-pip-Windows-installation) guide for Windows. 12 | 13 | Install dependencies (for now dependency :P) 14 | 15 | Navigate open up a terminal (cmd) then navigate to the folder you downloaded it in. Then run this command. 16 | 17 | > pip install -r requirments.txt 18 | 19 | 20 | Then to run it (Python 2.x) 21 | 22 | > python main.py 23 | 24 | ## Features 25 | 26 | Search: This will tell the program what images to search for. You can have multiple searches sepperated by commas. e.g. memes, dank memes. You can also check the box that says FP to get the images just on the front page 27 | 28 | Image Types: You can limit your search to just gifs or to everything but gifs (jpg, png) or just everything. 29 | 30 | Image quanity: This will set the amount of images to be downloaded (or just click the download all to get everything). The amount that you enter will be split up amount all the searches that you input (if you have more than one) 31 | 32 | Albums into folders: This function will put all the images from one album into one folder. Individual images will still be in the main folder 33 | -------------------------------------------------------------------------------- /featureList: -------------------------------------------------------------------------------- 1 | - add the ability to download multiple tags through the use of commas 2 | - change it so it will download equal amounts of images from each tag -------------------------------------------------------------------------------- /imgurScraper.py: -------------------------------------------------------------------------------- 1 | 2 | import requests, re, shutil, os, time 3 | 4 | class settingsObject: 5 | def __init__(self): 6 | self.downloadType = 0 7 | self.FP = False 8 | self.albumsInFolders = True 9 | 10 | def setDownloadType(self, val): 11 | self.downloadType = val 12 | 13 | def setFP(self, val): 14 | self.FP = val 15 | 16 | def setAlbumsInFolders(self, val): 17 | self.albumsInFolders = val 18 | 19 | def getDownloadType(self): 20 | return self.downloadType 21 | 22 | def getFP(self): 23 | return self.FP 24 | 25 | def getAlbumsInFolders(self): 26 | return self.albumsInFolders 27 | 28 | class scraperObject: 29 | def __init__(self): 30 | self.downloadFolder = None 31 | self.downloading = False 32 | self.downloadNum = 0 33 | self.galHashes = [] 34 | #gets a webpage 35 | def getPage(self, url): 36 | page = requests.get(url) 37 | page = page.content 38 | return page 39 | 40 | def getImageNameWhenJSONFails(self, galHash): 41 | page = self.getPage("http://imgur.com/gallery/" + galHash) 42 | try: 43 | index = page.index('itemprop="contentURL"') 44 | except: 45 | return None 46 | cur = index 47 | start = -1 48 | end = -1 49 | numberOfQuotes = 0 50 | while True: 51 | cur -= 1 52 | if page[cur] == '"': 53 | if numberOfQuotes < 2: 54 | numberOfQuotes += 1 55 | else: 56 | if start == -1: 57 | start = cur 58 | elif end == -1: 59 | end = cur 60 | break 61 | url = page[end + 15:start] 62 | if url != "thumbnailUrl": 63 | return url 64 | return None 65 | 66 | #a method that gets all the image names from a gallery hash in array format 67 | def getImagesNamesFromGalHash(self, galHash): 68 | reqJson = requests.get("http://imgur.com/ajaxalbums/getimages/" + galHash + "/hit.json") 69 | try: 70 | imageData = reqJson.json()["data"]["images"] 71 | except TypeError: #this will occur if the gif gallery only has one gif in it 72 | return [self.getImageNameWhenJSONFails(galHash)] 73 | except ValueError: 74 | print("****ERROR****") 75 | time.sleep(5) 76 | return self.getImagesNamesFromGalHash(galHash) 77 | names = [] 78 | for x in imageData: 79 | names.append(x["hash"] + x["ext"]) 80 | return names 81 | 82 | #replaces a * in a url with numbers from start to end and returns the array of them all 83 | def getScrollPageUrl(self, url, pageNum): 84 | index = url.index("*") 85 | newUrl = url[:index] + str(pageNum) + url[index + 1:] 86 | return newUrl 87 | 88 | #gets the gallary hashes from the scroll page 89 | def getGalHashesFromScrollPageUrl(self, url): 90 | page = self.getPage(url) 91 | pageCodes = [] 92 | for x in re.finditer('class="image-list-link"', page): 93 | cur = x.end() 94 | start = -1 95 | end = -1 96 | while True: 97 | cur += 1 98 | if page[cur] == '"': 99 | if start == -1: 100 | start = cur 101 | elif end == -1: 102 | end = cur 103 | break 104 | pageCodes.append(page[start + 9 + 1:end]) 105 | return pageCodes 106 | 107 | #A method that checks if the image is allready in the downloads folder 108 | #--------------------------------------------------------------------- 109 | #TODO make this process more effeicent by storing a list of all 110 | #the file names so you dont have to list the entire dir each time you get an image 111 | def checkIfImageIsDownloaded(self, imageName, path=None): 112 | if path == None: 113 | path = self.downloadFolder 114 | items = os.listdir(path) #gets all the items inside the download folder 115 | if imageName in items: 116 | return True 117 | return False 118 | 119 | #A method that sets the folder that the images will be downloaded to 120 | def changeDownloadFolder(self, folderName): 121 | self.downloadFolder = folderName + "/" 122 | items = os.listdir(".") 123 | self.makeFolderIfNotThere(folderName) 124 | 125 | def makeFolderIfNotThere(self, folderName): 126 | if not os.path.exists(folderName): 127 | os.mkdir(folderName) 128 | 129 | #A method that downloads a single image from imgur given it's name (hash + extension) 130 | def downloadImage(self, name, downloadType, updateCallback, path=None): 131 | if path == None: 132 | path = self.downloadFolder 133 | if name == None or name == "": 134 | return 135 | 136 | if name[len(name) - 1] != "g" and name[len(name) - 1] != "f": 137 | name = name[:len(name) - 2] 138 | 139 | #filters out all images of the set type 140 | if downloadType == 1 and name[len(name) - 1] == "f": 141 | return 142 | elif downloadType == 2 and name[len(name) - 1] != "f": 143 | return 144 | 145 | #downloads the image 146 | self.downloadNum += 1 147 | updateCallback(self.downloading, self.downloadNum) 148 | url = "http://i.imgur.com/" + name 149 | response = requests.get(url, stream=True) 150 | 151 | with open(path + name, "wb") as out_file: 152 | shutil.copyfileobj(response.raw, out_file) 153 | del response 154 | 155 | #stop the downloading 156 | def stopDownload(self): 157 | self.downloading = False 158 | 159 | #a method that was used to divide a number into a whole number as equal as possible 160 | def divideIntoEqualParts(self, num, n): 161 | nums = [] 162 | extra = num % n 163 | minimum = int(num / n) 164 | for x in range(n): #create the base array with the minimum value for all spots 165 | nums.append(minimum) 166 | 167 | for x in range(extra): #add the extra on to the numbers 168 | nums[x] += 1 169 | 170 | return nums 171 | 172 | #download all images from the search querry while keeping each search from the full search 173 | #(sepperated with ,) having an equal amount of images downloaded 174 | def downloadAllImagesFromSearch(self, search, totalLimit, settings, updateCallback): 175 | updateCallback(True, 0) 176 | tags = search.split(",") 177 | limits = self.divideIntoEqualParts(totalLimit, len(tags)) 178 | for x in range(len(tags)): 179 | currentTag = tags[x].strip() 180 | try: 181 | self.downloadAllImagesFromTag(currentTag, limits[x], settings, updateCallback) 182 | except RuntimeError: 183 | print("Finishing downloads... (To exit now press CTRL+C)") 184 | return #program has stopped 185 | if not self.downloading: 186 | return 187 | 188 | # a method that downloads all images given a tag search thing 189 | def downloadAllImagesFromTag(self, search, limit, settings, updateCallback): #gifs only, folder album, front page 190 | self.downloading = True 191 | self.downloadNum = 0 192 | pageNum = 0 193 | setNum = 0 194 | searchUrl = "http://imgur.com/search/score/all/page/*?scrolled&q=" + search + "&q_size_is_mpx=off" 195 | if settings.getFP(): 196 | searchUrl = "http://imgur.com/t/funny/viral/page/*/hit?scrolled&set=" 197 | downloadType = settings.getDownloadType() 198 | currentHash = "first" 199 | while True: 200 | scrollPageUrl = None 201 | if settings.getFP(): #having a sepprate if path from the normal one for the front page bc of the set thing 202 | scrollPageUrl = self.getScrollPageUrl(searchUrl, pageNum) + str(setNum) 203 | if setNum == 10: 204 | setNum = 0 205 | pageNum += 1 206 | else: 207 | setNum += 1 208 | else: 209 | scrollPageUrl = self.getScrollPageUrl(searchUrl, pageNum) 210 | pageNum += 1 211 | galHashes = self.getGalHashesFromScrollPageUrl(scrollPageUrl) 212 | if len(galHashes) == 0: 213 | break 214 | for galHash in galHashes: 215 | if not self.downloading or (self.downloadNum >= limit and limit != -1): 216 | self.doneDownloading(updateCallback) 217 | return 218 | names = self.getImagesNamesFromGalHash(galHash) 219 | if len(names) > 1 and settings.getAlbumsInFolders(): 220 | self.downloadAlbum(galHash, names, limit, downloadType, updateCallback) 221 | else: 222 | for name in names: 223 | if not self.checkIfImageIsDownloaded(name): 224 | if self.downloading: 225 | self.downloadImage(name, downloadType, updateCallback) 226 | else: 227 | self.doneDownloading(updateCallback) 228 | return 229 | self.doneDownloading(updateCallback) 230 | 231 | def doneDownloading(self, updateCallback): 232 | self.downloading = False 233 | updateCallback(False, 0) 234 | 235 | def downloadAlbum(self, name, imageHashes, limit, downloadType, updateCallback): 236 | albumPath = self.downloadFolder + name + "/" 237 | self.makeFolderIfNotThere(albumPath) 238 | for x in imageHashes: 239 | if not self.checkIfImageIsDownloaded(x, path=albumPath): 240 | if self.downloading and (self.downloadNum < limit or limit == -1): 241 | self.downloadImage(x, downloadType, updateCallback, path=albumPath) 242 | else: 243 | return 244 | 245 | def isDownloading(self): 246 | return self.downloading 247 | 248 | def getDownloadNum(self): 249 | return self.downloadNum 250 | 251 | 252 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import imgurScraper as IS 2 | import Tkinter, Tkconstants, tkFileDialog, tkMessageBox, tkFileDialog, threading, os 3 | 4 | class Interface: 5 | def __init__(self, top): 6 | self.downloadThread = None 7 | top.resizable(0, 0) 8 | frame = Tkinter.Frame(top, borderwidth=3) 9 | self.scraper = IS.scraperObject() 10 | 11 | frame.columnconfigure(0, weight=1) 12 | frame.columnconfigure(1, weight=3) 13 | 14 | #create all elements 15 | #who needs good design :P 16 | Tkinter.Label(frame, text="Folder: ").grid(row=0, sticky=Tkinter.W, padx=3, pady=3) 17 | text = os.path.dirname(os.path.realpath(__file__)) + "/defaultDownloadFolder" 18 | self.folderEntryVar = Tkinter.StringVar() 19 | self.folderEntryVar.set(text) 20 | self.folderEntry = Tkinter.Entry(frame, textvariable=self.folderEntryVar) 21 | self.folderEntry.grid(row=0, column=1, padx=3, pady=3, columnspan=2) 22 | 23 | Tkinter.Label(frame, text="Search: ").grid(row=1, sticky=Tkinter.W, padx=3, pady=3) 24 | self.searchEntry = Tkinter.Entry(frame) 25 | self.searchEntry.grid(row=1, column=1, padx=3, pady=3, columnspan=2) 26 | 27 | self.albumIntoFolders = Tkinter.IntVar() 28 | self.albumIntoFolders.set(1) 29 | albumIntoFoldersCheckbox = Tkinter.Checkbutton(frame, text="Put albums into folders", variable=self.albumIntoFolders) 30 | albumIntoFoldersCheckbox.grid(row=2, sticky=Tkinter.W, columnspan=2) 31 | 32 | self.imageType = Tkinter.IntVar() 33 | r1 = Tkinter.Radiobutton(frame, text="All", variable=self.imageType, value=0) 34 | r2 = Tkinter.Radiobutton(frame, text="Only images", variable=self.imageType, value=1) 35 | r3 = Tkinter.Radiobutton(frame, text="Only gifs", variable=self.imageType, value=2) 36 | r1.grid(row=3, sticky=Tkinter.W, column=0) 37 | r2.grid(row=3, sticky=Tkinter.W, column=1) 38 | r3.grid(row=3, sticky=Tkinter.W, column=2) 39 | 40 | self.frontPage = Tkinter.IntVar() 41 | frontPage = Tkinter.Checkbutton(frame, text="FP", variable=self.frontPage, command=self.toggleFrontPage) 42 | frontPage.grid(row=1, column=3, sticky=Tkinter.W) 43 | 44 | self.b1 = Tkinter.Button(frame, text="Download Images", command=self.startDownload) 45 | self.b1.grid(row=4, sticky=Tkinter.W, columnspan=2) 46 | 47 | self.b2 = Tkinter.Button(frame, text="Select", command=self.selectFolder) 48 | self.b2.grid(row=0, column=3) 49 | 50 | self.downloadAllVar = Tkinter.IntVar() 51 | downloadAllCheckbox = Tkinter.Checkbutton(frame, text="Download All", command=self.toggleDownloadAll, variable=self.downloadAllVar) 52 | downloadAllCheckbox.grid(row=0, column=4, columnspan=2) 53 | 54 | Tkinter.Label(frame, text="Num of Images: ").grid(row=1, column=4, sticky=Tkinter.E) 55 | spinnerDe = Tkinter.StringVar() 56 | spinnerDe.set("5000") 57 | self.imageNumSpinner = Tkinter.Spinbox(frame, textvariable=spinnerDe, width=4, from_=1, to=999999) 58 | self.imageNumSpinner.grid(row=1, column=5) 59 | 60 | self.downloadText = Tkinter.StringVar() 61 | Tkinter.Label(frame, textvariable=self.downloadText).grid(row=4, column=2, columnspan=4, sticky=Tkinter.E) 62 | self.downloadText.set("Not Downloading...") 63 | 64 | frame.grid() 65 | 66 | def displayError(self, title, errorBody): 67 | tkMessageBox.showwarning(title, errorBody) 68 | 69 | def finsihedDownload(self): 70 | try: 71 | self.downloadMoniter(False, 0) 72 | except RuntimeError: 73 | return 74 | tkMessageBox.showinfo("Done", "All the images have been downloaded.") 75 | 76 | def toggleDownloadAll(self): 77 | if self.downloadAllVar.get() == 1: 78 | self.imageNumSpinner.config(state=Tkinter.DISABLED) 79 | else: 80 | self.imageNumSpinner.config(state="normal") 81 | 82 | def toggleFrontPage(self): 83 | if self.frontPage.get() == 1: 84 | self.searchEntry.config(state=Tkinter.DISABLED) 85 | else: 86 | self.searchEntry.config(state="normal") 87 | 88 | def closeWindow(): 89 | if self.downloadThread != None: 90 | self.scraper.stopDownload() 91 | 92 | def downloadImages(self, search, limit, settings, done, updateCallback): 93 | self.scraper.downloadAllImagesFromSearch(search, limit, settings, updateCallback) 94 | done() 95 | 96 | def downloadMoniter(self, downloading, downloadNum): 97 | if(downloading): 98 | self.downloadText.set("Downloading Image: " + str(downloadNum)) 99 | else: 100 | self.downloadText.set("Not Downloading...") 101 | 102 | def startDownload(self): 103 | if self.scraper.isDownloading(): 104 | self.displayError("Downloading", "You are allready downloading something") 105 | limit = int(self.imageNumSpinner.get()) 106 | if self.downloadAllVar.get() == 1: 107 | limit = -1 108 | searchQ = self.searchEntry.get() 109 | settings = IS.settingsObject() 110 | if self.frontPage.get() == 1: 111 | settings.setFP(True) 112 | settings.setDownloadType(self.imageType.get()) 113 | if self.albumIntoFolders.get() == 0: 114 | settings.setAlbumsInFolders(False) 115 | self.scraper.changeDownloadFolder(self.folderEntryVar.get()) 116 | self.downloadThread = threading.Thread(target=self.downloadImages, args=(searchQ, limit, settings, self.finsihedDownload, self.downloadMoniter)) 117 | self.downloadThread.start() 118 | 119 | def selectFolder(self): 120 | directoryName = tkFileDialog.askdirectory() 121 | self.folderEntryVar.set(directoryName) 122 | 123 | master = Tkinter.Tk(); 124 | interface = Interface(master) 125 | master.wm_title("Meme Machine") 126 | master.mainloop() 127 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.13.0 2 | --------------------------------------------------------------------------------