├── .dockerignore ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── bin ├── Config.py ├── Converter.py ├── ConverterHandler.py ├── Downloader.py ├── DownloaderHandler.py ├── Helper.py ├── Migrator.py ├── Models.py ├── RssParser.py ├── Sender.py ├── SenderHandler.py ├── __init__.py ├── _version.py ├── models │ ├── Manga.py │ └── __init__.py └── sourceparser │ ├── Cdmnet.py │ ├── Mangafox.py │ ├── Mangastream.py │ └── __init__.py ├── config.ini ├── docker-compose.yml ├── m2em.py ├── migrations ├── 001_initial.py ├── 002_testmigration.py └── 003_filters.py └── requirements.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode 3 | test\ Kopie.db 4 | _test.db 5 | test.db 6 | __pycache__ 7 | comic/* 8 | data/* 9 | kindlegen.exe 10 | .idea 11 | venv 12 | main.db 13 | log/* 14 | docker-compose.yml 15 | Dockerfile 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .vscode 3 | test\ Kopie.db 4 | _test.db 5 | test.db 6 | __pycache__ 7 | comic/* 8 | data/* 9 | kindlegen.exe 10 | .idea 11 | venv 12 | main.db 13 | log/* 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-slim 2 | 3 | MAINTAINER Schemen 4 | 5 | 6 | WORKDIR /usr/src/app 7 | 8 | VOLUME /usr/src/app/data 9 | 10 | ENTRYPOINT ["/usr/bin/dumb-init", "--"] 11 | 12 | COPY requirements.txt ./ 13 | RUN apt-get update && apt-get install dumb-init gcc wget -y && \ 14 | rm -rf /var/lib/apt/lists/* && \ 15 | pip install --no-cache-dir -r requirements.txt && \ 16 | apt-get purge gcc -y && apt-get autoremove -y && apt-get clean 17 | 18 | RUN wget http://kindlegen.s3.amazonaws.com/kindlegen_linux_2.6_i386_v2_9.tar.gz -O /tmp/kindlegen.tar.gz && \ 19 | tar xvf /tmp/kindlegen.tar.gz -C /tmp && mv /tmp/kindlegen /usr/bin && \ 20 | rm -r /tmp/* 21 | 22 | 23 | COPY . . 24 | 25 | CMD [ "python","m2em.py", "--daemon", "-s"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Elia Ponzio 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # m2em - Manga to eManga 2 | 3 | ### Foreword 4 | I always had the issue of loving my kindle and ebooks and loving mangas. 5 | 6 | While I buy books/mangas to support the author and have a nice collection, I love & support the e-Format and only read on those as they're are way easier to use. 7 | 8 | Not living in Japan has me not really having any readable access of weekly chapters in eManga format, so I wanted to write something to help me out on that! 9 | 10 | ### Here comes M2EM 11 | 12 | M2em let's you automatically download Mangas via RSS Feed that updates at a configurable interval (and comics in the future?), convert them into eMangas and send them off via Email (Target being the Email to Kindle function of Amazon)! 13 | 14 | ## Supported Websites 15 | 16 | * Mangastream 17 | * MangaFox (With Splash Rendering container) 18 | * Cdmnet 19 | 20 | # Setup 21 | 22 | M2em requires Python3 and I highly recommend working in a virtualenv and if you want to use Mangasources which are JavaScript heavy, I actually recommend to use docker to deploy the m2em binary and the rendering service together. Some OS require the python-dev package! 23 | 24 | ## Docker Setup 25 | You can use the Dockerfile or the image schemen/m2em. All options in the config.ini are available as environment variable. Make sure you write the exactly the same! 26 | 27 | Have a look at the example Compose file in the repository. This will deploy two containers, m2em and splash. Splash is to render websites which use javascript. The alias (which you can add to your bashrc if you want) allows you to directly call the containerized application 28 | 29 | ``` 30 | docker-compose up -d 31 | alias m2em='sudo docker exec -it m2em_m2em_1 ./m2em.py' 32 | m2em -h 33 | ``` 34 | 35 | ## Create and install virtual environment 36 | ```x-sh 37 | git clone git@github.com:schemen/m2em.git && cd m2em 38 | virtualenv venv -p python3 39 | source venv/bin/activate 40 | pip install -r requirements.txt 41 | deactivate 42 | ``` 43 | 44 | ## dependencies 45 | * validators 46 | * texttable 47 | * requests 48 | * bs4 49 | * urllib3 50 | * feedparser 51 | * KindleComicConverter 52 | 53 | 54 | ## Optional dependencies 55 | * KindleGen v2.9+ in a directory reachable by your PATH or in KCC directory (For MOBI generation). For Windows, place the .exe in the same directory 56 | 57 | Get Kindlegen here: https://www.amazon.com/gp/feature.html?docId=1000765211 58 | 59 | 60 | ## Concept 61 | As a concept, M2em has different workers that run in a loop. All Chapter/user data is saved in a SQLite3 Database. 62 | * RssParser - Parsing the RSS feed and saving the information of each chapter 63 | * Downloader - Downloading the Mangas and saving them 64 | * Converter - Converting images into ebooks 65 | * Sender - Compiling & Sending Emails to users and marking them as SENT 66 | 67 | 68 | ### The Loop Run & Daemon Loop Run 69 | If you start m2em in loop mode (with or without --daemon) it will only consider any action with elements that that are 70 | younger than 24h hours. 71 | 72 | The use of that is having it running on the server 24/7, waiting for updates from the feeds and ONLY handling said updates. 73 | 74 | ### Direct action 75 | You can start a part of the loop without the restriction of 24h. Use the -a (--action) command with either element you wish to start. 76 | 77 | Example: if you wish to download all chapters you have saved in your database, you start the download action. 78 | ``` 79 | ./m2em.py --action downloader 80 | ``` 81 | ### Chapter action 82 | You can directly apply an action to one chapter with the options --download, --convert or --send. You need to pass 83 | the ID of said chapter, you can find that out with "-Lc" or "-lc". 84 | You can pass multiple IDs. 85 | 86 | Also, you can process N chapters with the "--process/-p" option: 87 | ``` 88 | ./m2em.py -p 100 #Downloads, Converts and Sends chapter with ID 100 89 | ``` 90 | 91 | 92 | ``` 93 | ./m2em.py --download 100 #Downloads chapter with ID 100 94 | ``` 95 | 96 | # Usage 97 | 98 | ### Help output: 99 | ``` 100 | usage: m2em.py [-h] [-af ADD_FEED] [-au] [-lm [LIST_MANGA]] [-lc] [-Lc] [-lf] 101 | [-lu] [-cd] [-s] [--send [SEND [SEND ...]]] 102 | [--convert [CONVERT [CONVERT ...]]] 103 | [--download [DOWNLOAD [DOWNLOAD ...]]] 104 | [-p [PROCESS [PROCESS ...]]] [-a ACTION] [-ss SWITCH_SEND] 105 | [-dc DELETE_CHAPTER] [-du DELETE_USER] [-df DELETE_FEED] 106 | [--daemon] [-d] [-v] 107 | 108 | Manga to eManga - m2em 109 | 110 | optional arguments: 111 | -h, --help show this help message and exit 112 | -af ADD_FEED, --add-feed ADD_FEED 113 | Add RSS Feed of Manga. Only Mangastream & MangaFox are 114 | supported 115 | -au, --add-user Adds new user 116 | -lm [LIST_MANGA], --list-manga [LIST_MANGA] 117 | Lists Manga saved in database. If a Manga is passed, 118 | lists chapters to said Manga 119 | -lc, --list-chapters Lists the last 10 Chapters 120 | -Lc, --list-chapters-all 121 | Lists all Chapters 122 | -lf, --list-feeds Lists all feeds 123 | -lu, --list-users Lists all Users 124 | -cd, --create-db Creates DB. Uses Configfile for Naming 125 | -s, --start Starts one loop 126 | --send [SEND [SEND ...]] 127 | Sends Chapter directly by chapter ID. Multiple IDs can 128 | be given 129 | --convert [CONVERT [CONVERT ...]] 130 | Converts Chapter directly by chapter ID. Multiple IDs 131 | can be given 132 | --download [DOWNLOAD [DOWNLOAD ...]] 133 | Downloads Chapter directly by chapter ID. Multiple IDs 134 | can be given 135 | -p [PROCESS [PROCESS ...]], --process [PROCESS [PROCESS ...]] 136 | Processes chapter(s) by chapter ID, Download, convert, 137 | send. Multiple IDs can be given 138 | -a ACTION, --action ACTION 139 | Start action. Options are: rssparser (collecting feed 140 | data), downloader, converter or sender 141 | -ss SWITCH_SEND, --switch-send SWITCH_SEND 142 | Pass ID of User. Switches said user Send eBook status 143 | -dc DELETE_CHAPTER, --delete-chapter DELETE_CHAPTER 144 | Pass ID of Chapter. Deletes said Chapter 145 | -du DELETE_USER, --delete-user DELETE_USER 146 | Pass ID of User. Deletes said User 147 | -df DELETE_FEED, --delete-feed DELETE_FEED 148 | Pass ID of Feed. Deletes said Feed 149 | --daemon Run as daemon 150 | -d, --debug Debug Mode 151 | -v, --version show program's version number and exit 152 | -f "filter_regex", --filter "filter_regex" 153 | Adds a filter(python regex format), to filter the 154 | title of any manga parsed. Example: "(?i)one-punch" 155 | -fl, --filter-list Lists all filters 156 | 157 | 158 | ``` 159 | 160 | ## Initial Data 161 | To have a working environment you need to add some initial data and create the database 162 | ```x-sh 163 | ./m2em.py --create-db # Create a DB 164 | ./m2em.py --add-feed # Add an RSS Feed you want to pull 165 | # Please note that you should set the sending AFTER a complete run for now 166 | ./m2em.py --add-user # Add a user 167 | 168 | ``` 169 | 170 | For the sending to work, you need to have an email account so the program can send from it. I recommend creating a new email account for this. Change the Email settings in the config.ini to be able to use it. 171 | 172 | ## Config File 173 | ``` 174 | [CONFIG] 175 | # Location relative to the code position 176 | SaveLocation = data/ 177 | # Database name 178 | Database = main.db 179 | # Duration the program sleeps after one run is finished in seconds 180 | Sleep = 900 181 | # Ebook output Format, check 182 | # https://github.com/ciromattia/kcc for more information 183 | EbookFormat = MOBI 184 | # Ebook Profile setting, check 185 | # https://github.com/ciromattia/kcc for more information 186 | EbookProfile = KV 187 | # If you want to run splash intependently change this setting 188 | SplashServer = http://splash:8050 189 | # Sender Email Server Settings 190 | SMTPServer = mail.example.com 191 | ServerPort = 587 192 | EmailAdress = comic@example.com 193 | EmailAdressPw = yourpassword 194 | ServerStartSSL = True 195 | ``` 196 | 197 | ## Examples 198 | 199 | If you want to check out the manga that are in the database: 200 | ``` 201 | ./m2em.py -lm 202 | ``` 203 | You can use this out put to refine the search! 204 | If you pass the manga name you get all chapters listed from that manga: 205 | ``` 206 | ./m2em.py -lm "Shokugeki no Souma" 207 | Listing all chapters of Shokugeki no Souma: 208 | ID MANGA CHAPTER CHAPTERNAME RSS ORIGIN SEND STATUS 209 | =================================================================================================== 210 | 112 Shokugeki no Souma 240 Not Cute https://mangastream.com/rss SENT 211 | 212 | 91 Shokugeki no Souma 238 The Queen's Tart https://mangastream.com/rss SENT 213 | 214 | 78 Shokugeki no Souma 239 Her Fighting Style https://mangastream.com/rss SENT 215 | ``` 216 | 217 | 218 | To start a single run through the workers, you can simply execute the main program: 219 | ``` 220 | ./m2em.py -s 221 | ``` 222 | 223 | If you wish to run the program as a daemon, start it with the option "--daemon" as well. It will re-run at the config "Sleep" in second. 224 | ``` 225 | ./m2em.py -s --daemon 226 | ``` 227 | 228 | If you wish to disable/enable sending status of a user, use the -ss command 229 | ``` 230 | ./m2em.py -ss 231 | ``` 232 | 233 | 234 | ### A complete run with nothing happening: 235 | ``` 236 | Starting Loop at 2017-11-15 18:13:05 237 | Starting RSS Data Fetcher! 238 | Checking for new Feed Data... 239 | Getting Feeds for https://mangastream.com/rss 240 | Finished Loading RSS Data 241 | Starting all outstanding Chapter Downloads! 242 | Finished all outstanding Chapter Downloads 243 | Starting recursive image conversion! 244 | Finished recursive image conversion! 245 | Starting to send all ebooks! 246 | Finished sending ebooks! 247 | ``` 248 | 249 | ## Other 250 | Everything else should be self-explanatory with the "-h" option. 251 | 252 | ## Known Issues 253 | * MangaFox has issues with SSL Verification on some systems. For now, Simply add the http feed. 254 | 255 | Please Open an issue if you find anything! 256 | 257 | # Acknowledgement 258 | I greatly thank Ciro Mattia Gonano and Paweł Jastrzębski that created the KCC Library that enables the automatic conversation into Ebooks that are compatible with all Comic features of the Kindle! 259 | https://github.com/ciromattia/kcc 260 | -------------------------------------------------------------------------------- /bin/Config.py: -------------------------------------------------------------------------------- 1 | """Config loading Module""" 2 | import configparser 3 | import os 4 | 5 | def load_config(location='config.ini'): 6 | """Function that returns a configuration as dict""" 7 | config = {} 8 | config_reader = configparser.ConfigParser() 9 | config_reader.optionxform = str 10 | config_reader.read(location) 11 | 12 | for key, value in config_reader.items("CONFIG"): 13 | if key in os.environ: 14 | config[key] = os.environ[key] 15 | else: 16 | config[key] = value 17 | 18 | return config 19 | -------------------------------------------------------------------------------- /bin/Converter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ Converter Module """ 3 | import logging 4 | import os 5 | import zipfile 6 | import subprocess 7 | import bin.Config as Config 8 | import bin.Helper as helper 9 | 10 | 11 | class Converter: 12 | """ This Class Converts the result of the Downloader class into Ebooks """ 13 | 14 | def __init__(self): 15 | self.saveloc = None 16 | self.ebformat = None 17 | self.ebprofile = None 18 | self.mangatitle = None 19 | self.manganame = None 20 | self.imagefolder = None 21 | self.eblocation = None 22 | self.cbzlocation = None 23 | self.chapterdate = None 24 | 25 | 26 | 27 | def data_collector(self, chapter): 28 | """ Method that collects data """ 29 | 30 | # Load config right at the start 31 | config = None 32 | if not config: 33 | config = Config.load_config() 34 | 35 | # Load configs required here 36 | self.saveloc = config["SaveLocation"] 37 | self.ebformat = config["EbookFormat"] 38 | self.ebprofile = config["EbookProfile"] 39 | 40 | 41 | # get relevant data of this Manga 42 | self.mangatitle = chapter.title 43 | self.manganame = chapter.manganame 44 | self.chapterdate = chapter.date 45 | 46 | # check if mangatitle or manganame contains ":" characters that OS can't handle as folders 47 | self.mangatitle = helper.sanetizeName(self.mangatitle) 48 | self.manganame = helper.sanetizeName(self.manganame) 49 | 50 | 51 | # create folder variables 52 | self.imagefolder = str(self.saveloc + self.manganame + "/" + 53 | self.mangatitle + "/images/") 54 | self.eblocation = str(self.saveloc + self.manganame + "/"+ 55 | self.mangatitle + "/" + self.mangatitle + "." + self.ebformat.lower()) 56 | self.cbzlocation = str(self.saveloc + self.manganame + "/"+ 57 | self.mangatitle + "/" + self.mangatitle + ".cbz") 58 | 59 | 60 | 61 | 62 | def cbz_creator(self): 63 | """ Method that converts images into CBZ""" 64 | 65 | # Create CBZ to make creation easier 66 | if os.path.exists(self.cbzlocation): 67 | logging.debug("Manga %s converted to CBZ already!", self.mangatitle) 68 | else: 69 | logging.info("Starting conversion to CBZ of %s...", self.mangatitle) 70 | 71 | 72 | logging.debug("Opening CBZ archive...") 73 | try: 74 | zfile = zipfile.ZipFile(self.cbzlocation, "w") 75 | except Exception as fail: 76 | logging.warning("Failed opening archive! %s", fail) 77 | 78 | 79 | 80 | logging.debug("Writing Images into CBZ") 81 | for img in sorted(os.listdir(self.imagefolder)): 82 | image = self.imagefolder + img 83 | logging.debug("Writing %s", image) 84 | zfile.write(image, img) 85 | 86 | zfile.close() 87 | 88 | 89 | 90 | def eb_creator(self): 91 | """ Method that creates the Ebook out of the CBZ """ 92 | 93 | # Start conversion to Ebook format! 94 | if os.path.exists(self.eblocation): 95 | logging.debug("Manga %s converted to Ebook already!", self.mangatitle) 96 | else: 97 | logging.info("Starting conversion to Ebook of %s...", self.mangatitle) 98 | 99 | try: 100 | subprocess.call(["kcc-c2e", "-p", self.ebprofile, "-f", self.ebformat, 101 | "-m", "-r", "2", "-u", "-s", self.cbzlocation]) 102 | except Exception as fail: 103 | logging.debug("Failed to convert epub %s", fail) 104 | -------------------------------------------------------------------------------- /bin/ConverterHandler.py: -------------------------------------------------------------------------------- 1 | """ Module that handles the workflow of the Converter Class """ 2 | import logging 3 | import os 4 | import bin.Helper as helper 5 | from bin.Converter import Converter 6 | 7 | def ConverterHandler(args): 8 | """ Function that handles the Converter in a loop """ 9 | 10 | # Load Chapters! 11 | chapters = helper.getChapters() 12 | 13 | 14 | # Start conversion loop! 15 | for chapter in chapters.iterator(): 16 | 17 | 18 | # Verify if chapter has been downloaded already 19 | if not helper.verifyDownload(chapter): 20 | logging.debug("Manga %s has not been downloaded!", chapter.title) 21 | else: 22 | 23 | 24 | # Spawn an Converter Object & get basic data from database & config 25 | current_conversation = Converter() 26 | current_conversation.data_collector(chapter) 27 | 28 | # Check if Download loop & Download task is selected 29 | if not args.start: 30 | current_conversation.cbz_creator() 31 | current_conversation.eb_creator() 32 | else: 33 | 34 | # Only start run if chapter is younger than 24h 35 | if helper.checkTime(current_conversation.chapterdate): 36 | current_conversation.cbz_creator() 37 | current_conversation.eb_creator() 38 | else: 39 | logging.debug("%s is older than 24h, will not be processed by daemon.", 40 | current_conversation.mangatitle) 41 | 42 | 43 | 44 | 45 | def directConverter(chapterids=[]): 46 | """ Function that handles direct calls of the Converter """ 47 | 48 | logging.debug("Following Chapters are directly converted:") 49 | logging.debug(chapterids) 50 | 51 | chapters = helper.getChaptersFromID(chapterids) 52 | 53 | 54 | if not chapters: 55 | logging.error("No Chapters found with said ID!") 56 | else: 57 | # Start conversion loop! 58 | for chapter in chapters: 59 | 60 | # Verify if chapter has been downloaded already 61 | if not helper.verifyDownload(chapter): 62 | logging.info("Manga %s has not been downloaded!", chapter[2]) 63 | else: 64 | 65 | 66 | # Spawn an Converter Object & get basic data from database & config 67 | current_conversation = Converter() 68 | current_conversation.data_collector(chapter) 69 | 70 | if os.path.exists(current_conversation.cbzlocation): 71 | logging.info("Manga %s converted to CBZ already!", 72 | current_conversation.mangatitle) 73 | else: 74 | current_conversation.cbz_creator() 75 | 76 | # Start conversion to Ebook format! 77 | if os.path.exists(current_conversation.eblocation): 78 | logging.info("Manga %s converted to Ebook already!", 79 | current_conversation.mangatitle) 80 | else: 81 | current_conversation.eb_creator() 82 | -------------------------------------------------------------------------------- /bin/Downloader.py: -------------------------------------------------------------------------------- 1 | """Downloader Module""" 2 | import logging 3 | import os 4 | import requests 5 | import bin.Config as Config 6 | import bin.Helper as helper 7 | import bin.sourceparser.Mangastream as msparser 8 | import bin.sourceparser.Mangafox as mxparser 9 | import bin.sourceparser.Cdmnet as cdmparser 10 | from PIL import Image 11 | from PIL import ImageOps 12 | from PIL import ImageFilter 13 | 14 | 15 | class Downloader: 16 | """Class to manage downloads""" 17 | 18 | def __init__(self): 19 | self.database = None 20 | self.saveloc = None 21 | self.mangastarturl = None 22 | self.mangapages = None 23 | self.mangatitle = None 24 | self.manganame = None 25 | self.chapterdate = None 26 | self.downloadfolder = None 27 | self.origin = None 28 | self.imageurls = None 29 | 30 | 31 | 32 | 33 | def data_collector(self, chapter): 34 | """Method to collect and fill data for the class""" 35 | 36 | # Load config right at the start 37 | config = None 38 | if not config: 39 | config = Config.load_config() 40 | 41 | # Load configs required here 42 | self.database = config["Database"] 43 | self.saveloc = config["SaveLocation"] 44 | 45 | # get relevant data of this Chapter 46 | self.mangastarturl = chapter.url 47 | self.mangapages = chapter.pages 48 | self.mangatitle = chapter.title 49 | self.manganame = chapter.manganame 50 | self.chapterdate = chapter.date 51 | 52 | # check if mangatitle or manganame contains ":" characters that OS can't handle as folders 53 | self.mangatitle = helper.sanetizeName(self.mangatitle) 54 | self.manganame = helper.sanetizeName(self.manganame) 55 | 56 | # Define Download location 57 | self.downloadfolder = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/images") 58 | 59 | # get Origin of manga (Which mangawebsite) 60 | self.origin = helper.getSourceURL(self.mangastarturl) 61 | 62 | # Initiate URL list 63 | self.imageurls = [] 64 | 65 | 66 | 67 | 68 | def data_processor(self): 69 | """Method that starts processing the collected data""" 70 | 71 | logging.info("Proccesing data for %s", self.mangatitle) 72 | 73 | 74 | # Get image urls! 75 | # Mangastream Parser 76 | if self.origin == "mangastream.com" or self.origin == "readms.net": 77 | urllist = msparser.getPagesUrl(self.mangastarturl, self.mangapages) 78 | 79 | # check if we have images to download 80 | if not len(urllist) == 0: 81 | 82 | # Turn Manga pages into Image links! 83 | logging.info("Starting download of %s...", self.mangatitle) 84 | counter = 0 85 | for i in urllist: 86 | counter = counter + 1 87 | self.downloader(i, counter, msparser.getImageUrl) 88 | 89 | 90 | # Finish :) 91 | logging.info("Finished download of %s!", self.mangatitle) 92 | 93 | # Mangafox Parser 94 | elif self.origin == "mangafox.me" or self.origin == "mangafox.la" or self.origin == "fanfox.net": 95 | urllist = mxparser.getPagesUrl(self.mangastarturl, self.mangapages) 96 | 97 | # check if we have images to download 98 | if not len(urllist) == 0: 99 | 100 | # Turn Manga pages into Image links! 101 | logging.info("Starting download of %s...", self.mangatitle) 102 | counter = 0 103 | for i in urllist: 104 | counter = counter + 1 105 | self.downloader(i, counter, mxparser.getImageUrl) 106 | 107 | 108 | # Finish :) 109 | logging.info("Finished download of %s!", self.mangatitle) 110 | 111 | # CDM Parser 112 | elif self.origin == "cdmnet.com.br": 113 | urllist = cdmparser.getPagesUrl(self.mangastarturl, self.mangapages) 114 | 115 | # check if we have images to download 116 | if not len(urllist) == 0: 117 | 118 | # Turn Manga pages into Image links! 119 | logging.info("Starting download of %s...", self.mangatitle) 120 | counter = 0 121 | for i in urllist: 122 | counter = counter + 1 123 | self.downloader(i, counter, cdmparser.getImageUrl) 124 | 125 | 126 | # Finish :) 127 | logging.info("Finished download of %s!", self.mangatitle) 128 | 129 | def downloader(self, url, counter, parser): 130 | """Method that downloads files""" 131 | 132 | # Check if we have the Download folder 133 | helper.createFolder(self.downloadfolder) 134 | 135 | imagepath = self.downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".png" 136 | tempdl = self.downloadfolder + "/" + str("{0:0=3d}".format(counter)) + ".tmp" 137 | 138 | # Download the image! 139 | f = open(tempdl, 'wb') 140 | f.write(requests.get(parser(url), headers={'referer': url}).content) 141 | f.close() 142 | 143 | # convert img to png 144 | imgtest = Image.open(tempdl) 145 | if imgtest.format != 'PNG': 146 | logging.debug("Image %s is not a PNG... convertig.", tempdl) 147 | imgtest.save(tempdl, "PNG") 148 | else: 149 | imgtest.close() 150 | 151 | # If everything is alright, write image to final name 152 | os.rename(tempdl, imagepath) 153 | 154 | 155 | # Cleanse image, remove footer 156 | # 157 | # I have borrowed this code from the kmanga project. 158 | # https://github.com/aplanas/kmanga/blob/master/mobi/mobi.py#L416 159 | # Thanks a lot to Alberto Planas for coming up with it! 160 | # 161 | if self.origin == "mangafox.me" or self.origin == "mangafox.la" or self.origin == "fanfox.net": 162 | logging.debug("Cleaning Mangafox Footer") 163 | img = Image.open(imagepath) 164 | _img = ImageOps.invert(img.convert(mode='L')) 165 | _img = _img.point(lambda x: x and 255) 166 | _img = _img.filter(ImageFilter.MinFilter(size=3)) 167 | _img = _img.filter(ImageFilter.GaussianBlur(radius=5)) 168 | _img = _img.point(lambda x: (x >= 48) and x) 169 | 170 | cleaned = img.crop(_img.getbbox()) if _img.getbbox() else img 171 | cleaned.save(imagepath) 172 | -------------------------------------------------------------------------------- /bin/DownloaderHandler.py: -------------------------------------------------------------------------------- 1 | """ Module to handle the workflow of with the Downloader Class """ 2 | import logging 3 | import os 4 | from shutil import move 5 | import bin.Helper as helper 6 | from bin.Downloader import Downloader 7 | 8 | 9 | def downloader(args): 10 | """ the downloader function """ 11 | 12 | # Make the query 13 | chapters = helper.getChapters() 14 | 15 | if args.start: 16 | logging.debug("The loop will only consider Chapters younger than 24h!") 17 | 18 | 19 | 20 | # Start Download loop! 21 | for chapter in chapters.iterator(): 22 | 23 | # Initialize Downloader class & load basic params 24 | current_chapter = Downloader() 25 | current_chapter.data_collector(chapter) 26 | 27 | 28 | # Check if the old DL location is being used and fix it! 29 | oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) 30 | newlocation = str(current_chapter.saveloc + current_chapter.manganame) 31 | if os.path.isdir(oldlocation): 32 | logging.info("Moving %s from old DL location to new one...", current_chapter.mangatitle) 33 | helper.createFolder(newlocation) 34 | move(oldlocation, newlocation) 35 | 36 | 37 | 38 | # Check if chapter needs to be downloaded 39 | if helper.verifyDownload(chapter): 40 | logging.debug("Manga %s downloaded already!", current_chapter.mangatitle) 41 | else: 42 | 43 | # Check if Download loop & Download task is selected 44 | if not args.start: 45 | current_chapter.data_processor() 46 | else: 47 | # Only start run if chapter is younger than 24h 48 | if helper.checkTime(current_chapter.chapterdate): 49 | current_chapter.data_processor() 50 | else: 51 | logging.debug("%s is older than 24h, will not be processed by daemon.", current_chapter.mangatitle) 52 | 53 | 54 | def directDownloader(chapterids=[]): 55 | """ Function to handle direct download calls """ 56 | logging.debug("Following Chapters are directly converted:") 57 | logging.debug(chapterids) 58 | 59 | 60 | chapters = helper.getChaptersFromID(chapterids) 61 | 62 | # Load Users 63 | users = helper.getUsers() 64 | 65 | # Debug Users: 66 | logging.debug("Userlist:") 67 | logging.debug(users) 68 | 69 | 70 | if not chapters: 71 | logging.error("No Chapters found with said ID!") 72 | else: 73 | # Start conversion loop! 74 | for chapter in chapters: 75 | 76 | # Initialize Downloader class & load basic params 77 | current_chapter = Downloader() 78 | current_chapter.data_collector(chapter) 79 | 80 | # Check if the old DL location is being used and fix it! 81 | oldlocation = str(current_chapter.saveloc + current_chapter.mangatitle) 82 | newlocation = str(current_chapter.saveloc + current_chapter.manganame) 83 | if os.path.isdir(oldlocation): 84 | logging.info("Moving %s from old DL location to new one...", current_chapter.mangatitle) 85 | helper.createFolder(newlocation) 86 | move(oldlocation, newlocation) 87 | 88 | # Check if chapter needs to be downloaded 89 | if helper.verifyDownload(chapter): 90 | logging.info("Manga %s downloaded already!", current_chapter.mangatitle) 91 | else: 92 | 93 | current_chapter.data_processor() 94 | -------------------------------------------------------------------------------- /bin/Helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import logging 3 | import shutil 4 | import datetime 5 | import os 6 | import texttable 7 | import requests 8 | import validators 9 | from dateutil import parser 10 | from urllib.parse import urlparse 11 | import bin.Config as Config 12 | from bin.Models import * 13 | import bin.sourceparser.Mangastream as msparser 14 | import bin.sourceparser.Mangafox as mxparser 15 | import bin.sourceparser.Cdmnet as cdmparser 16 | 17 | ''' 18 | 19 | Helper Unit. 20 | This File provides functions to other classes 21 | 22 | ''' 23 | 24 | # Load config right at the start 25 | config = None 26 | if not config: 27 | config = Config.load_config() 28 | 29 | 30 | ''' 31 | Create Database! 32 | ''' 33 | def createDB(): 34 | create_tables() 35 | 36 | ''' 37 | Function set manga as sent 38 | Returns: N/A 39 | ''' 40 | def setIsSent(mangaid): 41 | 42 | try: 43 | # Open DB 44 | db.connection() 45 | query = Chapter.update(issent=1).where(Chapter.chapterid == mangaid) 46 | query.execute() 47 | logging.debug("Set chapter with ID %s as sent", mangaid) 48 | except Exception as fail: 49 | logging.debug("Failed to save feed into database: %s", fail) 50 | 51 | 52 | 53 | 54 | ''' 55 | Function write a feed into the DB 56 | Returns: N/A 57 | ''' 58 | def writeFeed(url): 59 | 60 | # Connect to DB 61 | db.connection() 62 | 63 | # Insert Data 64 | feed = Feeds.create(url=url) 65 | feed.save() 66 | logging.info("Succesfully added \"%s\" to the List of RSS Feeds", url) 67 | 68 | # Close connection 69 | db.close() 70 | 71 | ''' 72 | Function that gets feed data and display it nicely 73 | Returns: N/A 74 | ''' 75 | def printFeeds(): 76 | 77 | table = texttable.Texttable() 78 | table.set_deco(texttable.Texttable.HEADER) 79 | table.set_cols_dtype(['i', # int 80 | 't',]) # text 81 | table.header(["ID", "URL"]) 82 | 83 | # Connect 84 | db.connection() 85 | 86 | for row in Feeds.select(): 87 | table.add_row([row.feedid, row.url]) 88 | 89 | # Close connection 90 | db.close() 91 | 92 | logging.info(table.draw()) 93 | 94 | ''' 95 | Function write a filter into the DB 96 | Returns: N/A 97 | ''' 98 | def writeFilter(filter_value): 99 | 100 | # Connect to DB 101 | db.connection() 102 | 103 | # Insert Data 104 | feed = Filter.create(filtervalue=filter_value) 105 | feed.save() 106 | logging.info("Succesfully added \"%s\" to the List of Filters", (filter_value)) 107 | 108 | # Close connection 109 | db.close() 110 | 111 | ''' 112 | Function that gets filter data and display it nicely 113 | Returns: N/A 114 | ''' 115 | def printFilters(): 116 | 117 | table = texttable.Texttable() 118 | table.set_deco(texttable.Texttable.HEADER) 119 | table.set_cols_dtype(['i', # int 120 | 't',]) # text 121 | table.header(["ID", "FILTER"]) 122 | 123 | # Connect 124 | db.connection() 125 | 126 | for row in Filter.select(): 127 | table.add_row([row.filterid, row.filtervalue]) 128 | 129 | # Close connection 130 | db.close() 131 | 132 | logging.info(table.draw()) 133 | 134 | ''' 135 | Function that gets feed data and display it nicely 136 | Returns: N/A 137 | ''' 138 | def printUsers(): 139 | 140 | table = texttable.Texttable() 141 | table.set_deco(texttable.Texttable.HEADER) 142 | table.set_cols_dtype(['i', # int 143 | 't', 144 | 't', 145 | 't', 146 | 't']) # text 147 | table.header(["ID", "USERNAME", "EMAIL", "KINDLE EMAIL", "SEND EBOOK"]) 148 | 149 | db.connection() 150 | for user in User.select(): 151 | if user.sendtokindle == 1: 152 | sendstatus = "YES" 153 | else: 154 | sendstatus = "NO" 155 | table.add_row([user.userid, user.name, user.email, user.kindle_mail, sendstatus]) 156 | db.close() 157 | logging.info(table.draw()) 158 | 159 | 160 | 161 | ''' 162 | Function that gets feed data and display it nicely 163 | Returns: N/A 164 | ''' 165 | def printChaptersAll(): 166 | 167 | # Make the query 168 | db.connection() 169 | chapters = Chapter.select().order_by(Chapter.chapterid) 170 | db.close() 171 | 172 | table = texttable.Texttable(max_width=120) 173 | table.set_deco(texttable.Texttable.HEADER) 174 | table.set_cols_align(["l", "l", "l", "l", "l", "l"]) 175 | table.set_cols_dtype(['i', # int 176 | 't', 177 | 't', 178 | 't', 179 | 't', 180 | 't']) # text 181 | table.header (["ID", "MANGA", "CHAPTER", "CHAPTERNAME", "RSS ORIGIN", "SEND STATUS"]) 182 | 183 | 184 | logging.info("Listing all chapters:") 185 | for row in chapters: 186 | # Rename row[8] 187 | if row.issent == 1: 188 | sendstatus = "SENT" 189 | else: 190 | sendstatus = "NOT SENT" 191 | table.add_row([row.chapterid, row.manganame, row.chapter, row.title+"\n", str(row.origin), sendstatus]) 192 | logging.info(table.draw()) 193 | 194 | 195 | 196 | ''' 197 | Function that creates user in an interactive shell 198 | Returns: N/A 199 | ''' 200 | def createUser(): 201 | 202 | # Start interactive Shell! 203 | while True: 204 | username = input("Enter User Name: ") 205 | if validators.truthy(username): 206 | break 207 | else: 208 | print("Please enter a valid username!") 209 | continue 210 | 211 | 212 | while True: 213 | email = input("Enter your Email: ") 214 | if validators.email(email): 215 | break 216 | else: 217 | print("Please enter a valid email!") 218 | continue 219 | 220 | while True: 221 | kindlemail = input("Enter your Kindle Email: ") 222 | if validators.email(kindlemail): 223 | break 224 | else: 225 | print("Please enter a valid Email!") 226 | continue 227 | 228 | while True: 229 | sendToKindle = input("Do you want to send Ebooks to this Account? [yes/no] ") 230 | if sendToKindle == "yes": 231 | break 232 | elif sendToKindle == "no": 233 | break 234 | else: 235 | print("Answer with yes or no.") 236 | continue 237 | 238 | logging.debug([username, email, kindlemail, sendToKindle]) 239 | 240 | # switch sendToKindle to 0 or 1 241 | if sendToKindle == "yes": 242 | sendToKindle = "1" 243 | else: 244 | sendToKindle = "0" 245 | 246 | # Save data now! 247 | db.connection() 248 | newuser = User.create(email=email, name=username, sendtokindle=sendToKindle, kindle_mail=kindlemail) 249 | 250 | try: 251 | newuser.save() 252 | except IntegrityError as fail: 253 | db.rollback() 254 | logging.error(fail) 255 | finally: 256 | logging.info("Succesfully added user %s!", username) 257 | db.close() 258 | 259 | ''' 260 | Switch User Config sendToKindle from True to False and False to True 261 | ''' 262 | def switchUserSend(userid): 263 | 264 | user = "" 265 | 266 | # Get User 267 | db.connection() 268 | try: 269 | user = User.get(User.userid == userid) 270 | except DoesNotExist: 271 | logging.error("User does with ID %s does not exist!", userid) 272 | 273 | if user: 274 | logging.debug("User is %s", user.name) 275 | if user.sendtokindle == 1: 276 | # Insert Data 277 | try: 278 | user.sendtokindle = 0 279 | user.save() 280 | logging.info("Disabled Ebook sending on user %s", user.name) 281 | except Exception as e: 282 | logging.debug("Failed to user status: %s", e) 283 | else: 284 | # Insert Data 285 | try: 286 | user.sendtokindle = 1 287 | logging.info("Enabling Ebook sending on user %s", user.name) 288 | user.save() 289 | except Exception as e: 290 | logging.debug("Failed to user status: %s", e) 291 | 292 | db.close() 293 | 294 | 295 | ''' 296 | Delete User! 297 | ''' 298 | def deleteUser(userid): 299 | 300 | # Get User 301 | db.connection() 302 | 303 | try: 304 | user = User.get(User.userid == userid) 305 | user.delete_instance() 306 | logging.info("Deleted user %s.", user.name) 307 | except DoesNotExist: 308 | logging.info("User with ID %s does not exist!", userid) 309 | 310 | db.close() 311 | 312 | 313 | 314 | 315 | ''' 316 | Delete Chapter! 317 | ''' 318 | def deleteChapter(chapterid): 319 | 320 | # Get Chapter 321 | db.connection() 322 | 323 | try: 324 | chapter = Chapter.get(Chapter.chapterid == chapterid) 325 | chapter.delete_instance() 326 | logging.info("Deleted Chapter %s.", chapter.title) 327 | except DoesNotExist: 328 | logging.info("Chapter with ID %s does not exist!", chapterid) 329 | 330 | db.close() 331 | 332 | 333 | ''' 334 | Delete Feed! 335 | ''' 336 | def deleteFeed(feedid): 337 | 338 | # Get Feed 339 | db.connection() 340 | 341 | try: 342 | feed = Feeds.get(Feeds.feedid == feedid) 343 | feed.delete_instance() 344 | logging.info("Deleted Feed \"%s\".", feed.url) 345 | except DoesNotExist: 346 | logging.info("Feed with ID %s does not exist!", feedid) 347 | 348 | db.close() 349 | 350 | ''' 351 | Function that prints the last 10 chapters 352 | Returns: N/A 353 | ''' 354 | def printChapters(): 355 | 356 | # Make the query 357 | db.connection() 358 | chapters = Chapter.select().order_by(-Chapter.chapterid).limit(10) 359 | db.close() 360 | 361 | table = texttable.Texttable(max_width=120) 362 | table.set_deco(texttable.Texttable.HEADER) 363 | table.set_cols_align(["l", "l", "l", "l", "l", "l"]) 364 | table.set_cols_dtype(['i', # int 365 | 't', 366 | 't', 367 | 't', 368 | 't', 369 | 't']) # text 370 | table.header(["ID", "MANGA", "CHAPTER", "CHAPTERNAME", "RSS ORIGIN", "SEND STATUS"]) 371 | 372 | logging.info("Listing the last 10 chapters:") 373 | for row in chapters: 374 | # Rename row[8] 375 | if row.issent == 1: 376 | sendstatus = "SENT" 377 | else: 378 | sendstatus = "NOT SENT" 379 | table.add_row([row.chapterid, row.manganame, row.chapter, row.title+"\n", str(row.origin), sendstatus]) 380 | logging.info(table.draw()) 381 | 382 | 383 | 384 | ''' 385 | Function that gets feed data returns it 386 | Returns: feeds 387 | ''' 388 | def getFeeds(): 389 | 390 | # Make the query 391 | db.connection() 392 | feeds = Feeds.select() 393 | db.close() 394 | 395 | return feeds 396 | 397 | 398 | 399 | ''' 400 | Function that gets chapters and returns it 401 | Returns: __chapterdata 402 | ''' 403 | def getChapters(): 404 | 405 | # Make the query 406 | db.connection() 407 | chapters = Chapter.select() 408 | 409 | return chapters 410 | 411 | 412 | ''' 413 | Function that gets chapters from IDs and returns it 414 | Returns: __chapterdata 415 | ''' 416 | def getChaptersFromID(chapterids): 417 | 418 | 419 | chapterdata = [] 420 | db.connection() 421 | 422 | for i in chapterids: 423 | # Get Data 424 | try: 425 | chapter = Chapter.select().where(Chapter.chapterid == i).get() 426 | chapterdata.append(chapter) 427 | except DoesNotExist: 428 | logging.error("Chapter with ID %s does not exist!", i) 429 | 430 | 431 | logging.debug("Passed chapters:") 432 | for i in chapterdata: 433 | logging.debug(i.title) 434 | return chapterdata 435 | 436 | 437 | 438 | ''' 439 | Function that gets chapters and returns it 440 | Returns: __userdata 441 | ''' 442 | def getUsers(): 443 | 444 | # Make the query 445 | db.connection() 446 | users = User.select() 447 | 448 | return users 449 | 450 | ''' 451 | Function that gets the current DB version for migrations 452 | Returns: $dbversion 453 | ''' 454 | def getMigrationVersion(): 455 | 456 | # Make the query 457 | db.connection() 458 | 459 | try: 460 | version = Migratehistory.select().order_by(Migratehistory.id.desc()).get().name 461 | except OperationalError as error: 462 | version = "" 463 | 464 | return version 465 | 466 | 467 | 468 | ''' 469 | Function to decide sourceparser (Mangafox or Mangastream as of now) 470 | Returns: SourceURL 471 | ''' 472 | def getSourceURL(url): 473 | SourceURL = urlparse(url).netloc 474 | 475 | return SourceURL 476 | 477 | 478 | 479 | ''' 480 | Function that gets Manga Data from Chapter URL 481 | Returns: mangadata (array) 482 | ''' 483 | def getMangaData(url, entry): 484 | 485 | # Get source of to decide which parser to use 486 | origin = getSourceURL(url) 487 | 488 | mangadata = [] 489 | # Mangastream Parser 490 | if origin == "mangastream.com" or origin == "readms.net": 491 | 492 | logging.debug("Getting Mangadata from Mangastream.com for %s", url) 493 | 494 | # Easy Stuff 495 | title = entry.title 496 | chapter_name = entry.description 497 | chapter_pubDate = entry.published 498 | 499 | # Load page once to hand it over to parser function 500 | logging.debug("Loading Page to gather data...") 501 | page = requests.get(url) 502 | 503 | # Getting the data 504 | manganame = msparser.getTitle(page) 505 | pages = msparser.getPages(page) 506 | chapter = msparser.getChapter(url) 507 | 508 | logging.debug("Mangadata succesfully loaded") 509 | 510 | mangadata = [manganame, pages, chapter, title, chapter_name, chapter_pubDate] 511 | 512 | # Mangafox Parser 513 | elif origin == "mangafox.me" or origin == "mangafox.la" or origin == "fanfox.net": 514 | logging.debug("Getting Mangadata from Mangafox. for %s", url) 515 | 516 | # Easy Stuff 517 | title = entry.title 518 | chapter_pubDate = entry.published 519 | 520 | # Load page once to hand it over to parser function 521 | logging.debug("Loading Page to gather data...") 522 | page = requests.get(url) 523 | 524 | # Getting the data 525 | manganame = mxparser.getTitle(page) 526 | pages = mxparser.getPages(page) 527 | chapter = mxparser.getChapter(url) 528 | chapter_name = mxparser.getChapterName(page) 529 | 530 | logging.debug("Mangadata succesfully loaded") 531 | 532 | mangadata = [manganame, pages, chapter, title, chapter_name, chapter_pubDate] 533 | 534 | # CDM Parser 535 | elif origin == "cdmnet.com.br": 536 | logging.debug("Getting Mangadata from CDM. for %s", url) 537 | 538 | # Easy Stuff 539 | title = entry.title 540 | chapter_pubDate = entry.published 541 | 542 | # Load page once to hand it over to parser function 543 | logging.debug("Loading Page to gather data...") 544 | page = requests.get(url) 545 | 546 | # Getting the data 547 | manganame = cdmparser.getTitle(page) 548 | pages = cdmparser.getPages(page) 549 | chapter = cdmparser.getChapter(url) 550 | chapter_name = cdmparser.getChapterName(page) 551 | 552 | logging.debug("Mangadata succesfully loaded") 553 | 554 | mangadata = [manganame, pages, chapter, title, chapter_name, chapter_pubDate] 555 | else: 556 | logging.error("Not supportet origin!") 557 | 558 | # Return mangadata 559 | return mangadata 560 | 561 | 562 | 563 | ''' 564 | Function to create folders 565 | ''' 566 | def createFolder(folder): 567 | if not os.path.exists(folder): 568 | os.makedirs(folder) 569 | logging.debug("Folder %s Created!", folder) 570 | else: 571 | logging.debug("Folder %s Exists!", folder) 572 | 573 | 574 | ''' 575 | Function that returns sanetized folder name 576 | ''' 577 | def sanetizeName(name): 578 | if ":" in name: 579 | name = name.replace(":", "_") 580 | return name 581 | elif "/" in name: 582 | name = name.replace("/", "") 583 | return name 584 | else: 585 | return name 586 | 587 | 588 | 589 | 590 | ''' 591 | Check if time is older than 24h 592 | Returns: true or false 593 | ''' 594 | def checkTime(checktime): 595 | 596 | 597 | dt = parser.parse(checktime) 598 | 599 | # Make sure checktime is TZ aware 600 | if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None: 601 | dt = dt.replace(tzinfo=datetime.timezone.utc) 602 | 603 | now = datetime.datetime.now(datetime.timezone.utc) 604 | 605 | delta = now - dt 606 | 607 | if delta.days == 0: 608 | return True 609 | else: 610 | return False 611 | 612 | 613 | 614 | ''' 615 | Verify if chapter has been downloaded 616 | Returns: true or false 617 | ''' 618 | def verifyDownload(chapter): 619 | 620 | saveloc = config["SaveLocation"] 621 | mangapages = chapter.pages 622 | mangatitle = chapter.title 623 | manganame = chapter.manganame 624 | 625 | # check if mangatitle or manganame contains ":" characters that OS can't handle as folders 626 | mangatitle = sanetizeName(mangatitle) 627 | manganame = sanetizeName(manganame) 628 | 629 | downloadfolder = str(saveloc + manganame + "/" + mangatitle + "/images") 630 | 631 | if not os.path.exists(downloadfolder): 632 | return False 633 | else: 634 | # First check checks if there is the right amount of files in the folder 635 | if len(os.listdir(downloadfolder)) != int(mangapages): 636 | shutil.rmtree(downloadfolder) 637 | return False 638 | else: 639 | # second check checks if there is an unfinished download 640 | for item in os.listdir(downloadfolder): 641 | 642 | if item.endswith(".tmp"): 643 | logging.debug("%s seems to be corrupted, removing all images for redownload"% mangatitle) 644 | shutil.rmtree(downloadfolder) 645 | return False 646 | return True 647 | 648 | 649 | 650 | ''' 651 | 652 | Init Logging! 653 | 654 | ''' 655 | def initialize_logger(output_dir, outputlevel): 656 | logger = logging.getLogger() 657 | logger.setLevel(logging.DEBUG) 658 | 659 | # create console handler and set level to info 660 | handler = logging.StreamHandler() 661 | if outputlevel == "debug": 662 | handler.setLevel(logging.DEBUG) 663 | formatter = logging.Formatter("%(levelname)s: %(message)s") 664 | else: 665 | handler.setLevel(logging.INFO) 666 | formatter = logging.Formatter("%(message)s") 667 | handler.setFormatter(formatter) 668 | logger.addHandler(handler) 669 | 670 | if not os.path.isdir(output_dir): 671 | createFolder(output_dir) 672 | 673 | 674 | # create error file handler and set level to error 675 | handler = logging.FileHandler(os.path.join(output_dir, "error.log"), encoding=None, delay="true") 676 | handler.setLevel(logging.ERROR) 677 | formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") 678 | handler.setFormatter(formatter) 679 | logger.addHandler(handler) 680 | 681 | # create debug file handler and set level to debug 682 | handler = logging.FileHandler(os.path.join(output_dir, "debug.log")) 683 | handler.setLevel(logging.DEBUG) 684 | formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") 685 | handler.setFormatter(formatter) 686 | logger.addHandler(handler) 687 | 688 | # create debug file handler and set level to debug - per run 1 file 689 | handler = logging.FileHandler(os.path.join(output_dir, "run.log"), "w") 690 | handler.setLevel(logging.DEBUG) 691 | formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") 692 | handler.setFormatter(formatter) 693 | logger.addHandler(handler) 694 | 695 | # create debug file handler and set level to info 696 | handler = logging.FileHandler(os.path.join(output_dir, "m2em.log")) 697 | handler.setLevel(logging.INFO) 698 | formatter = logging.Formatter("%(asctime)s; %(levelname)s - %(message)s") 699 | handler.setFormatter(formatter) 700 | logger.addHandler(handler) 701 | 702 | 703 | 704 | ''' 705 | Function that gets feed data and display it nicely 706 | Returns: N/A 707 | ''' 708 | def printManga(args): 709 | 710 | 711 | if args.list_manga == "all": 712 | 713 | workdata = [] 714 | # Get Data 715 | data = Chapter.select(Chapter.manganame) 716 | for i in data: 717 | workdata.append(i.manganame) 718 | tabledata = set(workdata) 719 | 720 | 721 | if tabledata: 722 | logging.info("Mangas with chapters in the database:") 723 | for i in tabledata: 724 | logging.info("* %s"% i) 725 | else: 726 | data = Chapter.select().where(Chapter.manganame == args.list_manga) 727 | 728 | if not data: 729 | logging.error("No Manga with that Name found!") 730 | else: 731 | # Reverse List to get newest first 732 | #__tabledata.reverse() 733 | 734 | table = texttable.Texttable(max_width=120) 735 | table.set_deco(texttable.Texttable.HEADER) 736 | table.set_cols_align(["l", "l", "l", "l", "l", "l"]) 737 | table.set_cols_dtype(['i', # int 738 | 't', 739 | 't', 740 | 't', 741 | 't', 742 | 't']) # text 743 | table.header (["ID", "MANGA", "CHAPTER", "CHAPTERNAME", "RSS ORIGIN", "SEND STATUS"]) 744 | 745 | 746 | logging.info("Listing all chapters of %s:"% args.list_manga) 747 | for row in data: 748 | # Rename row[8] 749 | if row.issent == 1: 750 | sendstatus = "SENT" 751 | else: 752 | sendstatus = "NOT SENT" 753 | table.add_row([row.chapterid, row.manganame, row.chapter, row.desc+"\n", str(row.origin), sendstatus]) 754 | logging.info(table.draw()) 755 | -------------------------------------------------------------------------------- /bin/Migrator.py: -------------------------------------------------------------------------------- 1 | from peewee_migrate import Router 2 | from peewee import SqliteDatabase 3 | 4 | import bin.Config as Config 5 | 6 | 7 | # Load config right at the start 8 | config = Config.load_config() 9 | 10 | db = SqliteDatabase(config['Database']) 11 | 12 | def migrate(): 13 | router = Router(db) 14 | router.run() 15 | 16 | # Create migration 17 | #router.create('initial') 18 | 19 | # Run migration/migrations 20 | #router.run('initial') 21 | 22 | # Run all unapplied migrations 23 | 24 | -------------------------------------------------------------------------------- /bin/Models.py: -------------------------------------------------------------------------------- 1 | """ Models Module """ 2 | from peewee import * 3 | import bin.Config as Config 4 | 5 | 6 | # Load config right at the start 7 | config = Config.load_config() 8 | 9 | db = SqliteDatabase(config['Database']) 10 | 11 | class ModelBase(Model): 12 | class Meta: 13 | database = db 14 | 15 | class User(ModelBase): 16 | email = TextField(null=True) 17 | name = TextField() 18 | kindle_mail = TextField(null=True) 19 | sendtokindle = IntegerField(null=True) 20 | userid = AutoField() 21 | 22 | class Chapter(ModelBase): 23 | chapter = TextField(null=True) 24 | chapterid = AutoField() 25 | date = TextField(null=True) 26 | desc = TextField(null=True) 27 | isconverted = IntegerField(null=True) 28 | ispulled = IntegerField(null=True) 29 | issent = IntegerField(null=True) 30 | manganame = TextField(null=True) 31 | origin = TextField(null=True) 32 | pages = IntegerField(null=True) 33 | title = TextField() 34 | url = TextField() 35 | 36 | class Feeds(ModelBase): 37 | feedid = AutoField() 38 | url = TextField() 39 | 40 | class Migratehistory(ModelBase): 41 | id = AutoField() 42 | name = CharField() 43 | migrated_at = DateTimeField() 44 | 45 | class Filter(ModelBase): 46 | filterid = AutoField() 47 | filtervalue = TextField() 48 | 49 | def create_tables(): 50 | db.connection() 51 | db.create_tables([User, Chapter, Feeds, Filter]) 52 | -------------------------------------------------------------------------------- /bin/RssParser.py: -------------------------------------------------------------------------------- 1 | """ RSS Parsing Module """ 2 | import logging 3 | import ssl 4 | import feedparser 5 | import re 6 | from bin.models.Manga import Manga 7 | from bin.Models import * 8 | 9 | # Remove verification need of feedparser 10 | ssl._create_default_https_context = ssl._create_unverified_context 11 | 12 | 13 | def RssParser(): 14 | """ Function that handles the coordination of rss parsing """ 15 | 16 | # Get all feeds 17 | db.connection() 18 | rssdata = Feeds.select().execute() 19 | 20 | logging.info("Checking for new Feed Data...") 21 | 22 | # loop through rss feeds 23 | for i in rssdata.iterator(): 24 | 25 | # Parse feed and check entries 26 | logging.info("Getting Feeds for %s", i.url) 27 | try: 28 | feed = feedparser.parse(str(i.url)) 29 | except Exception as identifier: 30 | logging.warning("Could not load feed: %s", identifier) 31 | 32 | for entry in feed.entries: 33 | current_manga = Manga() 34 | current_manga.load_from_feed(entry, str(feed.url)) 35 | 36 | # No need to continue if it is already saved :) 37 | if not current_manga.duplicated.exists(): 38 | 39 | # Check if any filters are set, continue as usual if not. 40 | if Filter.select().exists(): 41 | filters = Filter.select().execute() 42 | for filter_entry in filters.iterator(): 43 | 44 | # Save manga that match the filter 45 | if re.search(filter_entry.filtervalue, current_manga.title): 46 | current_manga.save() 47 | current_manga.print_manga() 48 | else: 49 | current_manga.save() 50 | current_manga.print_manga() 51 | 52 | -------------------------------------------------------------------------------- /bin/Sender.py: -------------------------------------------------------------------------------- 1 | """ Sending Module """ 2 | import logging 3 | import os 4 | import smtplib 5 | from email.mime.text import MIMEText 6 | from email.mime.multipart import MIMEMultipart 7 | from email.mime.base import MIMEBase 8 | from email.utils import formatdate, make_msgid 9 | from email.generator import Generator 10 | from email import encoders 11 | import bin.Config as Config 12 | import bin.Helper as helper 13 | 14 | try: 15 | from StringIO import StringIO 16 | except ImportError: 17 | from io import StringIO 18 | 19 | 20 | class Sender: 21 | """ Class that takes care of sending the ebooks to the users! """ 22 | 23 | def __init__(self): 24 | self.saveloc = None 25 | self.ebformat = None 26 | self.mangatitle = None 27 | self.manganame = None 28 | self.eblocation = None 29 | self.chapterdate = None 30 | self.smtpserver = None 31 | self.serverport = None 32 | self.emailadress = None 33 | self.password = None 34 | self.starttls = None 35 | self.mangaid = None 36 | self.issent = None 37 | self.chapterdate = None 38 | 39 | # Will be defined by handler 40 | self.users = None 41 | self.database = None 42 | 43 | 44 | 45 | def data_collector(self, chapter): 46 | """ Method that gathers data required for this class """ 47 | 48 | # Load config right at the start 49 | config = None 50 | if not config: 51 | config = Config.load_config() 52 | 53 | # Load configs required here 54 | self.saveloc = config["SaveLocation"] 55 | self.ebformat = config["EbookFormat"] 56 | self.smtpserver = config["SMTPServer"] 57 | self.serverport = config["ServerPort"] 58 | self.emailadress = config["EmailAddress"] 59 | self.password = config["EmailAddressPw"] 60 | self.starttls = config["ServerStartSSL"] 61 | 62 | 63 | # get relevant data of this Manga 64 | self.mangatitle = chapter.title 65 | self.chapterdate = chapter.date 66 | self.mangaid = int(chapter.chapterid) 67 | self.issent = int(chapter.issent) 68 | self.manganame = chapter.manganame 69 | 70 | # check if mangatitle or manganame contains ":" characters that OS can't handle as folders 71 | self.mangatitle = helper.sanetizeName(self.mangatitle) 72 | self.manganame = helper.sanetizeName(self.manganame) 73 | 74 | 75 | self.eblocation = str(self.saveloc + self.manganame + "/" + self.mangatitle + "/" + 76 | self.mangatitle + "." + self.ebformat.lower()) 77 | 78 | # Initialize emtpy Users table 79 | self.users = [] 80 | 81 | 82 | 83 | 84 | 85 | def send_eb(self): 86 | """ Method that sends data to the user! """ 87 | 88 | # Iterate through user 89 | for user in self.users: 90 | kindle_mail = user.kindle_mail 91 | shouldsend = user.sendtokindle 92 | user_mail = user.email 93 | 94 | # Check if user wants Mails 95 | if shouldsend == 1: 96 | 97 | logging.debug("Compiling Email for %s", user.name) 98 | 99 | 100 | # Compile Email 101 | msg = MIMEMultipart() 102 | msg['Subject'] = 'Ebook Delivery of %s' % self.mangatitle 103 | msg['Date'] = formatdate(localtime=True) 104 | msg['From'] = self.emailadress 105 | msg['To'] = kindle_mail 106 | msg['Message-ID'] = make_msgid() 107 | 108 | text = "Automatic Ebook delivery by m2em." 109 | msg.attach(MIMEText(text)) 110 | 111 | 112 | # Add Ebook as attachment 113 | ebfile = open(self.eblocation, 'rb') 114 | 115 | attachment = MIMEBase('application', 'octet-stream', 116 | name=os.path.basename(self.eblocation)) 117 | attachment.set_payload(ebfile.read()) 118 | ebfile.close() 119 | encoders.encode_base64(attachment) 120 | attachment.add_header('Content-Disposition', 'attachment', 121 | filename=os.path.basename(self.eblocation)) 122 | 123 | msg.attach(attachment) 124 | 125 | # Convert message to string 126 | sio = StringIO() 127 | gen = Generator(sio, mangle_from_=False) 128 | gen.flatten(msg) 129 | msg = sio.getvalue() 130 | 131 | # Send Email Off! 132 | # Debug Server Data 133 | logging.debug("Server: %s", self.smtpserver) 134 | logging.debug("Port: %s", self.serverport) 135 | 136 | try: 137 | server = smtplib.SMTP(self.smtpserver, self.serverport,) 138 | if self.starttls: 139 | server.starttls() 140 | server.ehlo() 141 | server.login(self.emailadress, self.password) 142 | #server.sendmail(emailadress, kindle_mail, msg.as_string()) 143 | server.sendmail(self.emailadress, kindle_mail, msg) 144 | server.close() 145 | logging.debug("Sent Ebook email to %s ", kindle_mail) 146 | self.send_confirmation(user_mail) 147 | except smtplib.SMTPException as fail: 148 | logging.debug("Could not send email! %s", fail) 149 | 150 | # Set Email as Sent 151 | helper.setIsSent(self.mangaid) 152 | logging.info("Sent %s to all requested users.", self.mangatitle) 153 | 154 | 155 | def send_confirmation(self, usermail): 156 | """ Method to send a confirmation mail to the user """ 157 | 158 | # Compile Email 159 | msg = MIMEMultipart() 160 | msg['Subject'] = 'Ebook Delivery of %s' % self.mangatitle 161 | msg['Date'] = formatdate(localtime=True) 162 | msg['From'] = self.emailadress 163 | msg['To'] = usermail 164 | msg['Message-ID'] = make_msgid() 165 | 166 | text = '%s has been delivered to your Kindle Email!' % self.mangatitle 167 | msg.attach(MIMEText(text)) 168 | 169 | # Convert message to string 170 | sio = StringIO() 171 | gen = Generator(sio, mangle_from_=False) 172 | gen.flatten(msg) 173 | msg = sio.getvalue() 174 | 175 | try: 176 | server = smtplib.SMTP(self.smtpserver, self.serverport, ) 177 | if self.starttls: 178 | server.starttls() 179 | server.ehlo() 180 | server.login(self.emailadress, self.password) 181 | server.sendmail(self.emailadress, usermail, msg) 182 | server.close() 183 | logging.debug("Sent confirmation email to %s ", usermail) 184 | except smtplib.SMTPException as fail: 185 | logging.debug("Could not send email! %s", fail) 186 | -------------------------------------------------------------------------------- /bin/SenderHandler.py: -------------------------------------------------------------------------------- 1 | """ Module to handle the sending workflow """ 2 | import logging 3 | import os 4 | import bin.Helper as helper 5 | from bin.Sender import Sender 6 | 7 | 8 | def SenderHandler(args): 9 | """ Function that handles the sending of ebooks when a loop is called """ 10 | 11 | # Get all Chapters 12 | chapters = helper.getChapters() 13 | 14 | # Load Users 15 | users = helper.getUsers() 16 | 17 | # Debug Users: 18 | logging.debug("Userlist:") 19 | for i in users: 20 | logging.debug(i.name) 21 | 22 | 23 | # Start conversion loop! 24 | for chapter in chapters.iterator(): 25 | 26 | # Initiate Sender class and fill it with data 27 | current_sender = Sender() 28 | current_sender.data_collector(chapter) 29 | current_sender.users = users 30 | 31 | # Check if ebook has been converted yet, else skip 32 | if not os.path.exists(current_sender.eblocation): 33 | logging.debug("Manga %s has not been converted yet.", current_sender.mangatitle) 34 | else: 35 | 36 | # Check if Chapter has been sent already 37 | if current_sender.issent != 0: 38 | logging.debug("%s has been sent already!", current_sender.mangatitle) 39 | else: 40 | 41 | # Check if Sender loop or Sender task is selected 42 | if not args.start: 43 | logging.info("Sending %s...", current_sender.mangatitle) 44 | current_sender.send_eb() 45 | else: 46 | 47 | # Only start run if chapter is younger than 24h 48 | if helper.checkTime(current_sender.chapterdate): 49 | logging.info("Sending %s...", current_sender.mangatitle) 50 | current_sender.send_eb() 51 | else: 52 | logging.debug("%s is older than 24h, will not be processed by daemon.", 53 | current_sender.mangatitle) 54 | 55 | 56 | def directSender(chapterids=[]): 57 | """ Function that handles the coordination of directly sending ebooks """ 58 | 59 | logging.debug("Following Chapters are directly sent:") 60 | logging.debug(chapterids) 61 | 62 | chapters = helper.getChaptersFromID(chapterids) 63 | 64 | # Load Users 65 | users = helper.getUsers() 66 | 67 | # Debug Users: 68 | logging.debug("Userlist:") 69 | for i in users: 70 | logging.debug(i.name) 71 | 72 | 73 | if not chapters: 74 | logging.error("No Chapters found with said ID!") 75 | else: 76 | # Start conversion loop! 77 | for chapter in chapters: 78 | 79 | # Initiate Sender class and fill it with data 80 | current_sender = Sender() 81 | current_sender.data_collector(chapter) 82 | current_sender.users = users 83 | 84 | # Check if ebook has been converted yet, else skip 85 | if not os.path.exists(current_sender.eblocation): 86 | logging.debug("Manga %s has not been converted yet.", current_sender.mangatitle) 87 | else: 88 | logging.info("Sending %s...", current_sender.mangatitle) 89 | current_sender.send_eb() 90 | -------------------------------------------------------------------------------- /bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/schemen/m2em/4b227d58492ddbd54aa14e42d074769c06385ae4/bin/__init__.py -------------------------------------------------------------------------------- /bin/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "v0.5.1" 2 | -------------------------------------------------------------------------------- /bin/models/Manga.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from bin.Models import * 3 | import bin.Helper as helper 4 | 5 | 6 | class Manga: 7 | 8 | def __init__(self): 9 | self.title = None 10 | self.manga_name = None 11 | self.chapter = None 12 | self.chapter_name = None 13 | self.chapter_pages = None 14 | self.chapter_pubDate = None 15 | self.chapter_link = None 16 | self.parent_feed = None 17 | self.database = None 18 | self.ispulled = None 19 | self.isconverted = None 20 | self.issent = None 21 | self.duplicated = None 22 | 23 | 24 | def load_from_feed(self, entry, parent_feed): 25 | self.chapter_link = entry.link 26 | 27 | # Check if link is already in DB to make sure only data gets downloaded that is not yet downloaded 28 | logging.debug("Checking if chapter is already saved...") 29 | db.connection() 30 | self.duplicated = Chapter.select().where(Chapter.url==self.chapter_link) 31 | 32 | if self.duplicated.exists(): 33 | logging.debug("Manga is already in Database! Skipping...") 34 | else: 35 | 36 | # Getting specific manga data 37 | logging.debug("Fetching Data from Weblink") 38 | mangadata = helper.getMangaData(self.chapter_link, entry) 39 | logging.debug("Finished Collecting Chapter Data!") 40 | 41 | self.manga_name = mangadata[0] 42 | self.title = mangadata[3] 43 | self.chapter = mangadata[2] 44 | self.chapter_name = mangadata[4] 45 | self.chapter_pages = mangadata[1] 46 | self.chapter_pubDate = mangadata[5] 47 | self.parent_feed = parent_feed 48 | 49 | # Set some defaul values 50 | self.ispulled = 0 51 | self.isconverted = 0 52 | self.issent = 0 53 | 54 | def print_manga(self): 55 | logging.debug("Title: {}".format(self.title)) 56 | logging.debug("Manga: {}".format(self.manga_name)) 57 | logging.debug("Chapter: {}".format(self.chapter)) 58 | logging.debug("Chapter Name: {}".format(self.chapter_name)) 59 | logging.debug("Chapter Pages: {}".format(self.chapter_pages)) 60 | logging.debug("Released on: {}".format(self.chapter_pubDate)) 61 | logging.debug("URL: {}".format(self.chapter_link)) 62 | logging.debug("Parent feed: {}".format(self.parent_feed)) 63 | 64 | 65 | def save(self): 66 | 67 | if self.duplicated.exists(): 68 | logging.debug("Manga is already in Database! Skipping...") 69 | else: 70 | logging.info("Saving Chapter Data for %s", self.title) 71 | db.connection() 72 | chapter = Chapter() 73 | chapter.chapter = self.chapter 74 | chapter.date = self.chapter_pubDate 75 | chapter.desc = self.chapter_name 76 | chapter.isconverted = self.isconverted 77 | chapter.ispulled = self.ispulled 78 | chapter.issent = self.issent 79 | chapter.manganame = self.manga_name 80 | chapter.origin = self.parent_feed 81 | chapter.pages = self.chapter_pages 82 | chapter.title = self.title 83 | chapter.url = self.chapter_link 84 | chapter.save() 85 | logging.info("Succesfully saved Data!") 86 | 87 | logging.debug("\n") 88 | -------------------------------------------------------------------------------- /bin/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/schemen/m2em/4b227d58492ddbd54aa14e42d074769c06385ae4/bin/models/__init__.py -------------------------------------------------------------------------------- /bin/sourceparser/Cdmnet.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ Cdmnet Parsing Module """ 3 | import logging 4 | import re 5 | from urllib.parse import urlparse 6 | import requests 7 | from bs4 import BeautifulSoup 8 | 9 | ''' 10 | 11 | CDM Parser 12 | 13 | 14 | ''' 15 | 16 | 17 | ''' 18 | get Manga Title 19 | Returns: title 20 | ''' 21 | def getTitle(page): 22 | title = None 23 | soup = BeautifulSoup(page.content, 'html.parser') 24 | 25 | #Get Manga Titel 26 | search = re.search('', str(soup)) 27 | try: 28 | title = search.group(1) 29 | except AttributeError: 30 | logging.error("No Title Fount!") 31 | 32 | return title 33 | 34 | 35 | ''' 36 | get Manga Chapter name 37 | Returns: Chapter name 38 | ''' 39 | def getChapterName(page): 40 | 41 | logging.debug("CDM has no Chapternames") 42 | chaptername = "" 43 | return chaptername 44 | 45 | 46 | ''' 47 | get Manga Pages 48 | Returns: integer pages 49 | ''' 50 | def getPages(page): 51 | soup = BeautifulSoup(page.content, 'html.parser') 52 | 53 | #Get Manga Titel 54 | search =re.search("var pages = \[.*'(.*?)',];", str(soup)) 55 | pages = search.group(1) 56 | return pages 57 | 58 | 59 | 60 | ''' 61 | get Manga chapter 62 | Returns: integer chapter 63 | ''' 64 | def getChapter(url): 65 | #soup = BeautifulSoup(page.content, 'html.parser') 66 | 67 | search = re.search('ler-online/(.*?)\Z', str(url)) 68 | chapter = search.group(1) 69 | return chapter 70 | 71 | ''' 72 | get Manga Pages URL 73 | Returns: urllist 74 | ''' 75 | def getPagesUrl(starturl,pages): 76 | pagesurllist=[] 77 | 78 | # Split URL to create list 79 | parsed = urlparse(starturl) 80 | 81 | # start url generator 82 | for page in range(pages): 83 | page = page + 1 84 | fullurl = parsed.scheme + "://" + parsed.netloc + parsed.path + "#" + str(page) 85 | pagesurllist.append(fullurl) 86 | 87 | logging.debug("All pages:") 88 | logging.debug(pagesurllist) 89 | return pagesurllist 90 | 91 | 92 | 93 | ''' 94 | get Manga Image URL 95 | Returns: urllist 96 | ''' 97 | def getImageUrl(pageurl): 98 | # Download Page 99 | page = requests.get(pageurl) 100 | soup = BeautifulSoup(page.content, 'html.parser') 101 | 102 | # Get CDN URL suffix 103 | search =re.search("var urlSulfix = '(.*?)';", str(soup)) 104 | cdnsuffix = search.group(1) 105 | 106 | # Get pagenumber 107 | var = re.search('ler-online/.*?#(.*?)\Z', str(pageurl)) 108 | pagenumber = var.group(1).zfill(2) 109 | 110 | 111 | imageurl = str(cdnsuffix + pagenumber + ".jpg") 112 | return imageurl 113 | -------------------------------------------------------------------------------- /bin/sourceparser/Mangafox.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ Mangafox Parsing Module """ 3 | import logging 4 | import re 5 | from urllib.parse import urlparse 6 | import requests 7 | from bs4 import BeautifulSoup 8 | import bin.Config as Config 9 | 10 | ''' 11 | 12 | MangaFox Parser 13 | 14 | 15 | ''' 16 | # Splash Rendering Service address 17 | config = Config.load_config() 18 | splash_server = config["SplashServer"] 19 | 20 | ''' 21 | get Manga Title 22 | Returns: title 23 | ''' 24 | def getTitle(page): 25 | title = None 26 | soup = BeautifulSoup(page.content, 'html.parser') 27 | 28 | #Get Manga Titel 29 | search = re.search('content="Read\s(.*?)\smanga online,', str(soup)) 30 | try: 31 | title = search.group(1) 32 | except AttributeError: 33 | logging.error("No Title Fount!") 34 | 35 | return title 36 | 37 | 38 | ''' 39 | get Manga Chapter name 40 | Returns: Chapter name 41 | ''' 42 | def getChapterName(page): 43 | soup = BeautifulSoup(page.content, 'html.parser') 44 | 45 | #Get Manga Titel 46 | search = re.search(': (.*?) at MangaFox', str(soup)) 47 | try: 48 | chaptername = search.group(1) 49 | except AttributeError: 50 | logging.debug("No Chapter name provided") 51 | chaptername = "" 52 | return chaptername 53 | 54 | 55 | ''' 56 | get Manga Pages 57 | Returns: integer pages 58 | ''' 59 | def getPages(page): 60 | soup = BeautifulSoup(page.content, 'html.parser') 61 | 62 | #Get Manga Titel 63 | search =re.search('var imagecount=(.*?);', str(soup)) 64 | pages = search.group(1) 65 | return pages 66 | 67 | 68 | 69 | ''' 70 | get Manga chapter 71 | Returns: integer chapter 72 | ''' 73 | def getChapter(url): 74 | #soup = BeautifulSoup(page.content, 'html.parser') 75 | 76 | #Get Manga Titel 77 | search = re.search('/c(.*?)/', str(url)) 78 | chapter = search.group(1) 79 | return chapter 80 | 81 | ''' 82 | get Manga Pages URL 83 | Returns: urllist 84 | ''' 85 | def getPagesUrl(starturl,pages): 86 | pagesurllist=[] 87 | 88 | # Split URL to create list 89 | parsed = urlparse(starturl) 90 | 91 | # get url loc 92 | urlpath = parsed.path 93 | 94 | # start url generator 95 | for page in range(pages): 96 | page = page + 1 97 | urlpathsplit = urlpath.split("/") 98 | urlpathsplit[-1] = str(page) 99 | fullurllocation = "/".join(urlpathsplit) 100 | fullurl = parsed.scheme + "://" + parsed.netloc + fullurllocation + ".html" 101 | pagesurllist.append(fullurl) 102 | 103 | logging.debug("All pages:") 104 | logging.debug(pagesurllist) 105 | return pagesurllist 106 | 107 | 108 | 109 | ''' 110 | get Manga Image URL 111 | Returns: urllist 112 | ''' 113 | def getImageUrl(pageurl): 114 | # Download Page 115 | 116 | # Splash LUA script 117 | script = """ 118 | splash.resource_timeout = 5 119 | splash:add_cookie{"IsAdult", "1", "/", domain="fanfox.net"} 120 | splash:on_request(function(request) 121 | if string.find(request.url, "tenmanga.com") ~= nil then 122 | request.abort() 123 | end 124 | end) 125 | splash:go(args.url) 126 | return splash:html() 127 | """ 128 | 129 | logging.debug("Sending rendering request to Splash") 130 | resp = requests.post(str(splash_server + "/run"), json={ 131 | 'lua_source': script, 132 | 'url': pageurl 133 | }) 134 | page = resp.content 135 | 136 | #Pass page to parser 137 | var =re.search('style=\"cursor:pointer\" src=\"//(.*?)\"', str(page)) 138 | 139 | logging.debug(var.group(1)) 140 | imageurl = "http://" + var.group(1) 141 | return imageurl 142 | -------------------------------------------------------------------------------- /bin/sourceparser/Mangastream.py: -------------------------------------------------------------------------------- 1 | """ Mangastream Parser Module """ 2 | #!/usr/bin/env python 3 | import logging 4 | import re 5 | import requests 6 | 7 | try: 8 | from urllib.parse import urlparse 9 | except ImportError: 10 | from urlparse import urlparse 11 | 12 | from bs4 import BeautifulSoup 13 | 14 | ''' 15 | 16 | MangaStream Parser 17 | 18 | 19 | ''' 20 | 21 | 22 | ''' 23 | get Manga Title 24 | Returns: title 25 | ''' 26 | def getTitle(page): 27 | soup = BeautifulSoup(page.content, 'html.parser') 28 | 29 | #Get Manga Titel 30 | var = soup.findAll("span", {"class":"hidden-xs hidden-sm"}) 31 | title = ''.join(var[0].findAll(text=True)) 32 | 33 | return title 34 | 35 | 36 | ''' 37 | get Manga Pages 38 | Returns: integer pages 39 | ''' 40 | def getPages(page): 41 | soup = BeautifulSoup(page.content, 'html.parser') 42 | 43 | #Get Manga Titel 44 | var1 = soup.body.findAll(text=re.compile("Last Page \((.*)\)")) 45 | pages = int(var1[0][11:-1]) 46 | return pages 47 | 48 | 49 | 50 | ''' 51 | get Manga chapter 52 | Returns: integer chapter 53 | ''' 54 | def getChapter(url): 55 | _var1 = urlparse(url).path 56 | _var2 = _var1.split("/") 57 | 58 | chapter = _var2[3] 59 | return chapter 60 | 61 | ''' 62 | get Manga Pages URL 63 | Returns: urllist 64 | ''' 65 | def getPagesUrl(starturl,pages): 66 | pagesurllist=[] 67 | 68 | # Split URL to create list 69 | parsed = urlparse(starturl) 70 | 71 | # get url loc 72 | urlpath = parsed.path 73 | 74 | # start url generator 75 | for page in range(pages): 76 | page = page + 1 77 | urlpathsplit = urlpath.split("/") 78 | urlpathsplit[-1] = str(page) 79 | fullurllocation = "/".join(urlpathsplit) 80 | fullurl = parsed.scheme + "://" + parsed.netloc + fullurllocation 81 | pagesurllist.append(fullurl) 82 | 83 | logging.debug("All pages:") 84 | logging.debug(pagesurllist) 85 | return pagesurllist 86 | 87 | 88 | 89 | ''' 90 | get Manga Image URL 91 | Returns: urllist 92 | ''' 93 | def getImageUrl(pageurl): 94 | # Download Page 95 | page = requests.get(pageurl) 96 | 97 | #Pass page to parser 98 | soup = BeautifulSoup(page.content, 'html.parser') 99 | var1 = soup.find(id='manga-page') 100 | 101 | imageurl = "https:" + var1['src'] 102 | return imageurl 103 | -------------------------------------------------------------------------------- /bin/sourceparser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/schemen/m2em/4b227d58492ddbd54aa14e42d074769c06385ae4/bin/sourceparser/__init__.py -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- 1 | [CONFIG] 2 | SaveLocation = data/ 3 | Database = data/main.db 4 | Sleep = 900 5 | EbookFormat = MOBI 6 | EbookProfile = KV 7 | SplashServer = http://splash:8050 8 | DisableMigrations = False 9 | # Sender Email Server Settings 10 | SMTPServer = mail.example.com 11 | ServerPort = 587 12 | EmailAddress = comic@example.com 13 | EmailAddressPw = yourpassword 14 | ServerStartSSL = True 15 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | m2em: 4 | image: schemen/m2em:latest 5 | environment: 6 | - SMTPServer=mail.example.com 7 | - EmailAddress=comic@example.com 8 | - EmailAddressPw=verysecurepassword 9 | volumes: 10 | - m2em:/usr/src/app/data 11 | 12 | splash: 13 | image: scrapinghub/splash 14 | command: --max-timeout 3600 15 | 16 | volumes: 17 | m2em: -------------------------------------------------------------------------------- /m2em.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ Main Wrapper Module """ 3 | import os 4 | import sys 5 | import logging 6 | import time 7 | import argparse 8 | import datetime 9 | import validators 10 | from bin._version import __version__ 11 | # Start of the fun! 12 | import bin.Config as Config 13 | import bin.Helper as helper 14 | import bin.RssParser as mparser 15 | import bin.DownloaderHandler as mdownloader 16 | import bin.ConverterHandler as mconverter 17 | import bin.SenderHandler as msender 18 | import bin.Migrator as migrator 19 | 20 | class M2em: 21 | """ Main Class """ 22 | 23 | def __init__(self): 24 | 25 | # Python 3 is required! 26 | if sys.version_info[0] < 3: 27 | sys.stdout.write("Sorry, requires Python 3.x, not Python 2.x\n") 28 | sys.exit(1) 29 | 30 | # Get args right at the start 31 | self.args = None 32 | if not self.args: 33 | self.read_arguments() 34 | 35 | 36 | # Load config right at the start 37 | self.config = None 38 | if not self.config: 39 | self.config = Config.load_config() 40 | logging.debug("Loaded Config:") 41 | logging.debug(self.config) 42 | 43 | # Check if Database exists, else create 44 | if not os.path.isfile(self.config["Database"]): 45 | helper.createFolder(self.config["SaveLocation"]) 46 | helper.createDB() 47 | 48 | # Check weather there are some database migrations 49 | mversion = helper.getMigrationVersion() + ".py" 50 | if self.config["DisableMigrations"] == "True": 51 | logging.debug("Migrations disabled! Current version: %s ", mversion) 52 | else: 53 | if mversion in os.listdir(os.getcwd() + "/migrations"): 54 | logging.debug("No migrations required! Current version: %s ", mversion) 55 | else: 56 | migrator.migrate() 57 | 58 | 59 | def read_arguments(self): 60 | """ function that reads all arguments """ 61 | 62 | # Get user Input 63 | parser = argparse.ArgumentParser(description='Manga to eManga - m2em') 64 | parser.add_argument("-af", "--add-feed", 65 | help="Add RSS Feed of Manga. Only Mangastream & MangaFox are supported") 66 | parser.add_argument("-au", "--add-user", help="Adds new user", 67 | action="store_true") 68 | parser.add_argument("-lm", "--list-manga", 69 | help="Lists Manga saved in database. If a Manga is passed, lists chapters to said Manga", 70 | nargs="?", const='all') 71 | parser.add_argument("-lc", "--list-chapters", help="Lists the last 10 Chapters", 72 | action="store_true") 73 | parser.add_argument("-Lc", "--list-chapters-all", help="Lists all Chapters", 74 | action="store_true") 75 | parser.add_argument("-lf", "--list-feeds", help="Lists all feeds", 76 | action="store_true") 77 | parser.add_argument("-lu", "--list-users", help="Lists all Users", 78 | action="store_true") 79 | parser.add_argument("-cd", "--create-db", help="Creates DB. Uses Configfile for Naming", 80 | action="store_true") 81 | parser.add_argument("-s", "--start", help="Starts one loop", 82 | action="store_true") 83 | parser.add_argument("--send", 84 | help="Sends Chapter directly by chapter ID. Multiple IDs can be given", 85 | default=[], nargs='*',) 86 | parser.add_argument("--convert", 87 | help="Converts Chapter directly by chapter ID. Multiple IDs can be given", 88 | default=[], nargs='*',) 89 | parser.add_argument("--download", 90 | help="Downloads Chapter directly by chapter ID. Multiple IDs can be given", 91 | default=[], nargs='*',) 92 | parser.add_argument("-p", "--process", 93 | help="Processes chapter(s) by chapter ID, Download, convert, send. Multiple IDs can be given", 94 | default=[], nargs='*',) 95 | parser.add_argument("-a", "--action", 96 | help="Start action. Options are: rssparser (collecting feed data), downloader, converter or sender ") 97 | parser.add_argument("-ss", "--switch-send", 98 | help="Pass ID of User. Switches said user Send eBook status") 99 | parser.add_argument("-dc", "--delete-chapter", 100 | help="Pass ID of Chapter. Deletes said Chapter") 101 | parser.add_argument("-du", "--delete-user", 102 | help="Pass ID of User. Deletes said User") 103 | parser.add_argument("-df", "--delete-feed", 104 | help="Pass ID of Feed. Deletes said Feed") 105 | parser.add_argument("--daemon", help="Run as daemon", 106 | action="store_true") 107 | parser.add_argument("-d", "--debug", help="Debug Mode", 108 | action="store_true") 109 | parser.add_argument('-v', '--version', 110 | action='version', version='%(prog)s ' + __version__) 111 | parser.add_argument('-f', '--filter', 112 | help='Adds a filter(python regex format), to filter the title of any manga parsed. Example: "(?i)one-punch"', 113 | nargs=1, metavar='"filter_regex"') 114 | parser.add_argument('-fl', '--filter-list', 115 | help='Lists all filters', 116 | action='store_true') 117 | 118 | self.args = parser.parse_args() 119 | 120 | # Logging 121 | if self.args.debug: 122 | outputlevel = "debug" 123 | else: 124 | outputlevel = "info" 125 | helper.initialize_logger("log/", outputlevel) 126 | 127 | 128 | # Check if Arguments are passed or not. At least one is required 129 | if self.args.action is None \ 130 | and self.args.add_feed is None \ 131 | and self.args.delete_chapter is None \ 132 | and self.args.delete_feed is None \ 133 | and self.args.delete_user is None \ 134 | and self.args.switch_send is None \ 135 | and self.args.add_user is False \ 136 | and self.args.list_manga is None \ 137 | and self.args.filter is None \ 138 | and self.args.filter_list is None \ 139 | and not any([self.args.add_user, 140 | self.args.create_db, 141 | self.args.daemon, 142 | self.args.list_chapters, 143 | self.args.list_chapters_all, 144 | self.args.list_feeds, 145 | self.args.list_users, 146 | self.args.download, 147 | self.args.convert, 148 | self.args.send, 149 | self.args.process, 150 | self.args.start,]): 151 | logging.error("At least one argument is required!") 152 | 153 | logging.debug("Passed arguments: \n %s", self.args) 154 | 155 | 156 | ''' 157 | Catch -af/--add-feed 158 | ''' 159 | def save_feed_to_db(self): 160 | logging.debug("Entered URL: %s", self.args.add_feed) 161 | if validators.url(self.args.add_feed): 162 | helper.writeFeed(self.args.add_feed) 163 | else: 164 | logging.error("You need to enter an URL!") 165 | 166 | ''' 167 | Catch -f/--filter 168 | ''' 169 | def add_filter(self): 170 | if len(self.args.filter) > 0: 171 | filter_value = self.args.filter[0] 172 | logging.debug("Entered filter: %s", filter_value) 173 | helper.writeFilter(filter_value) 174 | else: 175 | logging.error("You need to enter a filter value!") 176 | 177 | ''' 178 | Catch -s/--switch-user 179 | ''' 180 | def switch_user_status(self): 181 | logging.debug("Entered USERID: %s", self.args.switch_send) 182 | helper.switchUserSend(self.args.switch_send) 183 | 184 | 185 | ''' 186 | Delete Stuff functions 187 | ''' 188 | def delete_user(self): 189 | logging.debug("Entered USERID: %s", self.args.delete_user) 190 | helper.deleteUser(self.args.delete_user) 191 | 192 | def delete_chapter(self): 193 | logging.debug("Entered USERID: %s", self.args.delete_chapter) 194 | helper.deleteChapter(self.args.delete_chapter) 195 | 196 | def delete_feed(self): 197 | logging.debug("Entered USERID: %s", self.args.delete_feed) 198 | helper.deleteFeed(self.args.delete_feed) 199 | 200 | 201 | 202 | ''' 203 | Catch -lf/--list-feeds 204 | ''' 205 | def list_feeds(self): 206 | helper.printFeeds() 207 | 208 | ''' 209 | Catch -fl/--filter-list 210 | ''' 211 | def filter_list(self): 212 | helper.printFilters() 213 | ''' 214 | Catch -L/--list-chapters-all 215 | ''' 216 | def list_all_chapters(self): 217 | helper.printChaptersAll() 218 | 219 | 220 | ''' 221 | Catch -l/--list-chapters 222 | ''' 223 | def list_chapters(self): 224 | helper.printChapters() 225 | 226 | ''' 227 | Catch -lm/--list-manga 228 | ''' 229 | def list_manga(self): 230 | helper.printManga(self.args) 231 | 232 | 233 | 234 | ''' 235 | Catch --list-users 236 | ''' 237 | def list_users(self): 238 | helper.printUsers() 239 | 240 | ''' 241 | Catch -u/--add-user 242 | ''' 243 | def add_user(self): 244 | helper.createUser() 245 | 246 | ''' 247 | Catch -cd/--create-db 248 | ''' 249 | def create_db(self): 250 | helper.createDB() 251 | 252 | 253 | ''' 254 | Catch -a / --action 255 | ''' 256 | def start_action(self): 257 | 258 | # Start downloader 259 | if self.args.action == "downloader": 260 | logging.info("Starting downloader to get all outstanding/selected chapters") 261 | self.images_fetcher() 262 | logging.info("Finished downloading all chapters.") 263 | 264 | 265 | 266 | elif self.args.action == "rssparser": 267 | logging.info("Action '%s' is not yet implemented.", self.args.action) 268 | 269 | 270 | elif self.args.action == "converter": 271 | logging.info("Starting converter to convert all outstanding/selected chapters") 272 | self.image_converter() 273 | logging.info("Finished converting all chapters!") 274 | 275 | 276 | elif self.args.action == "sender": 277 | logging.info("Starting sender to send all outstanding/selected chapters") 278 | self.send_ebooks() 279 | logging.info("Finished sending all chapters!") 280 | 281 | else: 282 | logging.info("%s is not a valid action. Choose between 'rssparser', 'downloader', 'converter' or 'sender'", self.args.action) 283 | 284 | 285 | ''' 286 | direct callers 287 | ''' 288 | def send_chapter(self): 289 | msender.directSender(self.args.send) 290 | 291 | 292 | def convert_chapter(self): 293 | mconverter.directConverter(self.args.convert) 294 | 295 | 296 | def download_chapter(self): 297 | mdownloader.directDownloader(self.args.download) 298 | 299 | 300 | def process_chapter(self): 301 | mdownloader.directDownloader(self.args.process) 302 | mconverter.directConverter(self.args.process) 303 | msender.directSender(self.args.process) 304 | 305 | ''' 306 | This are the worker, one round 307 | ''' 308 | # Worker to get and parse rss feeds 309 | def parse_add_feeds(self): 310 | mparser.RssParser() 311 | 312 | # Worker to fetch all images 313 | def images_fetcher(self): 314 | mdownloader.downloader(self.args) 315 | 316 | # Worker to convert all downloaded chapters into ebooks 317 | def image_converter(self): 318 | mconverter.ConverterHandler(self.args) 319 | 320 | # Worker to convert all downloaded chapters into ebooks 321 | def send_ebooks(self): 322 | msender.SenderHandler(self.args) 323 | 324 | 325 | 326 | ''' 327 | Application Run & Daemon loop 328 | ''' 329 | def run(self): 330 | 331 | if self.args.add_feed: 332 | self.save_feed_to_db() 333 | return 334 | 335 | if self.args.switch_send: 336 | self.switch_user_status() 337 | return 338 | 339 | if self.args.list_feeds: 340 | self.list_feeds() 341 | return 342 | 343 | if self.args.list_chapters_all: 344 | self.list_all_chapters() 345 | return 346 | 347 | if self.args.list_chapters: 348 | self.list_chapters() 349 | return 350 | 351 | if self.args.list_manga: 352 | self.list_manga() 353 | return 354 | 355 | if self.args.add_user: 356 | self.add_user() 357 | return 358 | 359 | if self.args.list_users: 360 | self.list_users() 361 | return 362 | 363 | if self.args.delete_user: 364 | self.delete_user() 365 | return 366 | 367 | 368 | if self.args.delete_chapter: 369 | self.delete_chapter() 370 | return 371 | 372 | 373 | if self.args.delete_feed: 374 | self.delete_feed() 375 | return 376 | 377 | if self.args.create_db: 378 | self.create_db() 379 | return 380 | 381 | if self.args.action: 382 | self.start_action() 383 | return 384 | 385 | if self.args.send: 386 | self.send_chapter() 387 | return 388 | 389 | if self.args.download: 390 | self.download_chapter() 391 | return 392 | 393 | if self.args.convert: 394 | self.convert_chapter() 395 | return 396 | 397 | if self.args.process: 398 | self.process_chapter() 399 | return 400 | 401 | if self.args.filter: 402 | self.add_filter() 403 | return 404 | 405 | if self.args.filter_list: 406 | self.filter_list() 407 | return 408 | 409 | # Mainloop 410 | if self.args.start: 411 | daemon = True 412 | while daemon: 413 | if self.args.daemon: 414 | logging.info("Don't forget that the daemon only handles data younger than 24h Hours!") 415 | else: 416 | daemon = False 417 | 418 | logging.info("#########################") 419 | logging.info("Starting Loop at %s", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) 420 | 421 | 422 | logging.info("Starting RSS Data Fetcher!") 423 | self.parse_add_feeds() 424 | logging.info("Finished Loading RSS Data") 425 | 426 | logging.info("Starting all outstanding Chapter Downloads!") 427 | self.images_fetcher() 428 | logging.info("Finished all outstanding Chapter Downloads") 429 | 430 | logging.info("Starting recursive image conversion!") 431 | self.image_converter() 432 | logging.info("Finished recursive image conversion!") 433 | 434 | logging.info("Starting to send all ebooks!") 435 | self.send_ebooks() 436 | logging.info("Finished sending ebooks!") 437 | 438 | if daemon: 439 | logging.info("Sleeping for %s seconds...\n", (self.config["Sleep"])) 440 | time.sleep(int(self.config["Sleep"])) 441 | 442 | # Execute Main 443 | if __name__ == '__main__': 444 | me = M2em() 445 | me.run() 446 | -------------------------------------------------------------------------------- /migrations/001_initial.py: -------------------------------------------------------------------------------- 1 | """Peewee migrations -- 001_initial.py. 2 | 3 | Some examples (model - class or model name):: 4 | 5 | > Model = migrator.orm['model_name'] # Return model in current state by name 6 | 7 | > migrator.sql(sql) # Run custom SQL 8 | > migrator.python(func, *args, **kwargs) # Run python code 9 | > migrator.create_model(Model) # Create a model (could be used as decorator) 10 | > migrator.remove_model(model, cascade=True) # Remove a model 11 | > migrator.add_fields(model, **fields) # Add fields to a model 12 | > migrator.change_fields(model, **fields) # Change fields 13 | > migrator.remove_fields(model, *field_names, cascade=True) 14 | > migrator.rename_field(model, old_field_name, new_field_name) 15 | > migrator.rename_table(model, new_table_name) 16 | > migrator.add_index(model, *col_names, unique=False) 17 | > migrator.drop_index(model, *col_names) 18 | > migrator.add_not_null(model, *field_names) 19 | > migrator.drop_not_null(model, *field_names) 20 | > migrator.add_default(model, field_name, default) 21 | 22 | """ 23 | 24 | import datetime as dt 25 | import peewee as pw 26 | from decimal import ROUND_HALF_EVEN 27 | 28 | try: 29 | import playhouse.postgres_ext as pw_pext 30 | except ImportError: 31 | pass 32 | 33 | SQL = pw.SQL 34 | 35 | 36 | def migrate(migrator, database, fake=False, **kwargs): 37 | """Write your migrations here.""" 38 | 39 | 40 | 41 | def rollback(migrator, database, fake=False, **kwargs): 42 | """Write your rollback migrations here.""" 43 | 44 | -------------------------------------------------------------------------------- /migrations/002_testmigration.py: -------------------------------------------------------------------------------- 1 | """Peewee migrations -- 002_testmigration.py. 2 | 3 | Some examples (model - class or model name):: 4 | 5 | > Model = migrator.orm['model_name'] # Return model in current state by name 6 | 7 | > migrator.sql(sql) # Run custom SQL 8 | > migrator.python(func, *args, **kwargs) # Run python code 9 | > migrator.create_model(Model) # Create a model (could be used as decorator) 10 | > migrator.remove_model(model, cascade=True) # Remove a model 11 | > migrator.add_fields(model, **fields) # Add fields to a model 12 | > migrator.change_fields(model, **fields) # Change fields 13 | > migrator.remove_fields(model, *field_names, cascade=True) 14 | > migrator.rename_field(model, old_field_name, new_field_name) 15 | > migrator.rename_table(model, new_table_name) 16 | > migrator.add_index(model, *col_names, unique=False) 17 | > migrator.drop_index(model, *col_names) 18 | > migrator.add_not_null(model, *field_names) 19 | > migrator.drop_not_null(model, *field_names) 20 | > migrator.add_default(model, field_name, default) 21 | 22 | """ 23 | 24 | import datetime as dt 25 | import peewee as pw 26 | from decimal import ROUND_HALF_EVEN 27 | 28 | try: 29 | import playhouse.postgres_ext as pw_pext 30 | except ImportError: 31 | pass 32 | 33 | SQL = pw.SQL 34 | 35 | 36 | def migrate(migrator, database, fake=False, **kwargs): 37 | """Write your migrations here.""" 38 | migrator.python(testmigration) 39 | 40 | 41 | 42 | def rollback(migrator, database, fake=False, **kwargs): 43 | """Write your rollback migrations here.""" 44 | migrator.python(testmigration_rollback) 45 | 46 | 47 | def testmigration(): 48 | print("Rolling the test migration") 49 | 50 | def testmigration_rollback(): 51 | print("Reverting the test migration") 52 | -------------------------------------------------------------------------------- /migrations/003_filters.py: -------------------------------------------------------------------------------- 1 | """Peewee migrations -- 003_filters.py. 2 | 3 | Some examples (model - class or model name):: 4 | 5 | > Model = migrator.orm['model_name'] # Return model in current state by name 6 | 7 | > migrator.sql(sql) # Run custom SQL 8 | > migrator.python(func, *args, **kwargs) # Run python code 9 | > migrator.create_model(Model) # Create a model (could be used as decorator) 10 | > migrator.remove_model(model, cascade=True) # Remove a model 11 | > migrator.add_fields(model, **fields) # Add fields to a model 12 | > migrator.change_fields(model, **fields) # Change fields 13 | > migrator.remove_fields(model, *field_names, cascade=True) 14 | > migrator.rename_field(model, old_field_name, new_field_name) 15 | > migrator.rename_table(model, new_table_name) 16 | > migrator.add_index(model, *col_names, unique=False) 17 | > migrator.drop_index(model, *col_names) 18 | > migrator.add_not_null(model, *field_names) 19 | > migrator.drop_not_null(model, *field_names) 20 | > migrator.add_default(model, field_name, default) 21 | 22 | """ 23 | 24 | import datetime as dt 25 | from bin.Models import * 26 | import peewee as pw 27 | from decimal import ROUND_HALF_EVEN 28 | 29 | try: 30 | import playhouse.postgres_ext as pw_pext 31 | except ImportError: 32 | pass 33 | 34 | SQL = pw.SQL 35 | 36 | 37 | def migrate(migrator, database, fake=False, **kwargs): 38 | """Write your migrations here.""" 39 | 40 | @migrator.create_model 41 | class Filter(ModelBase): 42 | filterid = AutoField() 43 | filtervalue = TextField() 44 | 45 | 46 | 47 | def rollback(migrator, database, fake=False, **kwargs): 48 | """Write your rollback migrations here.""" 49 | 50 | @migrator.remove_model 51 | class Filter(ModelBase): 52 | filterid = AutoField() 53 | filtervalue = TextField() -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | validators==0.12.0 2 | texttable==1.1.1 3 | requests==2.20.0 4 | bs4==0.0.1 5 | urllib3==1.26.5 6 | feedparser==5.2.1 7 | KindleComicConverter==5.4.3 8 | peewee==3.7.0 9 | peewee-migrate==1.1.6 10 | python-dateutil==2.7.5 11 | --------------------------------------------------------------------------------