├── .gitignore ├── .gitmodules ├── README.md ├── animeDL.nim ├── animeDL.nimble ├── build_linux.nims ├── build_windows.nims └── scripts ├── animeID.nims └── royalroad.nims /.gitignore: -------------------------------------------------------------------------------- 1 | # Dot folders 2 | .git/ 3 | .github/ 4 | .idea/ 5 | 6 | # anime-dl binary 7 | animeDL 8 | 9 | # Downloads from anime-dl 10 | *.epub 11 | *.mp4 12 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "libs/nim-HLSManager"] 2 | path = libs/nim-HLSManager 3 | url = https://github.com/ShujianDou/nim-HLSManager 4 | [submodule "libs/ADLCore"] 5 | path = libs/ADLCore 6 | url = https://github.com/vrienstudios/ADLCore 7 | [submodule "libs/nim-epub"] 8 | path = libs/nim-epub 9 | url = https://github.com/ShujianDou/nim-epub 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # anime-dl 2 | > The front-end for the anime-dl project 3 | 4 | # NOTICE 5 | Embtaku is down & Only certain sites work (see status) 6 | 7 | > It may take awhile before I implement a new anime or movie site, since I don't consume this content that much anymore. 8 | 9 | >Every now and then I will come back and continue working on adding a new anime site; to see current progress, look at issues. 10 | 11 | >I forgot this, when I redesigned the project, but I will re-add the manga downloader over the weekend-- 12 | 13 | ## Table of Contents 14 | - [Site Status](#site-status) 15 | - [Installation](#installation) 16 | - [Building](#building) 17 | - [Usage](#usage) 18 | 19 | 20 |
The backing library: [ADLCore](https://github.com/vrienstudios/ADLCore) 21 | 22 |
Have any ideas or an issue? Feel free to create an issue or talk to us in the [Discord](https://discord.gg/WYTxbt2) 23 | 24 | > ## Site Status 25 | 26 | | SITE | Search | Download | 27 | |----------------------|----------|----------| 28 | | [Novel] | | | 29 | | NovelHall.com | YES | YES | 30 | | Shuba | WIP | WIP | 31 | | RoyalRoad.com | NO | YES | 32 | | [Manga] | | | 33 | | MangaKakalot.com | WIP | WIP | 34 | | [Anime] | X | X | 35 | | booster | WIP | WIP | 36 | | animeid.live (Espanol) | NO | YES | 37 | | [NSFW] | | | 38 | | HAnime.tv | YES | YES | 39 | 40 | ## Installation 41 | Download the latest release from the [releases page](https://github.com/vrienstudios/anime-dl/releases) 42 | 43 | ## Building 44 | Requirements: 45 | * [nim >= 1.6.6](https://nim-lang.org/install.html) 46 | * nimble (should come preinstalled with nim) 47 | * [git](https://git-scm.com/) 48 | * OpenSSL 49 | * Linux: 50 | * (Arch-based) ``sudo pacman -S openssl`` 51 | * (Debian-based) ``sudo apt install openssl`` 52 | * Windows (If you don't want to use the ones we provide): 53 | * https://wiki.openssl.org/index.php/Binaries 54 | 55 |
1. Clone the repo
56 | ``` 57 | git clone https://github.com/vrienstudios/anime-dl.git && cd anime-dl 58 | ``` 59 |
2. Install required nim modules:
60 | > Note: It is recommended you to check out the dependencies in the nimble file before doing this. 61 | ``` 62 | nimble installdeps 63 | ``` 64 |
3. Build with these commands:
65 | ```nimble build -d:ssl --threads:on``` 66 | 67 | ## Usage 68 | There are two ways to use the program-- 69 | 70 | You can simply execute the executable and follow the prompts, or you can follow the instructions below for simpler usage. 71 | > Note: The documentation on the program arguments are subject to change and are not encompassing. 72 | 73 | ``` 74 | Help: 75 | ./animeDL selector options 76 | ./animeDL ani -d -c HAnime -res x720 -url url 77 | ./animeDL ani -d -c vidstreamAni -res 1920x1080 -url url 78 | (Selectors) 79 | ani (Denominates video) 80 | nvl (Denominates text) 81 | mng (Denominates pictures) 82 | (Options) 83 | -d (specifies to download) 84 | -lim num:num (limit the amount of episodes/chapters) 85 | -c name (Set a custom downloader, useful for scripts) 86 | -dblk (specify to download more than one episode) 87 | -res wxh (Can be buggy at times) 88 | ``` 89 | 90 | 91 | 92 | https://user-images.githubusercontent.com/13159328/185725402-1425974b-2b15-4a79-99b4-ed4634f67c23.mp4 93 | 94 | -------------------------------------------------------------------------------- /animeDL.nim: -------------------------------------------------------------------------------- 1 | import system, strutils, httpclient, terminal, os, osproc, xmltree, times, uri 2 | import ADLCore 3 | 4 | proc getOccupiedMB(): string = 5 | return $(getOccupiedMem() / 1000000) 6 | proc getUserInput(): string = 7 | styledWrite(stdout, fgGreen, ">") 8 | return readLine(stdin) 9 | proc awaitInput() = 10 | styledWriteLine(stdout, fgGreen, "To continue, hit enter.") 11 | discard getUserInput() 12 | return 13 | proc printErr(err: string) = 14 | styledWriteLine(stdout, fgRed, err) 15 | proc printHelp() = 16 | styledWriteLine(stdout, fgGreen, "\r\n~ HELP ~") 17 | styledWriteLine(stdout, fgWhite, "down: Download\r\n down hostName|url searchTerm|url\r\n Example: down www.novelhall.com DairyCow\r\n Example 2: down https://www.novelhall.com/novels/dairycow\r\nsearch: meta (returns metadata,url,VidSrcUrl)\r\n meta host|url searchTerm|Url\r\n") 18 | return 19 | proc printOptions() = 20 | var idx, lineTrack: int16 = 0 21 | while idx < siteList.len: 22 | inc idx 23 | if idx mod 3 == 0: 24 | inc lineTrack 25 | styledWrite(stdout, fgWhite, " ", $idx, "):", siteList[idx - 1].identifier, "\r\n".repeat(lineTrack)) 26 | proc extractMetaContent(ctx: var DownloaderContext) = 27 | return 28 | proc promptSelectionChoice(ctx: var DownloaderContext) = 29 | return 30 | proc promptResolutionChoice(ctx: var DownloaderContext) = 31 | echo "Select a resolution!" 32 | for i in ctx.chapter.mainStream.subStreams: 33 | echo "$#) $# | $#" % [i.id, i.resolution, i.uri] 34 | let 35 | usr = getUserInput() 36 | ctx.selectResolution(usr) 37 | return 38 | proc downloadVideo(ctx: var DownloaderContext) = 39 | # Set Specific Video 40 | var i: int = 0 41 | while i < ctx.section.parts.len: 42 | var chap = ctx.section.parts[i] 43 | echo ctx.section.mdat.name 44 | echo chap.metadata.name 45 | if ctx.section.mdat.name != chap.metadata.name: 46 | inc i 47 | continue 48 | ctx.section.index = i 49 | break 50 | if ctx.chapter.selStream.len == 0: 51 | ctx.promptResolutionChoice() 52 | assert ctx.chapter.selStream.len != 0 53 | let startPath = "./" & ctx.chapter.metadata.name 54 | var idx: int = 0 55 | if fileExists(startPath & ".track"): 56 | let textAmnt: seq[string] = readAll(open(startPath & ".track", fmRead)).split('\n') 57 | idx = textAmnt.len - 2 58 | if idx < 0: idx = 0 59 | var 60 | file: File = open(startPath & ".ts", fmWrite) 61 | track: File = open(startPath & ".track", fmWrite) 62 | cursorDown 1 63 | for data in ctx.walkVideoContent(): 64 | eraseLine() 65 | styledWrite(stdout, "Got part ", fgGreen, $ctx.chapter.streamIndex, fgWhite, " out of ", fgGreen, $ctx.chapter.selStream.len) 66 | file.write data.text 67 | track.writeLine "g" 68 | track.flushFile() 69 | file.flushFile() 70 | file.close() 71 | track.close() 72 | removeFile(startPath & ".track") 73 | # TODO: Do the same for Windows 74 | when defined linux: 75 | let ffmpeg: string = execCmdEx("ls /bin | grep -x ffmpeg").output 76 | styledWriteLine(stdout, fgGreen, "Fixing HLS Container") 77 | if ffmpeg.len == 0 or ffmpeg[0..5] != "ffmpeg": return 78 | echo execCmdEx("ffmpeg -i \"$#\" -c copy \"$#\"" % [startPath & ".ts", startPath & ".mp4"], workingDir = getAppDir()).output 79 | if fileExists(startPath & ".mp4") and getFileSize(startPath & ".mp4") > 10: 80 | removeFile(startPath & ".ts") 81 | return 82 | proc downloadContent(ctx: var DownloaderContext) = 83 | if ctx.sections.len > 0 and ctx.section.sResult: 84 | ctx.promptSelectionChoice() 85 | assert ctx.setMetadata() 86 | echo $ctx 87 | awaitInput() 88 | assert ctx.setParts() 89 | if ctx.doPrep(): 90 | ctx.downloadVideo() 91 | return 92 | var epub: Epub3 = setupEpub(ctx.sections[0].mdat) 93 | ctx.buildCoverAndDefaultPage(epub) 94 | for section in ctx.walkSections(): 95 | if section.parts.len == 0: continue 96 | cursorDown 1 97 | for chapter in ctx.walkChapters(): 98 | eraseLine() 99 | styledWrite(stdout, fgWhite, "Got chapter ", chapter.metadata.name) 100 | if epub.isIn(chapter.metadata.name): continue 101 | assert ctx.setContent() 102 | epub += (chapter.metadata.name, chapter.contentSeq) 103 | chapter.contentSeq = @[] 104 | epub.write() 105 | return 106 | proc searchContent(ctx: var DownloaderContext, term: string) = 107 | assert ctx.setSearch(term) 108 | return 109 | proc processInput(input: string, path: string = "", take: seq[int] = @[]) = 110 | echo input 111 | let splitTerms = input.split(' ') 112 | if $input[0..3] == "help": 113 | printHelp() 114 | return 115 | if splitTerms.len < 2: 116 | printErr("No args?") 117 | return 118 | var ctx: DownloaderContext = generateContext(splitTerms[1]) 119 | case splitTerms[0]: 120 | of "down": 121 | if splitTerms.len == 2: 122 | if not splitTerms[1].isUrl(): 123 | printErr("arg (1) is not a URL and no other args") 124 | return 125 | elif splitTerms.len == 3: 126 | if not splitTerms[2].isUrl(): 127 | searchContent(ctx, splitTerms[2]) 128 | downloadContent(ctx) 129 | of "meta": 130 | if splitTerms.len > 2: 131 | if splitTerms[2].isUrl(): 132 | extractMetaContent(ctx) 133 | return 134 | searchContent(ctx, splitTerms[2]) 135 | extractMetaContent(ctx) 136 | else: 137 | printErr("arg error") 138 | return 139 | proc beginInteraction(defaultInput: string = "") = 140 | try: 141 | var input = defaultInput 142 | styledWriteLine(stdout, fgGreen, " ~ anime-dl ~ ") 143 | styledWriteLine(stdout, fgWhite, "Anime - Novel - Manga") 144 | styledWriteLine(stdout, fgWhite, " (Hint) Type \"help\"") 145 | if input == "": 146 | input = getUserInput() 147 | processInput(input) 148 | awaitInput() 149 | except: 150 | styledWriteLine(stdout, fgRed, "there was an error") 151 | 152 | let ps: int = paramCount() + 1 153 | var pidx: int = 0 154 | 155 | if ps > 0: 156 | var 157 | url: string 158 | dPath: string 159 | take: seq[int] = @[] 160 | metaOnly: bool 161 | while pidx < ps: 162 | var cstr = paramStr(pidx) 163 | if not cstr.isUrl(): 164 | case cstr: 165 | of "-l": 166 | inc pidx 167 | let limit = paramStr(pidx).split('-') 168 | for l in limit: 169 | take.add parseInt(l) 170 | continue 171 | of "-m": 172 | metaOnly = true 173 | inc pidx 174 | continue 175 | else: 176 | inc pidx 177 | continue 178 | url = cstr 179 | inc pidx 180 | if url == "": 181 | #Test ui mode 182 | beginInteraction() 183 | quit(0) 184 | processInput(if metaOnly: "meta " else: "down " & url, dPath, take) 185 | quit(0) 186 | while true: 187 | if isAdmin(): 188 | echo "Do not run this as sudoer or admin." 189 | quit(-1) 190 | beginInteraction() -------------------------------------------------------------------------------- /animeDL.nimble: -------------------------------------------------------------------------------- 1 | # Package 2 | 3 | version = "3.1.2" 4 | author = "VrienStudio" 5 | description = "Downloader/Scraper for anime" 6 | license = "GPLv3" 7 | srcDir = "src" 8 | bin = @["animeDL"] 9 | 10 | task installdeps, "Installs anime-dl dependencies from github": 11 | echo("Cloning dependencies from git...") 12 | createDir "libs/" 13 | withDir "libs/": 14 | exec("git clone https://github.com/ShujianDou/nim-HLSManager") 15 | exec("git clone https://github.com/ShujianDou/nim-epub") 16 | exec("git clone https://github.com/vrienstudios/ADLCore.git") 17 | echo("Installing dependencies...") 18 | # It is important for the dependencies to be installed in this order. 19 | exec "nimble install compiler" 20 | exec "nimble install halonium" 21 | exec "nimble install nimcrypto" 22 | exec "nimble install nimscripter" 23 | exec "nimble install zippy" 24 | exec "nimble install checksums" 25 | withDir "libs/nim-HLSManager": 26 | exec("nimble install -Y") 27 | withDir "libs/nim-epub": 28 | exec("nimble install -Y") 29 | withDir "libs/ADLCore": 30 | exec("nimble install -Y") 31 | 32 | # Dependencies 33 | 34 | requires "nim >= 1.6.6" 35 | requires "ADLCore == 0.2.0" 36 | requires "EPUB == 0.3.0" 37 | requires "compiler" 38 | -------------------------------------------------------------------------------- /build_linux.nims: -------------------------------------------------------------------------------- 1 | #[ 2 | cd ../libs/ADLCore 3 | rm -rf /home/shu/.nimble/pkgs2/ADLCore-0* 4 | nimble install -y 5 | cd ../.. 6 | nim c -d:ssl --threads:on -f ./animeDL.nim 7 | ]# 8 | 9 | if defined(Linux): 10 | withDir "libs/ADLCore": 11 | exec "rm -rf $HOME/.nimble/pkgs/ADLCore-0*" 12 | exec "nimble install -y" 13 | 14 | exec "nim c -d:ssl --threads:on -f ./animeDL.nim" 15 | exec "mkdir -p ./bin/amd64" 16 | mvFile "./animeDL", "./bin/amd64/animeDL" 17 | else: 18 | echo "This can not be ran on Windows, yet" -------------------------------------------------------------------------------- /build_windows.nims: -------------------------------------------------------------------------------- 1 | if defined(Linux): 2 | withDir "libs/ADLCore": 3 | exec "rm -rf $HOME/.nimble/pkgs/ADLCore-0*" 4 | exec "nimble install -y" 5 | exec "nim c -d:ssl -d:mingw --threads:on -f ./animeDL.nim" 6 | exec "mkdir -p ./bin/win64/" 7 | mvFile "./animeDL.exe", "./bin/win64/animeDL.exe" 8 | else: 9 | echo "build normally, don't run this if not on linux" -------------------------------------------------------------------------------- /scripts/animeID.nims: -------------------------------------------------------------------------------- 1 | # name:animeID (Spanish) 2 | # scraperType:ani 3 | # version:0.0.1 4 | # siteUri:https://animeid.live/ 5 | # hosts:animeid.live 6 | 7 | var defaultHeaders: seq[tuple[key: string, value: string]] = @[ 8 | ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0"), 9 | ("Accept-Encoding", "identity"), ("Accept", "*/*")] 10 | 11 | var 12 | page: XmlNode 13 | scriptID: int 14 | baseUri: string = "https://animeid.live/" 15 | defaultPage: string = "" 16 | currPage: string = "" 17 | 18 | proc SetID*(id: int) = 19 | scriptID = id 20 | proc SetDefaultPage*(page: string) = 21 | defaultPage = page 22 | proc AddHeader*(k: string, v: string) = 23 | defaultHeaders.add((k, v)) 24 | proc getHeaders*(): seq[tuple[key: string, value: string]] = 25 | return defaultHeaders 26 | proc procHttpTest*(): string = 27 | return processHttpRequest("newtab", scriptID, defaultHeaders, true) 28 | proc GetHLSStream*(): HLSStream = 29 | var 30 | ajaxUri = baseUri & "ajax.php?" & (parseHtml(SeekNode($page, "
")).child("iframe").attr("src").split("?")[^1]) & "&refer=none" 31 | jsonRespObj = parseJson(processHttpRequest(ajaxUri, scriptID, defaultHeaders, false)) 32 | m3Uri = jsonRespObj["source"].getElems()[0]["file"].getStr() 33 | hls = parseManifestInterp(processHttpRequest(m3Uri, scriptID, defaultHeaders, false)) 34 | hls.baseUri = join(m3Uri.split('/')[0..^2], "/") & "/" 35 | return hls 36 | proc GetMetaData*(): MetaData = 37 | var mdata = MetaData() 38 | if currPage != defaultPage: 39 | page = parseHtml(processHttpRequest(defaultPage, scriptID, defaultHeaders, false)) 40 | currPage = defaultPage 41 | let 42 | videoInfoLeft: XmlNode = parseHtml(SeekNode($page, "
")) 43 | videoDetails = parseHtml(SeekNode($videoInfoLeft, "
")) 44 | assert videoInfoLeft != nil 45 | mdata.name = videoInfoLeft.child("h1").innerText 46 | mdata.series = sanitizeString(videoDetails.child("span").innerText) 47 | mdata.description = sanitizeString(videoDetails.child("div").child("div").innerText) 48 | return mdata 49 | proc GetResolutions*(mainStream: HLSStream): seq[MediaStreamTuple] = 50 | var medStream: seq[MediaStreamTuple] = @[] 51 | var index: int = 0 52 | for segment in mainStream.parts: 53 | if segment.header == "#EXT-X-MEDIA:": 54 | var id: string 55 | var language: string 56 | var uri: string 57 | for param in segment.values: 58 | case param.key: 59 | of "GROUP-ID": id = param.value 60 | of "LANGUAGE": language = param.value 61 | of "URI": 62 | uri = param.value 63 | else: discard 64 | medStream.add((id: id, resolution: "", uri: uri, language: language, isAudio: true, bandWidth: "")) 65 | elif segment.header == "#EXT-X-STREAM-INF:": 66 | var bandwidth: string 67 | var resolution: string 68 | var id: string 69 | var uri: string 70 | for param in segment.values: 71 | case param.key: 72 | of "BANDWIDTH": bandwidth = param.value 73 | of "RESOLUTION": resolution = param.value 74 | of "AUDIO": id = param.value 75 | else: discard 76 | uri = indexStreamHead(mainStream.parts[index + 1], "URI") 77 | medStream.add((id: id, resolution: resolution, uri: mainStream.baseUri & uri, language: "", isAudio: false, bandWidth: bandwidth)) 78 | inc index 79 | return medStream 80 | proc Search*(str: string): seq[MetaData] = 81 | let content = processHttpRequest(baseUri & "ajax-search.html?keyword=" & str & "&id=-1", scriptID, defaultHeaders, false) 82 | let json = parseJson(content) 83 | var results: seq[MetaData] = @[] 84 | page = parseHtml(json["content"].getStr()) 85 | currPage = baseUri 86 | for a in page.findAll("a"): 87 | var data = MetaData() 88 | data.name = a.innerText 89 | data.uri = baseUri & a.attr("href") 90 | results.add(data) 91 | return results 92 | proc SetResolution*(mBase: tuple[s1: MediaStreamTuple, s2: string]): tuple[video: seq[string], audio: seq[string]] = 93 | var vManifest = parseManifestInterp(processHttpRequest(mBase.s1.uri, scriptID, defaultHeaders, false), mBase.s2) 94 | var vSeq: seq[string] = @[] 95 | for part in vManifest.parts: 96 | if part.header == "URI": 97 | vSeq.add(part.values[0].value) 98 | return (vSeq, @[]) 99 | proc GetNextVideoPart*(idx: int, videoStream: seq[string]): string = 100 | return processHttpRequest(videoStream[idx], scriptID, defaultHeaders, false) 101 | -------------------------------------------------------------------------------- /scripts/royalroad.nims: -------------------------------------------------------------------------------- 1 | # name:RoyalRoad 2 | # scraperType:nvl 3 | # version:0.0.1 4 | # siteUri:https://www.royalroad.com 5 | # hosts:royalroad.com 6 | 7 | var defaultHeaders: seq[tuple[key: string, value: string]] = @[ 8 | ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0"), 9 | ("Accept-Encoding", "identity"), ("Accept", "*/*")] 10 | 11 | var page: XmlNode 12 | var currPage: string 13 | var defaultPage: string 14 | var scriptID: int 15 | 16 | proc SetID*(id: int) = 17 | scriptID = id 18 | proc SetDefaultPage*(page: string) = 19 | defaultPage = page 20 | 21 | proc AddHeader*(k: string, v: string) = 22 | defaultHeaders.add((k, v)) 23 | proc getHeaders*(): seq[tuple[key: string, value: string]] = 24 | return defaultHeaders 25 | proc procHttpTest*(): string = 26 | return processHttpRequest("newtab", scriptID, defaultHeaders, true) 27 | 28 | proc GetChapterSequence*(): seq[Chapter] = 29 | # Assume we are on correct page, and that page var was set. 30 | var tableBody = parseHtml(SeekNode($page, "")) 31 | var chapters: seq[Chapter] = @[] 32 | for tableItem in tableBody.items: 33 | if tableItem.kind != xnElement: 34 | continue 35 | let chapterRowInfo = tableItem.child("td").child("a") 36 | var newChapter: Chapter = Chapter() 37 | newChapter.name = sanitizeString(chapterRowInfo.innerText) 38 | newChapter.uri = "https://www.royalroad.com" & chapterRowInfo.attr("href") 39 | chapters.add newChapter 40 | return chapters 41 | 42 | proc GetNodes*(chapter: Chapter): seq[TiNode] = 43 | var tinodes: seq[TiNode] = @[] 44 | let htmlData: string = processHttpRequest(chapter.uri, scriptID, defaultHeaders, false) 45 | let chapterNode: XmlNode = parseHtml(SeekNode(htmlData, "
")) 46 | # When it becomes available, search for italics and strong identifiers. 47 | for p in chapterNode.items: 48 | if p.kind == xnElement: 49 | tinodes.add TiNode(kind: NodeKind.paragraph, text: p.innerText) 50 | #Nicht dein spiel 51 | return tinodes 52 | proc GetMetaData*(): MetaData = 53 | currPage = defaultPage 54 | let pageContent = processHttpRequest(defaultPage, scriptID, defaultHeaders, false) 55 | page = parseHtml(pageContent) 56 | let 57 | ovNode: XmlNode = parseHtml(SeekNode(pageContent, "
")).child("div") 58 | coverUri = parseHtml(SeekNode($ovNode, "
")).child("img").attr("src") 59 | authorTitleNodeCombo = parseHtml(SeekNode($ovNode, "
")).child("div") 60 | title = parseHtml(SeekNode($authorTitleNodeCombo, "

")).innerText 61 | author = parseHtml(SeekNode($authorTitleNodeCombo, "")).child("a").innerText 62 | var mdata: MetaData = MetaData() 63 | mdata.name = title 64 | mdata.author = author 65 | mdata.coverUri = coverUri 66 | # Not the actual description/synopsis. 67 | #mdata.description = SeekNode(ndat.innerHtml, "
").innerText 68 | # Details is the first div, TOC is 2nd. 69 | mdata.description = parseHtml(SeekNode(pageContent, "
")).child("div").innerText 70 | return mdata 71 | --------------------------------------------------------------------------------